From 585bd82192c082341e5b84878488d3b4228df40f Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 21 Aug 2024 13:22:36 +0200 Subject: [PATCH 01/71] feat(ocis): add revisions purge tests and benchmarks Signed-off-by: jkoberg --- ocis/pkg/revisions/revisions.go | 118 +++++++++++++++++++---- ocis/pkg/revisions/revisions_test.go | 138 +++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 17 deletions(-) create mode 100644 ocis/pkg/revisions/revisions_test.go diff --git a/ocis/pkg/revisions/revisions.go b/ocis/pkg/revisions/revisions.go index 249dbf37e1..d2d1eac686 100644 --- a/ocis/pkg/revisions/revisions.go +++ b/ocis/pkg/revisions/revisions.go @@ -2,7 +2,6 @@ package revisions import ( - "errors" "fmt" "os" "path/filepath" @@ -26,25 +25,118 @@ type DelBlobstore interface { Delete(node *node.Node) error } -// PurgeRevisions removes all revisions from a storage provider. -func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool) error { +// PurgeRevisionsGlob removes all revisions from a storage provider using globbing. +func PurgeRevisionsGlob(pattern string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { if verbose { fmt.Println("Looking for nodes in", pattern) } - nodes, err := filepath.Glob(pattern) + ch := make(chan string) + go func() { + defer close(ch) + nodes, err := filepath.Glob(pattern) + if err != nil { + fmt.Println("error globbing", pattern, err) + return + } + + if len(nodes) == 0 { + fmt.Println("no nodes found. Double check storage path") + return + } + + for _, n := range nodes { + if _versionRegex.MatchString(n) { + ch <- n + } + } + }() + + return purgeRevisions(ch, bs, dryRun, verbose) +} + +// PurgeRevisionsWalk removes all revisions from a storage provider using walking. +func PurgeRevisionsWalk(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { + ch := make(chan string) + go func() { + defer close(ch) + err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error { + if err != nil { + fmt.Println("error walking", base, err) + return err + } + + if !_versionRegex.MatchString(info.Name()) { + return nil + } + + ch <- path + return nil + }) + if err != nil { + fmt.Println("error walking", base, err) + return + } + + }() + return purgeRevisions(ch, bs, dryRun, verbose) +} + +// PurgeRevisionsList removes all revisions from a storage provider using listing. +func PurgeRevisionsList(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { + ch := make(chan string) + go func() { + defer close(ch) + if err := listFolder(base, ch); err != nil { + fmt.Println("error listing", base, err) + return + } + }() + + return purgeRevisions(ch, bs, dryRun, verbose) +} + +func listFolder(path string, ch chan<- string) error { + children, err := os.ReadDir(path) if err != nil { return err } - if len(nodes) == 0 { - return errors.New("no nodes found, double check storage path") - } + for _, child := range children { + if child.IsDir() { + if err := listFolder(filepath.Join(path, child.Name()), ch); err != nil { + return err + } + } + if _versionRegex.MatchString(child.Name()) { + ch <- filepath.Join(path, child.Name()) + } + + } + return nil +} + +// PrintResults prints the results +func PrintResults(countFiles, countBlobs, countRevisions int, dryRun bool) error { + switch { + case countFiles == 0 && countRevisions == 0 && countBlobs == 0: + fmt.Println("❎ No revisions found. Storage provider is clean.") + case !dryRun: + fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + default: + fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + } + return nil +} + +func purgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) { countFiles := 0 countBlobs := 0 countRevisions := 0 - for _, d := range nodes { + + var err error + for d := range nodes { if !_versionRegex.MatchString(d) { continue } @@ -106,15 +198,7 @@ func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool) } } - switch { - case countFiles == 0 && countRevisions == 0 && countBlobs == 0: - fmt.Println("❎ No revisions found. Storage provider is clean.") - case !dryRun: - fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) - default: - fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) - } - return nil + return countFiles, countBlobs, countRevisions } func getBlobID(path string) (string, error) { diff --git a/ocis/pkg/revisions/revisions_test.go b/ocis/pkg/revisions/revisions_test.go new file mode 100644 index 0000000000..477ccbebd9 --- /dev/null +++ b/ocis/pkg/revisions/revisions_test.go @@ -0,0 +1,138 @@ +package revisions + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/google/uuid" + "github.com/test-go/testify/require" +) + +var ( + _basePath = "test_temp/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/" +) + +// func TestInit(t *testing.T) { +// initialize(10, 2) +// defer os.RemoveAll("test_temp") +// } + +func TestGlob30(t *testing.T) { testGlob(t, 10, 2) } +func TestGlob80(t *testing.T) { testGlob(t, 20, 3) } +func TestGlob250(t *testing.T) { testGlob(t, 50, 4) } +func TestGlob600(t *testing.T) { testGlob(t, 100, 5) } + +func TestWalk30(t *testing.T) { testWalk(t, 10, 2) } +func TestWalk80(t *testing.T) { testWalk(t, 20, 3) } +func TestWalk250(t *testing.T) { testWalk(t, 50, 4) } +func TestWalk600(t *testing.T) { testWalk(t, 100, 5) } + +func TestList30(t *testing.T) { testList(t, 10, 2) } +func TestList80(t *testing.T) { testList(t, 20, 3) } +func TestList250(t *testing.T) { testList(t, 50, 4) } +func TestList600(t *testing.T) { testList(t, 100, 5) } + +func BenchmarkGlob30(b *testing.B) { benchmarkGlob(b, 10, 2) } +func BenchmarkWalk30(b *testing.B) { benchmarkWalk(b, 10, 2) } +func BenchmarkList30(b *testing.B) { benchmarkList(b, 10, 2) } + +func BenchmarkGlob80(b *testing.B) { benchmarkGlob(b, 20, 3) } +func BenchmarkWalk80(b *testing.B) { benchmarkWalk(b, 20, 3) } +func BenchmarkList80(b *testing.B) { benchmarkList(b, 20, 3) } + +func BenchmarkGlob250(b *testing.B) { benchmarkGlob(b, 50, 4) } +func BenchmarkWalk250(b *testing.B) { benchmarkWalk(b, 50, 4) } +func BenchmarkList250(b *testing.B) { benchmarkList(b, 50, 4) } + +func BenchmarkGlob600(b *testing.B) { benchmarkGlob(b, 100, 5) } +func BenchmarkWalk600(b *testing.B) { benchmarkWalk(b, 100, 5) } +func BenchmarkList600(b *testing.B) { benchmarkList(b, 100, 5) } + +func BenchmarkGlob11000(b *testing.B) { benchmarkGlob(b, 1000, 10) } +func BenchmarkWalk11000(b *testing.B) { benchmarkWalk(b, 1000, 10) } +func BenchmarkList11000(b *testing.B) { benchmarkList(b, 1000, 10) } + +func BenchmarkGlob110000(b *testing.B) { benchmarkGlob(b, 10000, 10) } +func BenchmarkWalk110000(b *testing.B) { benchmarkWalk(b, 10000, 10) } +func BenchmarkList110000(b *testing.B) { benchmarkList(b, 10000, 10) } + +func benchmarkGlob(b *testing.B, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + for i := 0; i < b.N; i++ { + PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false) + } +} + +func benchmarkWalk(b *testing.B, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + for i := 0; i < b.N; i++ { + PurgeRevisionsWalk(_basePath, nil, false, false) + } +} + +func benchmarkList(b *testing.B, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + for i := 0; i < b.N; i++ { + PurgeRevisionsList(_basePath, nil, false, false) + } +} + +func testGlob(t *testing.T, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + _, _, revisions := PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false) + require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +} + +func testWalk(t *testing.T, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + _, _, revisions := PurgeRevisionsWalk(_basePath, nil, false, false) + require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +} + +func testList(t *testing.T, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + _, _, revisions := PurgeRevisionsList(_basePath, nil, false, false) + require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +} + +func initialize(numNodes int, numRevisions int) { + // create base path + if err := os.MkdirAll(_basePath, fs.ModePerm); err != nil { + fmt.Println("Error creating test_temp directory", err) + os.Exit(1) + } + + for i := 0; i < numNodes; i++ { + path := lookup.Pathify(uuid.New().String(), 4, 2) + dir := filepath.Dir(path) + if err := os.MkdirAll(_basePath+dir, fs.ModePerm); err != nil { + fmt.Println("Error creating test_temp directory", err) + os.Exit(1) + } + + if _, err := os.Create(_basePath + path); err != nil { + fmt.Println("Error creating file", err) + os.Exit(1) + } + for i := 0; i < numRevisions; i++ { + os.Create(_basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z") + } + } +} From 5ddd51ea84df403efb4da99b97dd151375e33ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Wed, 14 Aug 2024 18:21:15 +0200 Subject: [PATCH 02/71] refactor: simplify openInApp code and remove hostViewUrl (and edit) --- .../pkg/connector/fileconnector.go | 6 +- .../pkg/middleware/wopicontext.go | 2 - .../pkg/service/grpc/v0/service.go | 221 +++++++++--------- 3 files changed, 107 insertions(+), 122 deletions(-) diff --git a/services/collaboration/pkg/connector/fileconnector.go b/services/collaboration/pkg/connector/fileconnector.go index 0f9d42f67d..d23e93c735 100644 --- a/services/collaboration/pkg/connector/fileconnector.go +++ b/services/collaboration/pkg/connector/fileconnector.go @@ -1050,8 +1050,9 @@ func (f *FileConnector) CheckFileInfo(ctx context.Context) (*ConnectorResponse, // to get the folder we actually need to do a GetPath() request //BreadcrumbFolderName: path.Dir(statRes.Info.Path), - fileinfo.KeyHostViewURL: wopiContext.ViewAppUrl, - fileinfo.KeyHostEditURL: wopiContext.EditAppUrl, + // TODO: these URLs must point to ocis, which is hosting the editor's iframe + //fileinfo.KeyHostViewURL: wopiContext.ViewAppUrl, + //fileinfo.KeyHostEditURL: wopiContext.EditAppUrl, fileinfo.KeyEnableOwnerTermination: true, // only for collabora fileinfo.KeySupportsExtendedLockLength: true, @@ -1061,7 +1062,6 @@ func (f *FileConnector) CheckFileInfo(ctx context.Context) (*ConnectorResponse, fileinfo.KeySupportsDeleteFile: true, fileinfo.KeySupportsRename: true, - //fileinfo.KeyUserCanNotWriteRelative: true, fileinfo.KeyIsAnonymousUser: isAnonymousUser, fileinfo.KeyUserFriendlyName: userFriendlyName, fileinfo.KeyUserID: userId, diff --git a/services/collaboration/pkg/middleware/wopicontext.go b/services/collaboration/pkg/middleware/wopicontext.go index 1232da0317..e6b9dff0af 100644 --- a/services/collaboration/pkg/middleware/wopicontext.go +++ b/services/collaboration/pkg/middleware/wopicontext.go @@ -31,8 +31,6 @@ type WopiContext struct { FileReference *providerv1beta1.Reference User *userv1beta1.User ViewMode appproviderv1beta1.ViewMode - EditAppUrl string - ViewAppUrl string } // WopiContextAuthMiddleware will prepare an HTTP handler to be used as diff --git a/services/collaboration/pkg/service/grpc/v0/service.go b/services/collaboration/pkg/service/grpc/v0/service.go index 1568efe0ad..41a50df81a 100644 --- a/services/collaboration/pkg/service/grpc/v0/service.go +++ b/services/collaboration/pkg/service/grpc/v0/service.go @@ -2,10 +2,11 @@ package service import ( "context" - "fmt" + "errors" "net/url" "path" "strconv" + "strings" appproviderv1beta1 "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" @@ -80,147 +81,47 @@ func (s *Service) OpenInApp( Path: ".", } - // build a urlsafe and stable file reference that can be used for proxy routing, - // so that all sessions on one file end on the same office server - fileRef := helpers.HashResourceId(req.GetResourceInfo().GetId()) + logger := s.logger.With(). + Str("FileReference", providerFileRef.String()). + Str("ViewMode", req.GetViewMode().String()). + Str("Requester", user.GetId().String()). + Logger() // get the file extension to use the right wopi app url fileExt := path.Ext(req.GetResourceInfo().GetPath()) - var viewCommentAppURL string - var viewAppURL string - var editAppURL string - if viewCommentAppURLs, ok := s.appURLs["view_comment"]; ok { - if u, ok := viewCommentAppURLs[fileExt]; ok { - viewCommentAppURL = u - } - } - if viewAppURLs, ok := s.appURLs["view"]; ok { - if u, ok := viewAppURLs[fileExt]; ok { - viewAppURL = u - } - } - if editAppURLs, ok := s.appURLs["edit"]; ok { - if u, ok := editAppURLs[fileExt]; ok { - editAppURL = u - } - } - if editAppURL == "" && viewAppURL == "" && viewCommentAppURL == "" { - err := fmt.Errorf("OpenInApp: neither edit nor view app url found") - s.logger.Error(). - Err(err). - Str("FileReference", providerFileRef.String()). - Str("ViewMode", req.GetViewMode().String()). - Str("Requester", user.GetId().String()).Send() - return nil, err + // get the appURL we need to use + appURL := s.getAppUrl(fileExt, req.GetViewMode()) + if appURL == "" { + logger.Error().Msg("OpenInApp: neither edit nor view app URL found") + return nil, errors.New("neither edit nor view app URL found") } - if editAppURL == "" { - // assuming that an view action is always available in the /hosting/discovery manifest - // eg. Collabora does support viewing jpgs but no editing - // eg. OnlyOffice does support viewing pdfs but no editing - // there is no known case of supporting edit only without view - editAppURL = viewAppURL - } - if viewAppURL == "" { - // the URL of the end-user application in view mode when different (defaults to edit mod URL) - viewAppURL = editAppURL - } - // TODO: check if collabora will support an "edit" url in the future - if viewAppURL == "" && editAppURL == "" && viewCommentAppURL != "" { - // there are rare cases where neither view nor edit is supported but view_comment is - viewAppURL = viewCommentAppURL - // that can be the case for editable and viewable files - if req.GetViewMode() == appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE { - editAppURL = viewCommentAppURL - } - } - wopiSrcURL, err := url.Parse(s.config.Wopi.WopiSrc) + // append the parameters we need + appURL, err = s.addQueryToURL(appURL, req) if err != nil { - return nil, err - } - wopiSrcURL.Path = path.Join("wopi", "files", fileRef) - - addWopiSrcQueryParam := func(baseURL string) (string, error) { - u, err := url.Parse(baseURL) - if err != nil { - return "", err - } - - q := u.Query() - q.Add("WOPISrc", wopiSrcURL.String()) - - if s.config.Wopi.DisableChat { - q.Add("dchat", "1") - } - - lang := utils.ReadPlainFromOpaque(req.GetOpaque(), "lang") - - if lang != "" { - q.Add("ui", lang) // OnlyOffice - q.Add("lang", lang) // Collabora, Impact on the default document language of OnlyOffice - q.Add("UI_LLCC", lang) // Office365 - } - qs := q.Encode() - u.RawQuery = qs - - return u.String(), nil - } - - viewAppURL, err = addWopiSrcQueryParam(viewAppURL) - if err != nil { - s.logger.Error(). - Err(err). - Str("FileReference", providerFileRef.String()). - Str("ViewMode", req.GetViewMode().String()). - Str("Requester", user.GetId().String()). - Msg("OpenInApp: error parsing viewAppUrl") - return nil, err - } - editAppURL, err = addWopiSrcQueryParam(editAppURL) - if err != nil { - s.logger.Error(). - Err(err). - Str("FileReference", providerFileRef.String()). - Str("ViewMode", req.GetViewMode().String()). - Str("Requester", user.GetId().String()). - Msg("OpenInApp: error parsing editAppUrl") + logger.Error().Err(err).Msg("OpenInApp: error parsing appUrl") return nil, err } - appURL := viewAppURL - if req.GetViewMode() == appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE { - appURL = editAppURL - } - + // create the wopiContext and generate the token wopiContext := middleware.WopiContext{ AccessToken: req.GetAccessToken(), // it will be encrypted ViewOnlyToken: utils.ReadPlainFromOpaque(req.GetOpaque(), "viewOnlyToken"), FileReference: &providerFileRef, User: user, ViewMode: req.GetViewMode(), - EditAppUrl: editAppURL, - ViewAppUrl: viewAppURL, } accessToken, accessExpiration, err := middleware.GenerateWopiToken(wopiContext, s.config) if err != nil { - s.logger.Error(). - Err(err). - Str("FileReference", providerFileRef.String()). - Str("ViewMode", req.GetViewMode().String()). - Str("Requester", user.GetId().String()). - Msg("OpenInApp: error generating the token") + logger.Error().Err(err).Msg("OpenInApp: error generating the token") return &appproviderv1beta1.OpenInAppResponse{ Status: &rpcv1beta1.Status{Code: rpcv1beta1.Code_CODE_INTERNAL}, }, err } - s.logger.Debug(). - Str("FileReference", providerFileRef.String()). - Str("ViewMode", req.GetViewMode().String()). - Str("Requester", user.GetId().String()). - Msg("OpenInApp: success") + logger.Debug().Msg("OpenInApp: success") return &appproviderv1beta1.OpenInAppResponse{ Status: &rpcv1beta1.Status{Code: rpcv1beta1.Code_CODE_OK}, @@ -237,3 +138,89 @@ func (s *Service) OpenInApp( }, }, nil } + +// getAppUrlFor gets the appURL from the list of appURLs based on the +// action and file extension provided. If there is no match, an empty +// string will be returned. +func (s *Service) getAppUrlFor(action, fileExt string) string { + if actionURL, ok := s.appURLs[action]; ok { + if actionExtensionURL, ok := actionURL[fileExt]; ok { + return actionExtensionURL + } + } + return "" +} + +// getAppUrl will get the appURL that should be used based on the extension +// and the provided view mode. +// "view" urls will be chosen first, then if the view mode is "read/write", +// "edit" urls will be prioritized. Note that "view" url might be returned for +// "read/write" view mode if no "edit" url is found. +func (s *Service) getAppUrl(fileExt string, viewMode appproviderv1beta1.ViewMode) string { + // check view_comment action first (for collabora) + appURL := s.getAppUrlFor("view_comment", fileExt) + + // prioritize view action if possible + if viewAppURL := s.getAppUrlFor("view", fileExt); viewAppURL != "" { + appURL = viewAppURL + } + + // If read/write mode has been requested, prioritize edit action. + // Special case for collabora because it only provides one action per + // extension, + if viewMode == appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE || strings.ToLower(s.config.App.Name) == "collabora" { + if editAppURL := s.getAppUrlFor("edit", fileExt); editAppURL != "" { + appURL = editAppURL + } + } + + return appURL +} + +// addQueryToURL will add specific query parameters to the baseURL. These +// parameters are: +// * "WOPISrc" pointing to the requested resource in the OpenInAppRequest +// * "dchat" to disable the chat, based on configuration +// * "lang" (WOPI app dependent) with the language in the request. "lang" +// for collabora, "ui" for onlyoffice and "UI_LLCC" for the rest +func (s *Service) addQueryToURL(baseURL string, req *appproviderv1beta1.OpenInAppRequest) (string, error) { + u, err := url.Parse(baseURL) + if err != nil { + return "", err + } + + // build a urlsafe and stable file reference that can be used for proxy routing, + // so that all sessions on one file end on the same office server + fileRef := helpers.HashResourceId(req.GetResourceInfo().GetId()) + + wopiSrcURL, err := url.Parse(s.config.Wopi.WopiSrc) + if err != nil { + return "", err + } + wopiSrcURL.Path = path.Join("wopi", "files", fileRef) + + q := u.Query() + q.Add("WOPISrc", wopiSrcURL.String()) + + if s.config.Wopi.DisableChat { + q.Add("dchat", "1") + } + + lang := utils.ReadPlainFromOpaque(req.GetOpaque(), "lang") + + if lang != "" { + switch strings.ToLower(s.config.App.Name) { + case "collabora": + q.Add("lang", lang) + case "onlyoffice": + q.Add("ui", lang) + default: + q.Add("UI_LLCC", lang) + } + } + + qs := q.Encode() + u.RawQuery = qs + + return u.String(), nil +} From 3d287da5cea12e043bcf83f8b8610a97896770a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Fri, 16 Aug 2024 11:01:51 +0200 Subject: [PATCH 03/71] refactor: simplify getting the app url for collabora --- .../pkg/service/grpc/v0/service.go | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/services/collaboration/pkg/service/grpc/v0/service.go b/services/collaboration/pkg/service/grpc/v0/service.go index 41a50df81a..1458a4e27e 100644 --- a/services/collaboration/pkg/service/grpc/v0/service.go +++ b/services/collaboration/pkg/service/grpc/v0/service.go @@ -157,20 +157,28 @@ func (s *Service) getAppUrlFor(action, fileExt string) string { // "edit" urls will be prioritized. Note that "view" url might be returned for // "read/write" view mode if no "edit" url is found. func (s *Service) getAppUrl(fileExt string, viewMode appproviderv1beta1.ViewMode) string { - // check view_comment action first (for collabora) - appURL := s.getAppUrlFor("view_comment", fileExt) - // prioritize view action if possible - if viewAppURL := s.getAppUrlFor("view", fileExt); viewAppURL != "" { - appURL = viewAppURL - } + appURL := s.getAppUrlFor("view", fileExt) - // If read/write mode has been requested, prioritize edit action. - // Special case for collabora because it only provides one action per - // extension, - if viewMode == appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE || strings.ToLower(s.config.App.Name) == "collabora" { - if editAppURL := s.getAppUrlFor("edit", fileExt); editAppURL != "" { - appURL = editAppURL + if strings.ToLower(s.config.App.Name) == "collabora" { + // collabora provides only one action per extension. usual options + // are "view" (checked above), "edit" or "view_comment" (this last one + // is exclusive of collabora) + if appURL == "" { + if editURL := s.getAppUrlFor("edit", fileExt); editURL != "" { + return editURL + } + if commentURL := s.getAppUrlFor("view_comment", fileExt); commentURL != "" { + return commentURL + } + } + } else { + // If not collabora, there might be an edit action for the extension. + // If read/write mode has been requested, prioritize edit action. + if viewMode == appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE { + if editAppURL := s.getAppUrlFor("edit", fileExt); editAppURL != "" { + appURL = editAppURL + } } } From fe1712ebcae9df5444df12061a2bf5e0c3929d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Fri, 16 Aug 2024 14:08:09 +0200 Subject: [PATCH 04/71] fix: adjust unit tests --- .../pkg/connector/contentconnector_test.go | 12 +- .../pkg/connector/fileconnector_test.go | 6 +- .../pkg/service/grpc/v0/service_test.go | 175 ++++++------------ 3 files changed, 59 insertions(+), 134 deletions(-) diff --git a/services/collaboration/pkg/connector/contentconnector_test.go b/services/collaboration/pkg/connector/contentconnector_test.go index 38f308afcf..82ba9e9c42 100644 --- a/services/collaboration/pkg/connector/contentconnector_test.go +++ b/services/collaboration/pkg/connector/contentconnector_test.go @@ -52,10 +52,8 @@ var _ = Describe("ContentConnector", func() { }, Path: ".", }, - User: &userv1beta1.User{}, // Not used for now - ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, - EditAppUrl: "http://test.ex.prv/edit", - ViewAppUrl: "http://test.ex.prv/view", + User: &userv1beta1.User{}, // Not used for now + ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, } randomContent = "This is the content of the test.txt file" @@ -186,10 +184,8 @@ var _ = Describe("ContentConnector", func() { }, Path: ".", }, - User: &userv1beta1.User{}, // Not used for now - ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_VIEW_ONLY, - EditAppUrl: "http://test.ex.prv/edit", - ViewAppUrl: "http://test.ex.prv/view", + User: &userv1beta1.User{}, // Not used for now + ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_VIEW_ONLY, } ctx := middleware.WopiContextToCtx(context.Background(), wopiCtx) diff --git a/services/collaboration/pkg/connector/fileconnector_test.go b/services/collaboration/pkg/connector/fileconnector_test.go index 2f74ebb141..eb1ec5462c 100644 --- a/services/collaboration/pkg/connector/fileconnector_test.go +++ b/services/collaboration/pkg/connector/fileconnector_test.go @@ -82,9 +82,7 @@ var _ = Describe("FileConnector", func() { // }, //}, }, - ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, - EditAppUrl: "http://test.ex.prv/edit", - ViewAppUrl: "http://test.ex.prv/view", + ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, } }) @@ -1546,8 +1544,6 @@ var _ = Describe("FileConnector", func() { BaseFileName: "test.txt", BreadcrumbDocName: "test.txt", UserCanNotWriteRelative: false, - HostViewURL: "http://test.ex.prv/view", - HostEditURL: "http://test.ex.prv/edit", SupportsExtendedLockLength: true, SupportsGetLock: true, SupportsLocks: true, diff --git a/services/collaboration/pkg/service/grpc/v0/service_test.go b/services/collaboration/pkg/service/grpc/v0/service_test.go index e2361c090c..68f5aa1416 100644 --- a/services/collaboration/pkg/service/grpc/v0/service_test.go +++ b/services/collaboration/pkg/service/grpc/v0/service_test.go @@ -136,133 +136,66 @@ var _ = Describe("Discovery", func() { Expect(resp).To(BeNil()) }) - It("Success", func() { - ctx := context.Background() - nowTime := time.Now() + DescribeTable( + "Success", + func(appName, lang string, disableChat bool, expectedAppUrl string) { + ctx := context.Background() + nowTime := time.Now() - cfg.Wopi.WopiSrc = "https://wopiserver.test.prv" - cfg.Wopi.Secret = "my_supa_secret" + cfg.Wopi.WopiSrc = "https://wopiserver.test.prv" + cfg.Wopi.Secret = "my_supa_secret" + cfg.Wopi.DisableChat = disableChat + cfg.App.Name = appName - myself := &userv1beta1.User{ - Id: &userv1beta1.UserId{ - Idp: "myIdp", - OpaqueId: "opaque001", - Type: userv1beta1.UserType_USER_TYPE_PRIMARY, - }, - Username: "username", - } - - req := &appproviderv1beta1.OpenInAppRequest{ - ResourceInfo: &providerv1beta1.ResourceInfo{ - Id: &providerv1beta1.ResourceId{ - StorageId: "myStorage", - OpaqueId: "storageOpaque001", - SpaceId: "SpaceA", + myself := &userv1beta1.User{ + Id: &userv1beta1.UserId{ + Idp: "myIdp", + OpaqueId: "opaque001", + Type: userv1beta1.UserType_USER_TYPE_PRIMARY, }, - Path: "/path/to/file.docx", - }, - ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, - AccessToken: MintToken(myself, cfg.Wopi.Secret, nowTime), - } - req.Opaque = utils.AppendPlainToOpaque(req.Opaque, "lang", "de") + Username: "username", + } - gatewayClient.On("WhoAmI", mock.Anything, mock.Anything).Times(1).Return(&gatewayv1beta1.WhoAmIResponse{ - Status: status.NewOK(ctx), - User: myself, - }, nil) - - resp, err := srv.OpenInApp(ctx, req) - Expect(err).To(Succeed()) - Expect(resp.GetStatus().GetCode()).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(resp.GetAppUrl().GetMethod()).To(Equal("POST")) - Expect(resp.GetAppUrl().GetAppUrl()).To(Equal("https://test.server.prv/hosting/wopi/word/edit?UI_LLCC=de&WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&lang=de&ui=de")) - Expect(resp.GetAppUrl().GetFormParameters()["access_token_ttl"]).To(Equal(strconv.FormatInt(nowTime.Add(5*time.Hour).Unix()*1000, 10))) - }) - - It("Success", func() { - ctx := context.Background() - nowTime := time.Now() - - cfg.Wopi.WopiSrc = "https://wopiserver.test.prv" - cfg.Wopi.Secret = "my_supa_secret" - cfg.Wopi.DisableChat = true - - myself := &userv1beta1.User{ - Id: &userv1beta1.UserId{ - Idp: "myIdp", - OpaqueId: "opaque001", - Type: userv1beta1.UserType_USER_TYPE_PRIMARY, - }, - Username: "username", - } - - req := &appproviderv1beta1.OpenInAppRequest{ - ResourceInfo: &providerv1beta1.ResourceInfo{ - Id: &providerv1beta1.ResourceId{ - StorageId: "myStorage", - OpaqueId: "storageOpaque001", - SpaceId: "SpaceA", + req := &appproviderv1beta1.OpenInAppRequest{ + ResourceInfo: &providerv1beta1.ResourceInfo{ + Id: &providerv1beta1.ResourceId{ + StorageId: "myStorage", + OpaqueId: "storageOpaque001", + SpaceId: "SpaceA", + }, + Path: "/path/to/file.docx", }, - Path: "/path/to/file.docx", - }, - ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, - AccessToken: MintToken(myself, cfg.Wopi.Secret, nowTime), - } + ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, + AccessToken: MintToken(myself, cfg.Wopi.Secret, nowTime), + } + if lang != "" { + req.Opaque = utils.AppendPlainToOpaque(req.Opaque, "lang", lang) + } - gatewayClient.On("WhoAmI", mock.Anything, mock.Anything).Times(1).Return(&gatewayv1beta1.WhoAmIResponse{ - Status: status.NewOK(ctx), - User: myself, - }, nil) + gatewayClient.On("WhoAmI", mock.Anything, mock.Anything).Times(1).Return(&gatewayv1beta1.WhoAmIResponse{ + Status: status.NewOK(ctx), + User: myself, + }, nil) - resp, err := srv.OpenInApp(ctx, req) - Expect(err).To(Succeed()) - Expect(resp.GetStatus().GetCode()).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(resp.GetAppUrl().GetMethod()).To(Equal("POST")) - Expect(resp.GetAppUrl().GetAppUrl()).To(Equal("https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1")) - Expect(resp.GetAppUrl().GetFormParameters()["access_token_ttl"]).To(Equal(strconv.FormatInt(nowTime.Add(5*time.Hour).Unix()*1000, 10))) - }) - - It("Success", func() { - ctx := context.Background() - nowTime := time.Now() - - cfg.Wopi.WopiSrc = "https://wopiserver.test.prv" - cfg.Wopi.Secret = "my_supa_secret" - cfg.Wopi.DisableChat = true - - myself := &userv1beta1.User{ - Id: &userv1beta1.UserId{ - Idp: "myIdp", - OpaqueId: "opaque001", - Type: userv1beta1.UserType_USER_TYPE_PRIMARY, - }, - Username: "username", - } - - req := &appproviderv1beta1.OpenInAppRequest{ - ResourceInfo: &providerv1beta1.ResourceInfo{ - Id: &providerv1beta1.ResourceId{ - StorageId: "myStorage", - OpaqueId: "storageOpaque001", - SpaceId: "SpaceA", - }, - Path: "/path/to/file.docx", - }, - ViewMode: appproviderv1beta1.ViewMode_VIEW_MODE_READ_WRITE, - AccessToken: MintToken(myself, cfg.Wopi.Secret, nowTime), - } - - gatewayClient.On("WhoAmI", mock.Anything, mock.Anything).Times(1).Return(&gatewayv1beta1.WhoAmIResponse{ - Status: status.NewOK(ctx), - User: myself, - }, nil) - - resp, err := srv.OpenInApp(ctx, req) - Expect(err).To(Succeed()) - Expect(resp.GetStatus().GetCode()).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(resp.GetAppUrl().GetMethod()).To(Equal("POST")) - Expect(resp.GetAppUrl().GetAppUrl()).To(Equal("https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1")) - Expect(resp.GetAppUrl().GetFormParameters()["access_token_ttl"]).To(Equal(strconv.FormatInt(nowTime.Add(5*time.Hour).Unix()*1000, 10))) - }) + resp, err := srv.OpenInApp(ctx, req) + Expect(err).To(Succeed()) + Expect(resp.GetStatus().GetCode()).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(resp.GetAppUrl().GetMethod()).To(Equal("POST")) + Expect(resp.GetAppUrl().GetAppUrl()).To(Equal(expectedAppUrl)) + Expect(resp.GetAppUrl().GetFormParameters()["access_token_ttl"]).To(Equal(strconv.FormatInt(nowTime.Add(5*time.Hour).Unix()*1000, 10))) + }, + Entry("Microsoft chat no lang", "Microsoft", "", false, "https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e"), + Entry("Collabora chat no lang", "Collabora", "", false, "https://test.server.prv/hosting/wopi/word/view?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e"), + Entry("OnlyOffice chat no lang", "OnlyOffice", "", false, "https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e"), + Entry("Microsoft chat lang", "Microsoft", "de", false, "https://test.server.prv/hosting/wopi/word/edit?UI_LLCC=de&WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e"), + Entry("Collabora chat lang", "Collabora", "de", false, "https://test.server.prv/hosting/wopi/word/view?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&lang=de"), + Entry("OnlyOffice chat lang", "OnlyOffice", "de", false, "https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&ui=de"), + Entry("Microsoft no chat no lang", "Microsoft", "", true, "https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1"), + Entry("Collabora no chat no lang", "Collabora", "", true, "https://test.server.prv/hosting/wopi/word/view?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1"), + Entry("OnlyOffice no chat no lang", "OnlyOffice", "", true, "https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1"), + Entry("Microsoft no chat lang", "Microsoft", "de", true, "https://test.server.prv/hosting/wopi/word/edit?UI_LLCC=de&WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1"), + Entry("Collabora no chat lang", "Collabora", "de", true, "https://test.server.prv/hosting/wopi/word/view?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1&lang=de"), + Entry("OnlyOffice no chat lang", "OnlyOffice", "de", true, "https://test.server.prv/hosting/wopi/word/edit?WOPISrc=https%3A%2F%2Fwopiserver.test.prv%2Fwopi%2Ffiles%2F2f6ec18696dd1008106749bd94106e5cfad5c09e15de7b77088d03843e71b43e&dchat=1&ui=de"), + ) }) }) From ff9f8e78db0ecffc9bc05cc29a9a9b953539f544 Mon Sep 17 00:00:00 2001 From: ownClouders Date: Thu, 22 Aug 2024 00:55:34 +0000 Subject: [PATCH 05/71] [tx] updated from transifex --- .../l10n/locale/de/LC_MESSAGES/activitylog.po | 8 +-- .../l10n/locale/sq/LC_MESSAGES/activitylog.po | 62 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 services/activitylog/pkg/service/l10n/locale/sq/LC_MESSAGES/activitylog.po diff --git a/services/activitylog/pkg/service/l10n/locale/de/LC_MESSAGES/activitylog.po b/services/activitylog/pkg/service/l10n/locale/de/LC_MESSAGES/activitylog.po index 3881bb3c5e..603aea197c 100644 --- a/services/activitylog/pkg/service/l10n/locale/de/LC_MESSAGES/activitylog.po +++ b/services/activitylog/pkg/service/l10n/locale/de/LC_MESSAGES/activitylog.po @@ -11,7 +11,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: EMAIL\n" -"POT-Creation-Date: 2024-08-21 05:51+0000\n" +"POT-Creation-Date: 2024-08-22 00:53+0000\n" "PO-Revision-Date: 2024-08-20 10:13+0000\n" "Last-Translator: Martin , 2024\n" "Language-Team: German (https://app.transifex.com/owncloud-org/teams/6149/de/)\n" @@ -23,15 +23,15 @@ msgstr "" #: pkg/service/response.go:23 msgid "{user} added {resource} to {space}" -msgstr "{user} hat {resource} zum {space} hinzugefügt" +msgstr "{user} hat {resource} zu {space} hinzugefügt" #: pkg/service/response.go:31 msgid "{user} added {sharee} as member of {space}" -msgstr "{user} hat {sharee} als Mitglied zum {space} hinzugefügt" +msgstr "{user} hat {sharee} als Mitglied zu {space} hinzugefügt" #: pkg/service/response.go:24 msgid "{user} deleted {resource} from {space}" -msgstr "{user} hat {resource} vom {space} gelöscht" +msgstr "{user} hat {resource} in {space} gelöscht" #: pkg/service/response.go:25 msgid "{user} moved {resource} to {space}" diff --git a/services/activitylog/pkg/service/l10n/locale/sq/LC_MESSAGES/activitylog.po b/services/activitylog/pkg/service/l10n/locale/sq/LC_MESSAGES/activitylog.po new file mode 100644 index 0000000000..1e8ae34cf6 --- /dev/null +++ b/services/activitylog/pkg/service/l10n/locale/sq/LC_MESSAGES/activitylog.po @@ -0,0 +1,62 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +# Translators: +# Besnik Bleta , 2024 +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: EMAIL\n" +"POT-Creation-Date: 2024-08-22 00:53+0000\n" +"PO-Revision-Date: 2024-08-20 10:13+0000\n" +"Last-Translator: Besnik Bleta , 2024\n" +"Language-Team: Albanian (https://app.transifex.com/owncloud-org/teams/6149/sq/)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: sq\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: pkg/service/response.go:23 +msgid "{user} added {resource} to {space}" +msgstr "{user} shtoi {resource} te {space}" + +#: pkg/service/response.go:31 +msgid "{user} added {sharee} as member of {space}" +msgstr "{user} shtoi {sharee} si anëtar të {space}" + +#: pkg/service/response.go:24 +msgid "{user} deleted {resource} from {space}" +msgstr "{user} fshiu {resource} nga {space}" + +#: pkg/service/response.go:25 +msgid "{user} moved {resource} to {space}" +msgstr "{user} e shpuri {resource} te {space}" + +#: pkg/service/response.go:30 +msgid "{user} removed link to {resource}" +msgstr "{user} hoqi lidhjen për te {resource}" + +#: pkg/service/response.go:28 +msgid "{user} removed {sharee} from {resource}" +msgstr "{user} hoqi {sharee} nga {resource}" + +#: pkg/service/response.go:32 +msgid "{user} removed {sharee} from {space}" +msgstr "{user} hoqi {sharee} nga {space}" + +#: pkg/service/response.go:26 +msgid "{user} renamed {oldResource} to {resource}" +msgstr "{user} riemërtoi {oldResource} si {resource}" + +#: pkg/service/response.go:29 +msgid "{user} shared {resource} via link" +msgstr "{user} ndau me të tjerët {resource} përmes lidhjeje" + +#: pkg/service/response.go:27 +msgid "{user} shared {resource} with {sharee}" +msgstr "{user} ndau {resource} me {sharee}" From 474f7f753ce4880a24db79bceb3e150ca7d02e94 Mon Sep 17 00:00:00 2001 From: prashant-gurung899 Date: Thu, 22 Aug 2024 12:21:22 +0545 Subject: [PATCH 06/71] add test to check activity after language translation-german Signed-off-by: prashant-gurung899 --- tests/acceptance/features/apiActivities/activities.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/acceptance/features/apiActivities/activities.feature b/tests/acceptance/features/apiActivities/activities.feature index f9d1d414e5..dc930cb3ef 100644 --- a/tests/acceptance/features/apiActivities/activities.feature +++ b/tests/acceptance/features/apiActivities/activities.feature @@ -1261,7 +1261,7 @@ Feature: check activities "required": ["message","variables"], "properties": { "message": { - "const": "{user} hat {resource} zum {space} hinzugefügt" + "const": "{user} hat {resource} zu {space} hinzugefügt" }, "variables": { "type": "object", From 3cdc63820100c34a8adef0069a1b60b415da2d36 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 22 Aug 2024 09:28:02 +0200 Subject: [PATCH 07/71] remove code for store service Signed-off-by: Christian Richter --- ocis-pkg/config/config.go | 2 - ocis-pkg/config/defaultconfig.go | 2 - ocis/pkg/command/services.go | 6 - services/ocs/pkg/server/http/server.go | 25 +- services/proxy/pkg/command/server.go | 25 +- services/store/Makefile | 45 --- services/store/cmd/store/main.go | 19 - services/store/docker/Dockerfile.linux.amd64 | 19 - services/store/docker/Dockerfile.linux.arm | 19 - services/store/docker/Dockerfile.linux.arm64 | 19 - services/store/docker/manifest.tmpl | 22 -- services/store/pkg/command/health.go | 54 --- services/store/pkg/command/root.go | 34 -- services/store/pkg/command/server.go | 94 ----- services/store/pkg/command/version.go | 50 --- services/store/pkg/config/config.go | 28 -- services/store/pkg/config/debug.go | 9 - .../pkg/config/defaults/defaultconfig.go | 75 ---- services/store/pkg/config/grpc.go | 10 - services/store/pkg/config/log.go | 9 - services/store/pkg/config/parser/parse.go | 38 -- services/store/pkg/config/service.go | 6 - services/store/pkg/config/tracing.go | 21 -- services/store/pkg/logging/logging.go | 17 - services/store/pkg/metrics/metrics.go | 45 --- services/store/pkg/server/debug/option.go | 50 --- services/store/pkg/server/debug/server.go | 59 ---- services/store/pkg/server/grpc/option.go | 80 ----- services/store/pkg/server/grpc/server.go | 46 --- services/store/pkg/service/v0/option.go | 63 ---- services/store/pkg/service/v0/service.go | 332 ------------------ services/store/pkg/store/options.go | 20 -- services/store/pkg/store/store.go | 189 ---------- services/store/reflex.conf | 3 - 34 files changed, 16 insertions(+), 1519 deletions(-) delete mode 100644 services/store/Makefile delete mode 100644 services/store/cmd/store/main.go delete mode 100644 services/store/docker/Dockerfile.linux.amd64 delete mode 100644 services/store/docker/Dockerfile.linux.arm delete mode 100644 services/store/docker/Dockerfile.linux.arm64 delete mode 100644 services/store/docker/manifest.tmpl delete mode 100644 services/store/pkg/command/health.go delete mode 100644 services/store/pkg/command/root.go delete mode 100644 services/store/pkg/command/server.go delete mode 100644 services/store/pkg/command/version.go delete mode 100644 services/store/pkg/config/config.go delete mode 100644 services/store/pkg/config/debug.go delete mode 100644 services/store/pkg/config/defaults/defaultconfig.go delete mode 100644 services/store/pkg/config/grpc.go delete mode 100644 services/store/pkg/config/log.go delete mode 100644 services/store/pkg/config/parser/parse.go delete mode 100644 services/store/pkg/config/service.go delete mode 100644 services/store/pkg/config/tracing.go delete mode 100644 services/store/pkg/logging/logging.go delete mode 100644 services/store/pkg/metrics/metrics.go delete mode 100644 services/store/pkg/server/debug/option.go delete mode 100644 services/store/pkg/server/debug/server.go delete mode 100644 services/store/pkg/server/grpc/option.go delete mode 100644 services/store/pkg/server/grpc/server.go delete mode 100644 services/store/pkg/service/v0/option.go delete mode 100644 services/store/pkg/service/v0/service.go delete mode 100644 services/store/pkg/store/options.go delete mode 100644 services/store/pkg/store/store.go delete mode 100644 services/store/reflex.conf diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 79926807fb..84f4119acd 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -38,7 +38,6 @@ import ( storageshares "github.com/owncloud/ocis/v2/services/storage-shares/pkg/config" storagesystem "github.com/owncloud/ocis/v2/services/storage-system/pkg/config" storageusers "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" - store "github.com/owncloud/ocis/v2/services/store/pkg/config" thumbnails "github.com/owncloud/ocis/v2/services/thumbnails/pkg/config" userlog "github.com/owncloud/ocis/v2/services/userlog/pkg/config" users "github.com/owncloud/ocis/v2/services/users/pkg/config" @@ -117,7 +116,6 @@ type Config struct { StoragePublicLink *storagepublic.Config `yaml:"storage_public"` StorageShares *storageshares.Config `yaml:"storage_shares"` StorageUsers *storageusers.Config `yaml:"storage_users"` - Store *store.Config `yaml:"store"` Thumbnails *thumbnails.Config `yaml:"thumbnails"` Userlog *userlog.Config `yaml:"userlog"` Users *users.Config `yaml:"users"` diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index bd15668a76..bca348b4ef 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -37,7 +37,6 @@ import ( storageshares "github.com/owncloud/ocis/v2/services/storage-shares/pkg/config/defaults" storageSystem "github.com/owncloud/ocis/v2/services/storage-system/pkg/config/defaults" storageusers "github.com/owncloud/ocis/v2/services/storage-users/pkg/config/defaults" - store "github.com/owncloud/ocis/v2/services/store/pkg/config/defaults" thumbnails "github.com/owncloud/ocis/v2/services/thumbnails/pkg/config/defaults" userlog "github.com/owncloud/ocis/v2/services/userlog/pkg/config/defaults" users "github.com/owncloud/ocis/v2/services/users/pkg/config/defaults" @@ -90,7 +89,6 @@ func DefaultConfig() *Config { StorageShares: storageshares.DefaultConfig(), StorageSystem: storageSystem.DefaultConfig(), StorageUsers: storageusers.DefaultConfig(), - Store: store.DefaultConfig(), Thumbnails: thumbnails.DefaultConfig(), Userlog: userlog.DefaultConfig(), Users: users.DefaultConfig(), diff --git a/ocis/pkg/command/services.go b/ocis/pkg/command/services.go index 0920dd4e3b..378bd4a6f1 100644 --- a/ocis/pkg/command/services.go +++ b/ocis/pkg/command/services.go @@ -44,7 +44,6 @@ import ( storageshares "github.com/owncloud/ocis/v2/services/storage-shares/pkg/command" storagesystem "github.com/owncloud/ocis/v2/services/storage-system/pkg/command" storageusers "github.com/owncloud/ocis/v2/services/storage-users/pkg/command" - store "github.com/owncloud/ocis/v2/services/store/pkg/command" thumbnails "github.com/owncloud/ocis/v2/services/thumbnails/pkg/command" userlog "github.com/owncloud/ocis/v2/services/userlog/pkg/command" users "github.com/owncloud/ocis/v2/services/users/pkg/command" @@ -234,11 +233,6 @@ var svccmds = []register.Command{ cfg.StorageUsers.Commons = cfg.Commons }) }, - func(cfg *config.Config) *cli.Command { - return ServiceCommand(cfg, cfg.Store.Service.Name, store.GetCommands(cfg.Store), func(c *config.Config) { - cfg.Store.Commons = cfg.Commons - }) - }, func(cfg *config.Config) *cli.Command { return ServiceCommand(cfg, cfg.Thumbnails.Service.Name, thumbnails.GetCommands(cfg.Thumbnails), func(c *config.Config) { cfg.Thumbnails.Commons = cfg.Commons diff --git a/services/ocs/pkg/server/http/server.go b/services/ocs/pkg/server/http/server.go index d8efb98a8f..d09f168bd1 100644 --- a/services/ocs/pkg/server/http/server.go +++ b/services/ocs/pkg/server/http/server.go @@ -9,7 +9,6 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/service/http" "github.com/owncloud/ocis/v2/ocis-pkg/version" svc "github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0" - ocisstore "github.com/owncloud/ocis/v2/services/store/pkg/store" "go-micro.dev/v4" microstore "go-micro.dev/v4/store" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" @@ -38,22 +37,14 @@ func Server(opts ...Option) (http.Service, error) { } var signingKeyStore microstore.Store - if options.Config.SigningKeys.Store == "ocisstoreservice" { - signingKeyStore = ocisstore.NewStore( - microstore.Nodes(options.Config.SigningKeys.Nodes...), - microstore.Database("proxy"), - microstore.Table("signing-keys"), - ) - } else { - signingKeyStore = store.Create( - store.Store(options.Config.SigningKeys.Store), - store.TTL(options.Config.SigningKeys.TTL), - microstore.Nodes(options.Config.SigningKeys.Nodes...), - microstore.Database("proxy"), - microstore.Table("signing-keys"), - store.Authentication(options.Config.SigningKeys.AuthUsername, options.Config.SigningKeys.AuthPassword), - ) - } + signingKeyStore = store.Create( + store.Store(options.Config.SigningKeys.Store), + store.TTL(options.Config.SigningKeys.TTL), + microstore.Nodes(options.Config.SigningKeys.Nodes...), + microstore.Database("proxy"), + microstore.Table("signing-keys"), + store.Authentication(options.Config.SigningKeys.AuthUsername, options.Config.SigningKeys.AuthPassword), + ) handle := svc.NewService( svc.Logger(options.Logger), diff --git a/services/proxy/pkg/command/server.go b/services/proxy/pkg/command/server.go index 56919149cb..b0033c0157 100644 --- a/services/proxy/pkg/command/server.go +++ b/services/proxy/pkg/command/server.go @@ -37,7 +37,6 @@ import ( "github.com/owncloud/ocis/v2/services/proxy/pkg/staticroutes" "github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend" "github.com/owncloud/ocis/v2/services/proxy/pkg/userroles" - ocisstore "github.com/owncloud/ocis/v2/services/store/pkg/store" "github.com/urfave/cli/v2" "go-micro.dev/v4/selector" microstore "go-micro.dev/v4/store" @@ -67,22 +66,14 @@ func Server(cfg *config.Config) *cli.Command { ) var signingKeyStore microstore.Store - if cfg.PreSignedURL.SigningKeys.Store == "ocisstoreservice" { - signingKeyStore = ocisstore.NewStore( - microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...), - microstore.Database("proxy"), - microstore.Table("signing-keys"), - ) - } else { - signingKeyStore = store.Create( - store.Store(cfg.PreSignedURL.SigningKeys.Store), - store.TTL(cfg.PreSignedURL.SigningKeys.TTL), - microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...), - microstore.Database("proxy"), - microstore.Table("signing-keys"), - store.Authentication(cfg.PreSignedURL.SigningKeys.AuthUsername, cfg.PreSignedURL.SigningKeys.AuthPassword), - ) - } + signingKeyStore = store.Create( + store.Store(cfg.PreSignedURL.SigningKeys.Store), + store.TTL(cfg.PreSignedURL.SigningKeys.TTL), + microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...), + microstore.Database("proxy"), + microstore.Table("signing-keys"), + store.Authentication(cfg.PreSignedURL.SigningKeys.AuthUsername, cfg.PreSignedURL.SigningKeys.AuthPassword), + ) logger := logging.Configure(cfg.Service.Name, cfg.Log) traceProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) diff --git a/services/store/Makefile b/services/store/Makefile deleted file mode 100644 index c56f393a12..0000000000 --- a/services/store/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -SHELL := bash -NAME := store - -include ../../.make/recursion.mk - -############ tooling ############ -ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI -include ../../.bingo/Variables.mk -endif - -############ go tooling ############ -include ../../.make/go.mk - -############ release ############ -include ../../.make/release.mk - -############ docs generate ############ -include ../../.make/docs.mk - -.PHONY: docs-generate -docs-generate: config-docs-generate \ - grpc-docs-generate - -############ generate ############ -include ../../.make/generate.mk - -.PHONY: ci-go-generate -ci-go-generate: protobuf # CI runs ci-node-generate automatically before this target - -.PHONY: ci-node-generate -ci-node-generate: - - -############ protobuf ############ -include ../../.make/protobuf.mk - -.PHONY: protobuf -protobuf: buf-generate - -############ licenses ############ -.PHONY: ci-node-check-licenses -ci-node-check-licenses: - -.PHONY: ci-node-save-licenses -ci-node-save-licenses: diff --git a/services/store/cmd/store/main.go b/services/store/cmd/store/main.go deleted file mode 100644 index 4384dfab8c..0000000000 --- a/services/store/cmd/store/main.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "syscall" - - "github.com/owncloud/ocis/v2/services/store/pkg/command" - "github.com/owncloud/ocis/v2/services/store/pkg/config/defaults" -) - -func main() { - cfg := defaults.DefaultConfig() - cfg.Context, _ = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP) - if err := command.Execute(cfg); err != nil { - os.Exit(1) - } -} diff --git a/services/store/docker/Dockerfile.linux.amd64 b/services/store/docker/Dockerfile.linux.amd64 deleted file mode 100644 index 5ae75d704d..0000000000 --- a/services/store/docker/Dockerfile.linux.amd64 +++ /dev/null @@ -1,19 +0,0 @@ -FROM amd64/alpine:latest - -RUN apk update && \ - apk upgrade && \ - apk add ca-certificates mailcap && \ - rm -rf /var/cache/apk/* && \ - echo 'hosts: files dns' >| /etc/nsswitch.conf - -LABEL maintainer="ownCloud GmbH " \ - org.label-schema.name="oCIS Store" \ - org.label-schema.vendor="ownCloud GmbH" \ - org.label-schema.schema-version="1.0" - -EXPOSE 9460 - -ENTRYPOINT ["/usr/bin/ocis-store"] -CMD ["server"] - -COPY bin/ocis-store /usr/bin/ocis-store diff --git a/services/store/docker/Dockerfile.linux.arm b/services/store/docker/Dockerfile.linux.arm deleted file mode 100644 index 3a48681fd0..0000000000 --- a/services/store/docker/Dockerfile.linux.arm +++ /dev/null @@ -1,19 +0,0 @@ -FROM arm32v6/alpine:latest - -RUN apk update && \ - apk upgrade && \ - apk add ca-certificates mailcap && \ - rm -rf /var/cache/apk/* && \ - echo 'hosts: files dns' >| /etc/nsswitch.conf - -LABEL maintainer="ownCloud GmbH " \ - org.label-schema.name="oCIS Store" \ - org.label-schema.vendor="ownCloud GmbH" \ - org.label-schema.schema-version="1.0" - -EXPOSE 9460 - -ENTRYPOINT ["/usr/bin/ocis-store"] -CMD ["server"] - -COPY bin/ocis-store /usr/bin/ocis-store diff --git a/services/store/docker/Dockerfile.linux.arm64 b/services/store/docker/Dockerfile.linux.arm64 deleted file mode 100644 index 2e57fea4f2..0000000000 --- a/services/store/docker/Dockerfile.linux.arm64 +++ /dev/null @@ -1,19 +0,0 @@ -FROM arm64v8/alpine:latest - -RUN apk update && \ - apk upgrade && \ - apk add ca-certificates mailcap && \ - rm -rf /var/cache/apk/* && \ - echo 'hosts: files dns' >| /etc/nsswitch.conf - -LABEL maintainer="ownCloud GmbH " \ - org.label-schema.name="oCIS Store" \ - org.label-schema.vendor="ownCloud GmbH" \ - org.label-schema.schema-version="1.0" - -EXPOSE 9460 - -ENTRYPOINT ["/usr/bin/ocis-store"] -CMD ["server"] - -COPY bin/ocis-store /usr/bin/ocis-store diff --git a/services/store/docker/manifest.tmpl b/services/store/docker/manifest.tmpl deleted file mode 100644 index faef5d9659..0000000000 --- a/services/store/docker/manifest.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -image: owncloud/ocis-store:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} -{{#if build.tags}} -tags: -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: owncloud/ocis-store:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64 - platform: - architecture: amd64 - os: linux - - image: owncloud/ocis-store:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64 - platform: - architecture: arm64 - variant: v8 - os: linux - - image: owncloud/ocis-store:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm - platform: - architecture: arm - variant: v6 - os: linux diff --git a/services/store/pkg/command/health.go b/services/store/pkg/command/health.go deleted file mode 100644 index 6d2abc564f..0000000000 --- a/services/store/pkg/command/health.go +++ /dev/null @@ -1,54 +0,0 @@ -package command - -import ( - "fmt" - "net/http" - - "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" - "github.com/owncloud/ocis/v2/services/store/pkg/config" - "github.com/owncloud/ocis/v2/services/store/pkg/config/parser" - "github.com/owncloud/ocis/v2/services/store/pkg/logging" - "github.com/urfave/cli/v2" -) - -// Health is the entrypoint for the health command. -func Health(cfg *config.Config) *cli.Command { - return &cli.Command{ - Name: "health", - Usage: "check health status", - Category: "info", - Before: func(c *cli.Context) error { - return configlog.ReturnError(parser.ParseConfig(cfg)) - }, - Action: func(c *cli.Context) error { - logger := logging.Configure(cfg.Service.Name, cfg.Log) - - resp, err := http.Get( - fmt.Sprintf( - "http://%s/healthz", - cfg.Debug.Addr, - ), - ) - - if err != nil { - logger.Fatal(). - Err(err). - Msg("Failed to request health check") - } - - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - logger.Fatal(). - Int("code", resp.StatusCode). - Msg("Health seems to be in bad state") - } - - logger.Debug(). - Int("code", resp.StatusCode). - Msg("Health got a good state") - - return nil - }, - } -} diff --git a/services/store/pkg/command/root.go b/services/store/pkg/command/root.go deleted file mode 100644 index d105b28f9f..0000000000 --- a/services/store/pkg/command/root.go +++ /dev/null @@ -1,34 +0,0 @@ -package command - -import ( - "os" - - "github.com/owncloud/ocis/v2/ocis-pkg/clihelper" - "github.com/owncloud/ocis/v2/services/store/pkg/config" - "github.com/urfave/cli/v2" -) - -// GetCommands provides all commands for this service -func GetCommands(cfg *config.Config) cli.Commands { - return []*cli.Command{ - // start this service - Server(cfg), - - // interaction with this service - - // infos about this service - Health(cfg), - Version(cfg), - } -} - -// Execute is the entry point for the ocis-store command. -func Execute(cfg *config.Config) error { - app := clihelper.DefaultApp(&cli.App{ - Name: "store", - Usage: "Service to store values for ocis services", - Commands: GetCommands(cfg), - }) - - return app.RunContext(cfg.Context, os.Args) -} diff --git a/services/store/pkg/command/server.go b/services/store/pkg/command/server.go deleted file mode 100644 index 8971d75517..0000000000 --- a/services/store/pkg/command/server.go +++ /dev/null @@ -1,94 +0,0 @@ -package command - -import ( - "context" - "fmt" - - "github.com/oklog/run" - - "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" - ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" - "github.com/owncloud/ocis/v2/ocis-pkg/tracing" - "github.com/owncloud/ocis/v2/ocis-pkg/version" - "github.com/owncloud/ocis/v2/services/store/pkg/config" - "github.com/owncloud/ocis/v2/services/store/pkg/config/parser" - "github.com/owncloud/ocis/v2/services/store/pkg/logging" - "github.com/owncloud/ocis/v2/services/store/pkg/metrics" - "github.com/owncloud/ocis/v2/services/store/pkg/server/debug" - "github.com/owncloud/ocis/v2/services/store/pkg/server/grpc" - "github.com/urfave/cli/v2" -) - -// Server is the entrypoint for the server command. -func Server(cfg *config.Config) *cli.Command { - return &cli.Command{ - Name: "server", - Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name), - Category: "server", - Before: func(c *cli.Context) error { - return configlog.ReturnFatal(parser.ParseConfig(cfg)) - }, - Action: func(c *cli.Context) error { - logger := logging.Configure(cfg.Service.Name, cfg.Log) - traceProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) - if err != nil { - return err - } - cfg.GrpcClient, err = ogrpc.NewClient( - append(ogrpc.GetClientOptions(cfg.GRPCClientTLS), ogrpc.WithTraceProvider(traceProvider))..., - ) - if err != nil { - return err - } - - var ( - gr = run.Group{} - ctx, cancel = context.WithCancel(c.Context) - metrics = metrics.New() - ) - - defer cancel() - - metrics.BuildInfo.WithLabelValues(version.GetString()).Set(1) - - { - server := grpc.Server( - grpc.Logger(logger), - grpc.Context(ctx), - grpc.Config(cfg), - grpc.Metrics(metrics), - grpc.TraceProvider(traceProvider), - ) - - gr.Add(server.Run, func(err error) { - logger.Error(). - Err(err). - Str("server", "grpc"). - Msg("Shutting down server") - - cancel() - }) - } - - { - server, err := debug.Server( - debug.Logger(logger), - debug.Context(ctx), - debug.Config(cfg), - ) - - if err != nil { - logger.Error().Err(err).Str("server", "debug").Msg("Failed to initialize server") - return err - } - - gr.Add(server.ListenAndServe, func(_ error) { - _ = server.Shutdown(ctx) - cancel() - }) - } - - return gr.Run() - }, - } -} diff --git a/services/store/pkg/command/version.go b/services/store/pkg/command/version.go deleted file mode 100644 index 81fcb3533a..0000000000 --- a/services/store/pkg/command/version.go +++ /dev/null @@ -1,50 +0,0 @@ -package command - -import ( - "fmt" - "os" - - "github.com/owncloud/ocis/v2/ocis-pkg/registry" - "github.com/owncloud/ocis/v2/ocis-pkg/version" - - tw "github.com/olekukonko/tablewriter" - "github.com/owncloud/ocis/v2/services/store/pkg/config" - "github.com/urfave/cli/v2" -) - -// Version prints the service versions of all running instances. -func Version(cfg *config.Config) *cli.Command { - return &cli.Command{ - Name: "version", - Usage: "print the version of this binary and the running service instances", - Category: "info", - Action: func(c *cli.Context) error { - fmt.Println("Version: " + version.GetString()) - fmt.Printf("Compiled: %s\n", version.Compiled()) - fmt.Println("") - - reg := registry.GetRegistry() - services, err := reg.GetService(cfg.GRPC.Namespace + "." + cfg.Service.Name) - if err != nil { - fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err)) - return err - } - - if len(services) == 0 { - fmt.Println("No running " + cfg.Service.Name + " service found.") - return nil - } - - table := tw.NewWriter(os.Stdout) - table.SetHeader([]string{"Version", "Address", "Id"}) - table.SetAutoFormatHeaders(false) - for _, s := range services { - for _, n := range s.Nodes { - table.Append([]string{s.Version, n.Address, n.Id}) - } - } - table.Render() - return nil - }, - } -} diff --git a/services/store/pkg/config/config.go b/services/store/pkg/config/config.go deleted file mode 100644 index aac6ba8e22..0000000000 --- a/services/store/pkg/config/config.go +++ /dev/null @@ -1,28 +0,0 @@ -package config - -import ( - "context" - - "github.com/owncloud/ocis/v2/ocis-pkg/shared" - "go-micro.dev/v4/client" -) - -// Config combines all available configuration parts. -type Config struct { - Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service - - Service Service `yaml:"-"` - - Tracing *Tracing `yaml:"tracing"` - Log *Log `yaml:"log"` - Debug Debug `yaml:"debug"` - - GRPC GRPCConfig `yaml:"grpc"` - - GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"` - GrpcClient client.Client `yaml:"-"` - - Datapath string `yaml:"data_path" env:"STORE_DATA_PATH" desc:"The directory where the filesystem storage will store ocis settings. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/store." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - - Context context.Context `yaml:"-"` -} diff --git a/services/store/pkg/config/debug.go b/services/store/pkg/config/debug.go deleted file mode 100644 index a3b3d31dbb..0000000000 --- a/services/store/pkg/config/debug.go +++ /dev/null @@ -1,9 +0,0 @@ -package config - -// Debug defines the available debug configuration. -type Debug struct { - Addr string `yaml:"addr" env:"STORE_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Token string `yaml:"token" env:"STORE_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Pprof bool `yaml:"pprof" env:"STORE_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Zpages bool `yaml:"zpages" env:"STORE_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` -} diff --git a/services/store/pkg/config/defaults/defaultconfig.go b/services/store/pkg/config/defaults/defaultconfig.go deleted file mode 100644 index c07c3161e2..0000000000 --- a/services/store/pkg/config/defaults/defaultconfig.go +++ /dev/null @@ -1,75 +0,0 @@ -package defaults - -import ( - "path" - - "github.com/owncloud/ocis/v2/ocis-pkg/config/defaults" - "github.com/owncloud/ocis/v2/ocis-pkg/structs" - "github.com/owncloud/ocis/v2/services/store/pkg/config" -) - -// FullDefaultConfig returns a fully initialized default configuration -func FullDefaultConfig() *config.Config { - cfg := DefaultConfig() - EnsureDefaults(cfg) - Sanitize(cfg) - return cfg -} - -// DefaultConfig returns a basic default configuration -func DefaultConfig() *config.Config { - return &config.Config{ - Debug: config.Debug{ - Addr: "127.0.0.1:9464", - Token: "", - Pprof: false, - Zpages: false, - }, - GRPC: config.GRPCConfig{ - Addr: "127.0.0.1:9460", - Namespace: "com.owncloud.api", - }, - Service: config.Service{ - Name: "store", - }, - Datapath: path.Join(defaults.BaseDataPath(), "store"), - } -} - -// EnsureDefaults adds default values to the configuration if they are not set yet -func EnsureDefaults(cfg *config.Config) { - // provide with defaults for shared logging, since we need a valid destination address for "envdecode". - if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { - cfg.Log = &config.Log{ - Level: cfg.Commons.Log.Level, - Pretty: cfg.Commons.Log.Pretty, - Color: cfg.Commons.Log.Color, - File: cfg.Commons.Log.File, - } - } else if cfg.Log == nil { - cfg.Log = &config.Log{} - } - // provide with defaults for shared tracing, since we need a valid destination address for "envdecode". - if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { - cfg.Tracing = &config.Tracing{ - Enabled: cfg.Commons.Tracing.Enabled, - Type: cfg.Commons.Tracing.Type, - Endpoint: cfg.Commons.Tracing.Endpoint, - Collector: cfg.Commons.Tracing.Collector, - } - } else if cfg.Tracing == nil { - cfg.Tracing = &config.Tracing{} - } - - if cfg.GRPCClientTLS == nil && cfg.Commons != nil { - cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS) - } - if cfg.GRPC.TLS == nil && cfg.Commons != nil { - cfg.GRPC.TLS = structs.CopyOrZeroValue(cfg.Commons.GRPCServiceTLS) - } -} - -// Sanitize sanitized the configuration -func Sanitize(cfg *config.Config) { - // nothing to sanitize here atm -} diff --git a/services/store/pkg/config/grpc.go b/services/store/pkg/config/grpc.go deleted file mode 100644 index 3a2d24bd90..0000000000 --- a/services/store/pkg/config/grpc.go +++ /dev/null @@ -1,10 +0,0 @@ -package config - -import "github.com/owncloud/ocis/v2/ocis-pkg/shared" - -// GRPCConfig defines the available grpc configuration. -type GRPCConfig struct { - Addr string `yaml:"addr" env:"STORE_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Namespace string `yaml:"-"` - TLS *shared.GRPCServiceTLS `yaml:"tls"` -} diff --git a/services/store/pkg/config/log.go b/services/store/pkg/config/log.go deleted file mode 100644 index 66d1bbf4e9..0000000000 --- a/services/store/pkg/config/log.go +++ /dev/null @@ -1,9 +0,0 @@ -package config - -// Log defines the available log configuration. -type Log struct { - Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;STORE_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;STORE_LOG_PRETTY" desc:"Activates pretty log output." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;STORE_LOG_COLOR" desc:"Activates colorized log output." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - File string `mapstructure:"file" env:"OCIS_LOG_FILE;STORE_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` -} diff --git a/services/store/pkg/config/parser/parse.go b/services/store/pkg/config/parser/parse.go deleted file mode 100644 index ebe57d4150..0000000000 --- a/services/store/pkg/config/parser/parse.go +++ /dev/null @@ -1,38 +0,0 @@ -package parser - -import ( - "errors" - - ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" - "github.com/owncloud/ocis/v2/services/store/pkg/config" - "github.com/owncloud/ocis/v2/services/store/pkg/config/defaults" - - "github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode" -) - -// ParseConfig loads configuration from known paths. -func ParseConfig(cfg *config.Config) error { - err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg) - if err != nil { - return err - } - - defaults.EnsureDefaults(cfg) - - // load all env variables relevant to the config in the current context. - if err := envdecode.Decode(cfg); err != nil { - // no environment variable set for this config is an expected "error" - if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) { - return err - } - } - - // sanitize config - defaults.Sanitize(cfg) - - return Validate(cfg) -} - -func Validate(cfg *config.Config) error { - return nil -} diff --git a/services/store/pkg/config/service.go b/services/store/pkg/config/service.go deleted file mode 100644 index d1eac383f0..0000000000 --- a/services/store/pkg/config/service.go +++ /dev/null @@ -1,6 +0,0 @@ -package config - -// Service defines the available service configuration. -type Service struct { - Name string `yaml:"-"` -} diff --git a/services/store/pkg/config/tracing.go b/services/store/pkg/config/tracing.go deleted file mode 100644 index 4ee04982e7..0000000000 --- a/services/store/pkg/config/tracing.go +++ /dev/null @@ -1,21 +0,0 @@ -package config - -import "github.com/owncloud/ocis/v2/ocis-pkg/tracing" - -// Tracing defines the available tracing configuration. -type Tracing struct { - Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORE_TRACING_ENABLED" desc:"Activates tracing." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORE_TRACING_TYPE" desc:"The type of tracing. Defaults to '', which is the same as 'jaeger'. Allowed tracing types are 'jaeger' and '' as of now." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORE_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` - Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORE_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset." introductionVersion:"pre5.0" deprecationVersion:"5.0" removalVersion:"7.0.0" deprecationInfo:"The store service is optional and will be removed."` -} - -// Convert Tracing to the tracing package's Config struct. -func (t Tracing) Convert() tracing.Config { - return tracing.Config{ - Enabled: t.Enabled, - Type: t.Type, - Endpoint: t.Endpoint, - Collector: t.Collector, - } -} diff --git a/services/store/pkg/logging/logging.go b/services/store/pkg/logging/logging.go deleted file mode 100644 index b65d932592..0000000000 --- a/services/store/pkg/logging/logging.go +++ /dev/null @@ -1,17 +0,0 @@ -package logging - -import ( - "github.com/owncloud/ocis/v2/ocis-pkg/log" - "github.com/owncloud/ocis/v2/services/store/pkg/config" -) - -// Configure initializes a service-specific logger instance. -func Configure(name string, cfg *config.Log) log.Logger { - return log.NewLogger( - log.Name(name), - log.Level(cfg.Level), - log.Pretty(cfg.Pretty), - log.Color(cfg.Color), - log.File(cfg.File), - ) -} diff --git a/services/store/pkg/metrics/metrics.go b/services/store/pkg/metrics/metrics.go deleted file mode 100644 index 67c46e77fe..0000000000 --- a/services/store/pkg/metrics/metrics.go +++ /dev/null @@ -1,45 +0,0 @@ -package metrics - -import "github.com/prometheus/client_golang/prometheus" - -var ( - // Namespace defines the namespace for the defines metrics. - Namespace = "ocis" - - // Subsystem defines the subsystem for the defines metrics. - Subsystem = "store" -) - -// Metrics defines the available metrics of this service. -type Metrics struct { - // Counter *prometheus.CounterVec - BuildInfo *prometheus.GaugeVec -} - -// New initializes the available metrics. -func New() *Metrics { - m := &Metrics{ - // Counter: prometheus.NewCounterVec(prometheus.CounterOpts{ - // Namespace: Namespace, - // Subsystem: Subsystem, - // Name: "greet_total", - // Help: "How many greeting requests processed", - // }, []string{}), - BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: Namespace, - Subsystem: Subsystem, - Name: "build_info", - Help: "Build Information", - }, []string{"version"}), - } - - // prometheus.Register( - // m.Counter, - // ) - - _ = prometheus.Register( - m.BuildInfo, - ) - - return m -} diff --git a/services/store/pkg/server/debug/option.go b/services/store/pkg/server/debug/option.go deleted file mode 100644 index 1613a3ef87..0000000000 --- a/services/store/pkg/server/debug/option.go +++ /dev/null @@ -1,50 +0,0 @@ -package debug - -import ( - "context" - - "github.com/owncloud/ocis/v2/ocis-pkg/log" - "github.com/owncloud/ocis/v2/services/store/pkg/config" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - Logger log.Logger - Context context.Context - Config *config.Config -} - -// newOptions initializes the available default options. -func newOptions(opts ...Option) Options { - opt := Options{} - - for _, o := range opts { - o(&opt) - } - - return opt -} - -// Logger provides a function to set the logger option. -func Logger(val log.Logger) Option { - return func(o *Options) { - o.Logger = val - } -} - -// Context provides a function to set the context option. -func Context(val context.Context) Option { - return func(o *Options) { - o.Context = val - } -} - -// Config provides a function to set the config option. -func Config(val *config.Config) Option { - return func(o *Options) { - o.Config = val - } -} diff --git a/services/store/pkg/server/debug/server.go b/services/store/pkg/server/debug/server.go deleted file mode 100644 index f10bba3014..0000000000 --- a/services/store/pkg/server/debug/server.go +++ /dev/null @@ -1,59 +0,0 @@ -package debug - -import ( - "io" - "net/http" - - "github.com/owncloud/ocis/v2/ocis-pkg/service/debug" - "github.com/owncloud/ocis/v2/ocis-pkg/version" - "github.com/owncloud/ocis/v2/services/store/pkg/config" -) - -// Server initializes the debug service and server. -func Server(opts ...Option) (*http.Server, error) { - options := newOptions(opts...) - - return debug.NewService( - debug.Logger(options.Logger), - debug.Name(options.Config.Service.Name), - debug.Version(version.GetString()), - debug.Address(options.Config.Debug.Addr), - debug.Token(options.Config.Debug.Token), - debug.Pprof(options.Config.Debug.Pprof), - debug.Zpages(options.Config.Debug.Zpages), - debug.Health(health(options.Config)), - debug.Ready(ready(options.Config)), - ), nil -} - -// health implements the health check. -func health(cfg *config.Config) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - - // TODO: check if services are up and running - - _, err := io.WriteString(w, http.StatusText(http.StatusOK)) - // io.WriteString should not fail but if it does, we want to know. - if err != nil { - panic(err) - } - } -} - -// ready implements the ready check. -func ready(cfg *config.Config) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - - // TODO: check if services are up and running - - _, err := io.WriteString(w, http.StatusText(http.StatusOK)) - // io.WriteString should not fail but if it does, we want to know. - if err != nil { - panic(err) - } - } -} diff --git a/services/store/pkg/server/grpc/option.go b/services/store/pkg/server/grpc/option.go deleted file mode 100644 index 5e0d7549a4..0000000000 --- a/services/store/pkg/server/grpc/option.go +++ /dev/null @@ -1,80 +0,0 @@ -package grpc - -import ( - "context" - - "github.com/owncloud/ocis/v2/ocis-pkg/log" - "github.com/owncloud/ocis/v2/services/store/pkg/config" - "github.com/owncloud/ocis/v2/services/store/pkg/metrics" - "go.opentelemetry.io/otel/trace" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - Name string - Logger log.Logger - Context context.Context - Config *config.Config - Metrics *metrics.Metrics - TraceProvider trace.TracerProvider -} - -// newOptions initializes the available default options. -func newOptions(opts ...Option) Options { - opt := Options{} - - for _, o := range opts { - o(&opt) - } - - return opt -} - -// Name provides a name for the service. -func Name(val string) Option { - return func(o *Options) { - o.Name = val - } -} - -// Logger provides a function to set the logger option. -func Logger(val log.Logger) Option { - return func(o *Options) { - o.Logger = val - } -} - -// Context provides a function to set the context option. -func Context(val context.Context) Option { - return func(o *Options) { - o.Context = val - } -} - -// Config provides a function to set the config option. -func Config(val *config.Config) Option { - return func(o *Options) { - o.Config = val - } -} - -// Metrics provides a function to set the metrics option. -func Metrics(val *metrics.Metrics) Option { - return func(o *Options) { - o.Metrics = val - } -} - -// TraceProvider provides a function to configure the trace provider -func TraceProvider(traceProvider trace.TracerProvider) Option { - return func(o *Options) { - if traceProvider != nil { - o.TraceProvider = traceProvider - } else { - o.TraceProvider = trace.NewNoopTracerProvider() - } - } -} diff --git a/services/store/pkg/server/grpc/server.go b/services/store/pkg/server/grpc/server.go deleted file mode 100644 index c765fd45a4..0000000000 --- a/services/store/pkg/server/grpc/server.go +++ /dev/null @@ -1,46 +0,0 @@ -package grpc - -import ( - "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" - "github.com/owncloud/ocis/v2/ocis-pkg/version" - storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" - svc "github.com/owncloud/ocis/v2/services/store/pkg/service/v0" -) - -// Server initializes a new go-micro service ready to run -func Server(opts ...Option) grpc.Service { - options := newOptions(opts...) - - service, err := grpc.NewServiceWithClient( - options.Config.GrpcClient, - grpc.TLSEnabled(options.Config.GRPC.TLS.Enabled), - grpc.TLSCert( - options.Config.GRPC.TLS.Cert, - options.Config.GRPC.TLS.Key, - ), - grpc.Namespace(options.Config.GRPC.Namespace), - grpc.Name(options.Config.Service.Name), - grpc.Version(version.GetString()), - grpc.Context(options.Context), - grpc.Address(options.Config.GRPC.Addr), - grpc.Logger(options.Logger), - grpc.TraceProvider(options.TraceProvider), - ) - if err != nil { - options.Logger.Fatal().Err(err).Msg("Error creating store service") - return grpc.Service{} - } - - hdlr, err := svc.New( - svc.Logger(options.Logger), - svc.Config(options.Config), - ) - if err != nil { - options.Logger.Fatal().Err(err).Msg("could not initialize service handler") - } - if err = storesvc.RegisterStoreHandler(service.Server(), hdlr); err != nil { - options.Logger.Fatal().Err(err).Msg("could not register service handler") - } - - return service -} diff --git a/services/store/pkg/service/v0/option.go b/services/store/pkg/service/v0/option.go deleted file mode 100644 index 46bea78990..0000000000 --- a/services/store/pkg/service/v0/option.go +++ /dev/null @@ -1,63 +0,0 @@ -package service - -import ( - "github.com/owncloud/ocis/v2/ocis-pkg/log" - "github.com/owncloud/ocis/v2/services/store/pkg/config" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - Logger log.Logger - Config *config.Config - - Database, Table string - Nodes []string -} - -func newOptions(opts ...Option) Options { - opt := Options{} - - for _, o := range opts { - o(&opt) - } - - return opt -} - -// Logger provides a function to set the logger option. -func Logger(val log.Logger) Option { - return func(o *Options) { - o.Logger = val - } -} - -// Database configures the database option. -func Database(val *config.Config) Option { - return func(o *Options) { - o.Config = val - } -} - -// Table configures the Table option. -func Table(val *config.Config) Option { - return func(o *Options) { - o.Config = val - } -} - -// Nodes configures the Nodes option. -func Nodes(val *config.Config) Option { - return func(o *Options) { - o.Config = val - } -} - -// Config configures the Config option. -func Config(val *config.Config) Option { - return func(o *Options) { - o.Config = val - } -} diff --git a/services/store/pkg/service/v0/service.go b/services/store/pkg/service/v0/service.go deleted file mode 100644 index 2645342a28..0000000000 --- a/services/store/pkg/service/v0/service.go +++ /dev/null @@ -1,332 +0,0 @@ -package service - -import ( - "context" - "fmt" - "os" - "path/filepath" - - "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - "github.com/owncloud/ocis/v2/ocis-pkg/log" - storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" - storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" - "github.com/owncloud/ocis/v2/services/store/pkg/config" - merrors "go-micro.dev/v4/errors" - "google.golang.org/protobuf/encoding/protojson" -) - -// BleveDocument wraps the generated Record.Metadata and adds a property that is used to distinguish documents in the index. -type BleveDocument struct { - Metadata map[string]*storemsg.Field `json:"metadata"` - Database string `json:"database"` - Table string `json:"table"` -} - -// New returns a new instance of Service -func New(opts ...Option) (s *Service, err error) { - options := newOptions(opts...) - logger := options.Logger - cfg := options.Config - - recordsDir := filepath.Join(cfg.Datapath, "databases") - { - var fi os.FileInfo - if fi, err = os.Stat(recordsDir); err != nil { - if os.IsNotExist(err) { - // create store directory - if err = os.MkdirAll(recordsDir, 0700); err != nil { - return nil, err - } - } - } else if !fi.IsDir() { - return nil, fmt.Errorf("%s is not a directory", recordsDir) - } - } - - indexMapping := bleve.NewIndexMapping() - // keep all symbols in terms to allow exact matching, eg. emails - indexMapping.DefaultAnalyzer = keyword.Name - - s = &Service{ - id: cfg.GRPC.Namespace + "." + cfg.Service.Name, - log: logger, - Config: cfg, - } - - indexDir := filepath.Join(cfg.Datapath, "index.bleve") - // for now recreate index on every start - if err = os.RemoveAll(indexDir); err != nil { - return nil, err - } - if s.index, err = bleve.New(indexDir, indexMapping); err != nil { - return - } - if err = s.indexRecords(recordsDir); err != nil { - return nil, err - } - return -} - -// Service implements the AccountsServiceHandler interface -type Service struct { - id string - log log.Logger - Config *config.Config - index bleve.Index -} - -// Read implements the StoreHandler interface. -func (s *Service) Read(c context.Context, rreq *storesvc.ReadRequest, rres *storesvc.ReadResponse) error { - if len(rreq.Key) != 0 { - id := getID(rreq.Options.Database, rreq.Options.Table, rreq.Key) - file := filepath.Join(s.Config.Datapath, "databases", id) - - var data []byte - rec := &storemsg.Record{} - data, err := os.ReadFile(file) - if err != nil { - return merrors.NotFound(s.id, "could not read record") - } - - if err = protojson.Unmarshal(data, rec); err != nil { - return merrors.InternalServerError(s.id, "could not unmarshal record") - } - - rres.Records = append(rres.Records, rec) - return nil - } - - s.log.Info().Interface("request", rreq).Msg("read request") - if rreq.Options.Where != nil { - // build bleve query - // execute search - // fetch the actual record if there's a hit - dtq := bleve.NewTermQuery(rreq.Options.Database) - ttq := bleve.NewTermQuery(rreq.Options.Table) - dtq.SetField("database") - ttq.SetField("table") - - query := bleve.NewConjunctionQuery(dtq, ttq) - for k, v := range rreq.Options.Where { - ntq := bleve.NewTermQuery(v.Value) - ntq.SetField("metadata." + k + ".value") - query.AddQuery(ntq) - } - - searchRequest := bleve.NewSearchRequest(query) - var searchResult *bleve.SearchResult - searchResult, err := s.index.Search(searchRequest) - if err != nil { - s.log.Error().Err(err).Msg("could not execute bleve search") - return merrors.InternalServerError(s.id, "could not execute bleve search: %v", err.Error()) - } - - for _, hit := range searchResult.Hits { - rec := &storemsg.Record{} - - dest := filepath.Join(s.Config.Datapath, "databases", hit.ID) - - var data []byte - data, err := os.ReadFile(dest) - s.log.Info().Str("path", dest).Interface("hit", hit).Msgf("hit info") - if err != nil { - s.log.Info().Str("path", dest).Interface("hit", hit).Msgf("file not found") - return merrors.NotFound(s.id, "could not read record") - } - - if err = protojson.Unmarshal(data, rec); err != nil { - return merrors.InternalServerError(s.id, "could not unmarshal record") - } - - rres.Records = append(rres.Records, rec) - } - return nil - } - - return merrors.InternalServerError(s.id, "neither id nor metadata present") -} - -// Write implements the StoreHandler interface. -func (s *Service) Write(c context.Context, wreq *storesvc.WriteRequest, wres *storesvc.WriteResponse) error { - id := getID(wreq.Options.Database, wreq.Options.Table, wreq.Record.Key) - file := filepath.Join(s.Config.Datapath, "databases", id) - - var bytes []byte - bytes, err := protojson.Marshal(wreq.Record) - if err != nil { - return merrors.InternalServerError(s.id, "could not marshal record") - } - - err = os.MkdirAll(filepath.Dir(file), 0700) - if err != nil { - return err - } - err = os.WriteFile(file, bytes, 0600) - if err != nil { - return merrors.InternalServerError(s.id, "could not write record") - } - - doc := BleveDocument{ - Metadata: wreq.Record.Metadata, - Database: wreq.Options.Database, - Table: wreq.Options.Table, - } - if err := s.index.Index(id, doc); err != nil { - s.log.Error().Err(err).Interface("document", doc).Msg("could not index record metadata") - return err - } - - return nil -} - -// Delete implements the StoreHandler interface. -func (s *Service) Delete(c context.Context, dreq *storesvc.DeleteRequest, dres *storesvc.DeleteResponse) error { - id := getID(dreq.Options.Database, dreq.Options.Table, dreq.Key) - file := filepath.Join(s.Config.Datapath, "databases", id) - if err := os.Remove(file); err != nil { - if os.IsNotExist(err) { - return merrors.NotFound(s.id, "could not find record") - } - - return merrors.InternalServerError(s.id, "could not delete record") - } - - if err := s.index.Delete(id); err != nil { - s.log.Error().Err(err).Str("id", id).Msg("could not remove record from index") - return merrors.InternalServerError(s.id, "could not remove record from index") - } - - return nil -} - -// List implements the StoreHandler interface. -func (s *Service) List(context.Context, *storesvc.ListRequest, storesvc.Store_ListStream) error { - return nil -} - -// Databases implements the StoreHandler interface. -func (s *Service) Databases(c context.Context, dbreq *storesvc.DatabasesRequest, dbres *storesvc.DatabasesResponse) error { - file := filepath.Join(s.Config.Datapath, "databases") - f, err := os.Open(file) - if err != nil { - return merrors.InternalServerError(s.id, "could not open database directory") - } - defer f.Close() - - dnames, err := f.Readdirnames(0) - if err != nil { - return merrors.InternalServerError(s.id, "could not read database directory") - } - - dbres.Databases = dnames - return nil -} - -// Tables implements the StoreHandler interface. -func (s *Service) Tables(ctx context.Context, in *storesvc.TablesRequest, out *storesvc.TablesResponse) error { - file := filepath.Join(s.Config.Datapath, "databases", in.Database) - f, err := os.Open(file) - if err != nil { - return merrors.InternalServerError(s.id, "could not open tables directory") - } - defer f.Close() - - tnames, err := f.Readdirnames(0) - if err != nil { - return merrors.InternalServerError(s.id, "could not read tables directory") - } - - out.Tables = tnames - return nil -} - -// TODO sanitize key. As it may contain invalid characters, such as slashes. -// file: /tmp/ocis-store/databases/{database}/{table}/{record.key}. -func getID(database string, table string, key string) string { - // TODO sanitize input. - return filepath.Join(database, table, key) -} - -func (s Service) indexRecords(recordsDir string) (err error) { - - // TODO use filepath.Walk to clean up code - rh, err := os.Open(recordsDir) - if err != nil { - return merrors.InternalServerError(s.id, "could not open database directory") - } - defer rh.Close() - - dbs, err := rh.Readdirnames(0) - if err != nil { - return merrors.InternalServerError(s.id, "could not read databases directory") - } - - for i := range dbs { - tp := filepath.Join(s.Config.Datapath, "databases", dbs[i]) - th, err := os.Open(tp) - if err != nil { - s.log.Error().Err(err).Str("database", dbs[i]).Msg("could not open database directory") - continue - } - defer th.Close() - - tables, err := th.Readdirnames(0) - if err != nil { - s.log.Error().Err(err).Str("database", dbs[i]).Msg("could not read database directory") - continue - } - - for j := range tables { - - tp := filepath.Join(s.Config.Datapath, "databases", dbs[i], tables[j]) - kh, err := os.Open(tp) - if err != nil { - s.log.Error().Err(err).Str("database", dbs[i]).Str("table", tables[i]).Msg("could not open table directory") - continue - } - defer kh.Close() - - keys, err := kh.Readdirnames(0) - if err != nil { - s.log.Error().Err(err).Str("database", dbs[i]).Str("table", tables[i]).Msg("could not read table directory") - continue - } - - for k := range keys { - - id := getID(dbs[i], tables[j], keys[k]) - kp := filepath.Join(s.Config.Datapath, "databases", id) - - // read record - var data []byte - rec := &storemsg.Record{} - data, err = os.ReadFile(kp) - if err != nil { - s.log.Error().Err(err).Str("id", id).Msg("could not read record") - continue - } - - if err = protojson.Unmarshal(data, rec); err != nil { - s.log.Error().Err(err).Str("id", id).Msg("could not unmarshal record") - continue - } - - // index record - doc := BleveDocument{ - Metadata: rec.Metadata, - Database: dbs[i], - Table: tables[j], - } - if err := s.index.Index(id, doc); err != nil { - s.log.Error().Err(err).Interface("document", doc).Str("id", id).Msg("could not index record metadata") - continue - } - - s.log.Debug().Str("id", id).Msg("indexed record") - } - } - } - - return -} diff --git a/services/store/pkg/store/options.go b/services/store/pkg/store/options.go deleted file mode 100644 index 949425c795..0000000000 --- a/services/store/pkg/store/options.go +++ /dev/null @@ -1,20 +0,0 @@ -package store - -import ( - "context" - - "go-micro.dev/v4/client" - "go-micro.dev/v4/store" -) - -type grpcClientContextKey struct{} - -// WithGRPCClient sets the grpc client -func WithGRPCClient(c client.Client) store.Option { - return func(o *store.Options) { - if o.Context == nil { - o.Context = context.Background() - } - o.Context = context.WithValue(o.Context, grpcClientContextKey{}, c) - } -} diff --git a/services/store/pkg/store/store.go b/services/store/pkg/store/store.go deleted file mode 100644 index 47e7ee57df..0000000000 --- a/services/store/pkg/store/store.go +++ /dev/null @@ -1,189 +0,0 @@ -package store - -import ( - "context" - "errors" - "net/http" - "time" - - "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" - storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" - storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" - "go-micro.dev/v4/client" - merrors "go-micro.dev/v4/errors" - "go-micro.dev/v4/logger" - "go-micro.dev/v4/store" - "go-micro.dev/v4/util/cmd" -) - -// DefaultDatabase is the namespace that the store -// will use if no namespace is provided. -var ( - DefaultDatabase = "proxy" - DefaultTable = "signing-keys" -) - -type oss struct { - ctx context.Context - options store.Options - svc storesvc.StoreService -} - -func init() { - cmd.DefaultStores["ocisstoreservice"] = NewStore -} - -// NewStore returns a micro store.Store wrapper to access the micro store service. -// It only implements the minimal Read and Write options that are used by the proxy and ocs services -// Deprecated: use a different micro.Store implementation like nats-js-ks -func NewStore(opts ...store.Option) store.Store { - options := store.Options{ - Context: context.Background(), - Database: DefaultDatabase, - Table: DefaultTable, - Logger: logger.DefaultLogger, - Nodes: []string{"com.owncloud.api.store"}, - } - - for _, o := range opts { - o(&options) - } - - c, ok := options.Context.Value(grpcClientContextKey{}).(client.Client) - if !ok { - var err error - c, err = grpc.NewClient() - if err != nil { - options.Logger.Fields(map[string]interface{}{"err": err}).Log(logger.FatalLevel, "ocisstoreservice could not create new grpc client") - } - } - svc := storesvc.NewStoreService(options.Nodes[0], c) - - s := &oss{ - ctx: context.Background(), - options: options, - svc: svc, - } - - return s -} - -// Init initializes the store by configuring a storeservice and initializing -// a grpc client if it has not been passed as a context option. -func (s *oss) Init(opts ...store.Option) error { - for _, o := range opts { - o(&s.options) - } - return s.configure() -} - -func (s *oss) configure() error { - c, ok := s.options.Context.Value(grpcClientContextKey{}).(client.Client) - if !ok { - var err error - c, err = grpc.NewClient() - if err != nil { - logger.Fatal("ocisstoreservice could not create new grpc client:", err) - } - } - if len(s.options.Nodes) < 1 { - return errors.New("no node configured") - } - s.svc = storesvc.NewStoreService(s.options.Nodes[0], c) - return nil -} - -// Options allows you to view the current options. -func (s *oss) Options() store.Options { - return s.options -} - -// Read takes a single key name and optional ReadOptions. It returns matching []*Record or an error. -// Only database and table options are used. -func (s *oss) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { - options := store.ReadOptions{ - Database: s.options.Database, - Table: s.options.Table, - } - - for _, o := range opts { - o(&options) - } - - res, err := s.svc.Read(context.Background(), &storesvc.ReadRequest{ - Options: &storemsg.ReadOptions{ - Database: options.Database, - Table: options.Table, - // Other options ignored - }, - Key: key, - }) - - if err != nil { - e := merrors.Parse(err.Error()) - if e.Code == http.StatusNotFound { - return nil, store.ErrNotFound - } - return nil, err - } - - records := make([]*store.Record, 0, len(res.Records)) - for _, record := range res.Records { - r := &store.Record{ - Key: record.Key, - Value: record.Value, - Metadata: map[string]interface{}{}, - Expiry: time.Duration(record.Expiry), - } - for k, v := range record.Metadata { - r.Metadata[k] = v.Value // we only support string - } - records = append(records, r) - } - return records, nil -} - -// Write() writes a record to the store, and returns an error if the record was not written. -func (s *oss) Write(r *store.Record, opts ...store.WriteOption) error { - options := store.WriteOptions{ - Database: s.options.Database, - Table: s.options.Table, - } - - for _, o := range opts { - o(&options) - } - _, err := s.svc.Write(context.Background(), &storesvc.WriteRequest{ - Options: &storemsg.WriteOptions{ - Database: options.Database, - Table: options.Table, - }, - Record: &storemsg.Record{ - Key: r.Key, - Value: r.Value, - // No expiry supported - }, - }) - - return err -} - -// Delete is not implemented for the ocis service -func (s *oss) Delete(key string, opts ...store.DeleteOption) error { - return errors.ErrUnsupported -} - -// List is not implemented for the ocis service -func (s *oss) List(opts ...store.ListOption) ([]string, error) { - return nil, errors.ErrUnsupported -} - -// Close does nothing -func (s *oss) Close() error { - return nil -} - -// String returns the name of the implementation. -func (s *oss) String() string { - return "ocisstoreservice" -} diff --git a/services/store/reflex.conf b/services/store/reflex.conf deleted file mode 100644 index eecf7cc27c..0000000000 --- a/services/store/reflex.conf +++ /dev/null @@ -1,3 +0,0 @@ -# backend --r '^(cmd|pkg)/.*\.go$' -R '^node_modules/' -s -- sh -c 'make bin/ocis-store-debug && bin/ocis-store-debug --log-level debug server --debug-pprof --debug-zpages' -' From a1f250b231fc64ce040c6a03e9c70f4b0b71cf9e Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 22 Aug 2024 09:40:20 +0200 Subject: [PATCH 08/71] remove docs for store-service Signed-off-by: Christian Richter --- docs/services/general-info/port-ranges.md | 4 ++-- docs/services/store/.gitignore | 1 - docs/services/store/_index.md | 18 ------------------ docs/services/store/configuration.md | 15 --------------- 4 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 docs/services/store/.gitignore delete mode 100644 docs/services/store/_index.md delete mode 100644 docs/services/store/configuration.md diff --git a/docs/services/general-info/port-ranges.md b/docs/services/general-info/port-ranges.md index ae027bbee8..14f9c52957 100644 --- a/docs/services/general-info/port-ranges.md +++ b/docs/services/general-info/port-ranges.md @@ -49,7 +49,7 @@ We also suggest using the last port in your extensions' range as a debug/metrics | 9180-9184 | FREE (formerly used by accounts) | | 9185-9189 | [thumbnails]({{< ref "../thumbnails/_index.md" >}}) | | 9190-9194 | [settings]({{< ref "../settings/_index.md" >}}) | -| 9195-9197 | [activitylog]({{< ref "../activitylog/_index.md" >}}) | +| 9195-9197 | [activitylog]({{< ref "../activitylog/_index.md" >}}) | | 9198-9199 | [auth-service]({{< ref "../auth-service/_index.md" >}}) | | 9200-9204 | [proxy]({{< ref "../proxy/_index.md" >}}) | | 9205-9209 | [proxy]({{< ref "../proxy/_index.md" >}}) | @@ -103,7 +103,7 @@ We also suggest using the last port in your extensions' range as a debug/metrics | 9445-9449 | FREE | | 9450-9454 | FREE | | 9455-9459 | FREE | -| 9460-9464 | [store]({{< ref "../store/_index.md" >}}) | +| 9460-9464 | FREE (formerly used by store-service) | | 9465-9469 | FREE | | 9470-9474 | FREE | | 9475-9479 | FREE | diff --git a/docs/services/store/.gitignore b/docs/services/store/.gitignore deleted file mode 100644 index 63536ebfa2..0000000000 --- a/docs/services/store/.gitignore +++ /dev/null @@ -1 +0,0 @@ -grpc.md diff --git a/docs/services/store/_index.md b/docs/services/store/_index.md deleted file mode 100644 index 9277a24b8d..0000000000 --- a/docs/services/store/_index.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: "Store" -date: 2018-05-02T00:00:00+00:00 -weight: 20 -geekdocRepo: https://github.com/owncloud/ocis -geekdocEditPath: edit/master/docs/services/store -geekdocFilePath: _index.md -geekdocCollapseSection: true ---- - -## Abstract - -This service provides ... - - -## Table of Contents - -{{< toc-tree >}} diff --git a/docs/services/store/configuration.md b/docs/services/store/configuration.md deleted file mode 100644 index c0070e39fe..0000000000 --- a/docs/services/store/configuration.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Service Configuration -date: 2018-05-02T00:00:00+00:00 -weight: 20 -geekdocRepo: https://github.com/owncloud/ocis -geekdocEditPath: edit/master/docs/services/store -geekdocFilePath: configuration.md -geekdocCollapseSection: true ---- - -## Example YAML Config - -{{< include file="services/_includes/store-config-example.yaml" language="yaml" >}} - -{{< include file="services/_includes/store_configvars.md" >}} \ No newline at end of file From 30c3592c345e3daf00df9df7c5a4bd5b9bd0a448 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 22 Aug 2024 09:44:47 +0200 Subject: [PATCH 09/71] remove store references from Makefile Signed-off-by: Christian Richter --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3c3ba483f7..d594613951 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,6 @@ OCIS_MODULES = \ services/storage-publiclink \ services/storage-shares \ services/storage-users \ - services/store \ services/thumbnails \ services/userlog \ services/users \ @@ -221,7 +220,7 @@ go-coverage: .PHONY: protobuf protobuf: - @for mod in ./services/thumbnails ./services/store ./services/settings; do \ + @for mod in ./services/thumbnails ./services/settings; do \ echo -n "% protobuf $$mod: "; $(MAKE) --no-print-directory -C $$mod protobuf || exit 1; \ done From 84fed8997e8fc2919e94e3318a06a60f83dc6b1c Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 22 Aug 2024 10:27:32 +0200 Subject: [PATCH 10/71] make sonarcloud happy Signed-off-by: Christian Richter --- services/ocs/pkg/server/http/server.go | 3 +-- services/proxy/pkg/command/server.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/services/ocs/pkg/server/http/server.go b/services/ocs/pkg/server/http/server.go index d09f168bd1..eba4c07190 100644 --- a/services/ocs/pkg/server/http/server.go +++ b/services/ocs/pkg/server/http/server.go @@ -36,8 +36,7 @@ func Server(opts ...Option) (http.Service, error) { return http.Service{}, fmt.Errorf("could not initialize http service: %w", err) } - var signingKeyStore microstore.Store - signingKeyStore = store.Create( + signingKeyStore := store.Create( store.Store(options.Config.SigningKeys.Store), store.TTL(options.Config.SigningKeys.TTL), microstore.Nodes(options.Config.SigningKeys.Nodes...), diff --git a/services/proxy/pkg/command/server.go b/services/proxy/pkg/command/server.go index b0033c0157..4c2fb9a237 100644 --- a/services/proxy/pkg/command/server.go +++ b/services/proxy/pkg/command/server.go @@ -65,8 +65,7 @@ func Server(cfg *config.Config) *cli.Command { store.Authentication(cfg.OIDC.UserinfoCache.AuthUsername, cfg.OIDC.UserinfoCache.AuthPassword), ) - var signingKeyStore microstore.Store - signingKeyStore = store.Create( + signingKeyStore := store.Create( store.Store(cfg.PreSignedURL.SigningKeys.Store), store.TTL(cfg.PreSignedURL.SigningKeys.TTL), microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...), From 3669322e099d6c695c39ff72b8f87d243916dd98 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 22 Aug 2024 11:04:59 +0200 Subject: [PATCH 11/71] remove protogen & generate protobuf code Signed-off-by: Christian Richter --- protogen/.mockery.yaml | 4 - .../v0/mocks/event_history_service.go | 2 +- .../v0/mocks/policies_provider_service.go | 2 +- .../v0/mocks/search_provider_service.go | 2 +- .../settings/v0/mocks/value_service.go | 2 +- .../services/store/v0/mocks/store_service.go | 484 --------- .../gen/ocis/services/store/v0/store.pb.go | 929 ------------------ .../ocis/services/store/v0/store.pb.micro.go | 254 ----- .../ocis/services/store/v0/store.swagger.json | 242 ----- .../thumbnails/v0/mocks/thumbnail_service.go | 2 +- 10 files changed, 5 insertions(+), 1918 deletions(-) delete mode 100644 protogen/gen/ocis/services/store/v0/mocks/store_service.go delete mode 100644 protogen/gen/ocis/services/store/v0/store.pb.go delete mode 100644 protogen/gen/ocis/services/store/v0/store.pb.micro.go delete mode 100644 protogen/gen/ocis/services/store/v0/store.swagger.json diff --git a/protogen/.mockery.yaml b/protogen/.mockery.yaml index cf7902b040..6da3b73eaa 100644 --- a/protogen/.mockery.yaml +++ b/protogen/.mockery.yaml @@ -16,10 +16,6 @@ packages: github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0: interfaces: ValueService: - github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0: - interfaces: - StoreService: github.com/owncloud/ocis/v2/protogen/gen/ocis/services/thumbnails/v0: interfaces: ThumbnailService: - \ No newline at end of file diff --git a/protogen/gen/ocis/services/eventhistory/v0/mocks/event_history_service.go b/protogen/gen/ocis/services/eventhistory/v0/mocks/event_history_service.go index c532f80efa..f5d298b1af 100644 --- a/protogen/gen/ocis/services/eventhistory/v0/mocks/event_history_service.go +++ b/protogen/gen/ocis/services/eventhistory/v0/mocks/event_history_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.40.1. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/protogen/gen/ocis/services/policies/v0/mocks/policies_provider_service.go b/protogen/gen/ocis/services/policies/v0/mocks/policies_provider_service.go index b4917f2be5..7ee055c8c1 100644 --- a/protogen/gen/ocis/services/policies/v0/mocks/policies_provider_service.go +++ b/protogen/gen/ocis/services/policies/v0/mocks/policies_provider_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.40.1. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/protogen/gen/ocis/services/search/v0/mocks/search_provider_service.go b/protogen/gen/ocis/services/search/v0/mocks/search_provider_service.go index eeca1aa556..ba73046b52 100644 --- a/protogen/gen/ocis/services/search/v0/mocks/search_provider_service.go +++ b/protogen/gen/ocis/services/search/v0/mocks/search_provider_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.40.1. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/protogen/gen/ocis/services/settings/v0/mocks/value_service.go b/protogen/gen/ocis/services/settings/v0/mocks/value_service.go index 8a099b699d..c653ddc6de 100644 --- a/protogen/gen/ocis/services/settings/v0/mocks/value_service.go +++ b/protogen/gen/ocis/services/settings/v0/mocks/value_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.40.1. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/protogen/gen/ocis/services/store/v0/mocks/store_service.go b/protogen/gen/ocis/services/store/v0/mocks/store_service.go deleted file mode 100644 index 81d65f1548..0000000000 --- a/protogen/gen/ocis/services/store/v0/mocks/store_service.go +++ /dev/null @@ -1,484 +0,0 @@ -// Code generated by mockery v2.40.1. DO NOT EDIT. - -package mocks - -import ( - context "context" - - client "go-micro.dev/v4/client" - - mock "github.com/stretchr/testify/mock" - - v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" -) - -// StoreService is an autogenerated mock type for the StoreService type -type StoreService struct { - mock.Mock -} - -type StoreService_Expecter struct { - mock *mock.Mock -} - -func (_m *StoreService) EXPECT() *StoreService_Expecter { - return &StoreService_Expecter{mock: &_m.Mock} -} - -// Databases provides a mock function with given fields: ctx, in, opts -func (_m *StoreService) Databases(ctx context.Context, in *v0.DatabasesRequest, opts ...client.CallOption) (*v0.DatabasesResponse, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for Databases") - } - - var r0 *v0.DatabasesResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *v0.DatabasesRequest, ...client.CallOption) (*v0.DatabasesResponse, error)); ok { - return rf(ctx, in, opts...) - } - if rf, ok := ret.Get(0).(func(context.Context, *v0.DatabasesRequest, ...client.CallOption) *v0.DatabasesResponse); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v0.DatabasesResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *v0.DatabasesRequest, ...client.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// StoreService_Databases_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Databases' -type StoreService_Databases_Call struct { - *mock.Call -} - -// Databases is a helper method to define mock.On call -// - ctx context.Context -// - in *v0.DatabasesRequest -// - opts ...client.CallOption -func (_e *StoreService_Expecter) Databases(ctx interface{}, in interface{}, opts ...interface{}) *StoreService_Databases_Call { - return &StoreService_Databases_Call{Call: _e.mock.On("Databases", - append([]interface{}{ctx, in}, opts...)...)} -} - -func (_c *StoreService_Databases_Call) Run(run func(ctx context.Context, in *v0.DatabasesRequest, opts ...client.CallOption)) *StoreService_Databases_Call { - _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]client.CallOption, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(client.CallOption) - } - } - run(args[0].(context.Context), args[1].(*v0.DatabasesRequest), variadicArgs...) - }) - return _c -} - -func (_c *StoreService_Databases_Call) Return(_a0 *v0.DatabasesResponse, _a1 error) *StoreService_Databases_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *StoreService_Databases_Call) RunAndReturn(run func(context.Context, *v0.DatabasesRequest, ...client.CallOption) (*v0.DatabasesResponse, error)) *StoreService_Databases_Call { - _c.Call.Return(run) - return _c -} - -// Delete provides a mock function with given fields: ctx, in, opts -func (_m *StoreService) Delete(ctx context.Context, in *v0.DeleteRequest, opts ...client.CallOption) (*v0.DeleteResponse, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for Delete") - } - - var r0 *v0.DeleteResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *v0.DeleteRequest, ...client.CallOption) (*v0.DeleteResponse, error)); ok { - return rf(ctx, in, opts...) - } - if rf, ok := ret.Get(0).(func(context.Context, *v0.DeleteRequest, ...client.CallOption) *v0.DeleteResponse); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v0.DeleteResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *v0.DeleteRequest, ...client.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// StoreService_Delete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Delete' -type StoreService_Delete_Call struct { - *mock.Call -} - -// Delete is a helper method to define mock.On call -// - ctx context.Context -// - in *v0.DeleteRequest -// - opts ...client.CallOption -func (_e *StoreService_Expecter) Delete(ctx interface{}, in interface{}, opts ...interface{}) *StoreService_Delete_Call { - return &StoreService_Delete_Call{Call: _e.mock.On("Delete", - append([]interface{}{ctx, in}, opts...)...)} -} - -func (_c *StoreService_Delete_Call) Run(run func(ctx context.Context, in *v0.DeleteRequest, opts ...client.CallOption)) *StoreService_Delete_Call { - _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]client.CallOption, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(client.CallOption) - } - } - run(args[0].(context.Context), args[1].(*v0.DeleteRequest), variadicArgs...) - }) - return _c -} - -func (_c *StoreService_Delete_Call) Return(_a0 *v0.DeleteResponse, _a1 error) *StoreService_Delete_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *StoreService_Delete_Call) RunAndReturn(run func(context.Context, *v0.DeleteRequest, ...client.CallOption) (*v0.DeleteResponse, error)) *StoreService_Delete_Call { - _c.Call.Return(run) - return _c -} - -// List provides a mock function with given fields: ctx, in, opts -func (_m *StoreService) List(ctx context.Context, in *v0.ListRequest, opts ...client.CallOption) (v0.Store_ListService, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for List") - } - - var r0 v0.Store_ListService - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *v0.ListRequest, ...client.CallOption) (v0.Store_ListService, error)); ok { - return rf(ctx, in, opts...) - } - if rf, ok := ret.Get(0).(func(context.Context, *v0.ListRequest, ...client.CallOption) v0.Store_ListService); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(v0.Store_ListService) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *v0.ListRequest, ...client.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// StoreService_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' -type StoreService_List_Call struct { - *mock.Call -} - -// List is a helper method to define mock.On call -// - ctx context.Context -// - in *v0.ListRequest -// - opts ...client.CallOption -func (_e *StoreService_Expecter) List(ctx interface{}, in interface{}, opts ...interface{}) *StoreService_List_Call { - return &StoreService_List_Call{Call: _e.mock.On("List", - append([]interface{}{ctx, in}, opts...)...)} -} - -func (_c *StoreService_List_Call) Run(run func(ctx context.Context, in *v0.ListRequest, opts ...client.CallOption)) *StoreService_List_Call { - _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]client.CallOption, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(client.CallOption) - } - } - run(args[0].(context.Context), args[1].(*v0.ListRequest), variadicArgs...) - }) - return _c -} - -func (_c *StoreService_List_Call) Return(_a0 v0.Store_ListService, _a1 error) *StoreService_List_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *StoreService_List_Call) RunAndReturn(run func(context.Context, *v0.ListRequest, ...client.CallOption) (v0.Store_ListService, error)) *StoreService_List_Call { - _c.Call.Return(run) - return _c -} - -// Read provides a mock function with given fields: ctx, in, opts -func (_m *StoreService) Read(ctx context.Context, in *v0.ReadRequest, opts ...client.CallOption) (*v0.ReadResponse, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for Read") - } - - var r0 *v0.ReadResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *v0.ReadRequest, ...client.CallOption) (*v0.ReadResponse, error)); ok { - return rf(ctx, in, opts...) - } - if rf, ok := ret.Get(0).(func(context.Context, *v0.ReadRequest, ...client.CallOption) *v0.ReadResponse); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v0.ReadResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *v0.ReadRequest, ...client.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// StoreService_Read_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Read' -type StoreService_Read_Call struct { - *mock.Call -} - -// Read is a helper method to define mock.On call -// - ctx context.Context -// - in *v0.ReadRequest -// - opts ...client.CallOption -func (_e *StoreService_Expecter) Read(ctx interface{}, in interface{}, opts ...interface{}) *StoreService_Read_Call { - return &StoreService_Read_Call{Call: _e.mock.On("Read", - append([]interface{}{ctx, in}, opts...)...)} -} - -func (_c *StoreService_Read_Call) Run(run func(ctx context.Context, in *v0.ReadRequest, opts ...client.CallOption)) *StoreService_Read_Call { - _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]client.CallOption, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(client.CallOption) - } - } - run(args[0].(context.Context), args[1].(*v0.ReadRequest), variadicArgs...) - }) - return _c -} - -func (_c *StoreService_Read_Call) Return(_a0 *v0.ReadResponse, _a1 error) *StoreService_Read_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *StoreService_Read_Call) RunAndReturn(run func(context.Context, *v0.ReadRequest, ...client.CallOption) (*v0.ReadResponse, error)) *StoreService_Read_Call { - _c.Call.Return(run) - return _c -} - -// Tables provides a mock function with given fields: ctx, in, opts -func (_m *StoreService) Tables(ctx context.Context, in *v0.TablesRequest, opts ...client.CallOption) (*v0.TablesResponse, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for Tables") - } - - var r0 *v0.TablesResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *v0.TablesRequest, ...client.CallOption) (*v0.TablesResponse, error)); ok { - return rf(ctx, in, opts...) - } - if rf, ok := ret.Get(0).(func(context.Context, *v0.TablesRequest, ...client.CallOption) *v0.TablesResponse); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v0.TablesResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *v0.TablesRequest, ...client.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// StoreService_Tables_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Tables' -type StoreService_Tables_Call struct { - *mock.Call -} - -// Tables is a helper method to define mock.On call -// - ctx context.Context -// - in *v0.TablesRequest -// - opts ...client.CallOption -func (_e *StoreService_Expecter) Tables(ctx interface{}, in interface{}, opts ...interface{}) *StoreService_Tables_Call { - return &StoreService_Tables_Call{Call: _e.mock.On("Tables", - append([]interface{}{ctx, in}, opts...)...)} -} - -func (_c *StoreService_Tables_Call) Run(run func(ctx context.Context, in *v0.TablesRequest, opts ...client.CallOption)) *StoreService_Tables_Call { - _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]client.CallOption, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(client.CallOption) - } - } - run(args[0].(context.Context), args[1].(*v0.TablesRequest), variadicArgs...) - }) - return _c -} - -func (_c *StoreService_Tables_Call) Return(_a0 *v0.TablesResponse, _a1 error) *StoreService_Tables_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *StoreService_Tables_Call) RunAndReturn(run func(context.Context, *v0.TablesRequest, ...client.CallOption) (*v0.TablesResponse, error)) *StoreService_Tables_Call { - _c.Call.Return(run) - return _c -} - -// Write provides a mock function with given fields: ctx, in, opts -func (_m *StoreService) Write(ctx context.Context, in *v0.WriteRequest, opts ...client.CallOption) (*v0.WriteResponse, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for Write") - } - - var r0 *v0.WriteResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *v0.WriteRequest, ...client.CallOption) (*v0.WriteResponse, error)); ok { - return rf(ctx, in, opts...) - } - if rf, ok := ret.Get(0).(func(context.Context, *v0.WriteRequest, ...client.CallOption) *v0.WriteResponse); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v0.WriteResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *v0.WriteRequest, ...client.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// StoreService_Write_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Write' -type StoreService_Write_Call struct { - *mock.Call -} - -// Write is a helper method to define mock.On call -// - ctx context.Context -// - in *v0.WriteRequest -// - opts ...client.CallOption -func (_e *StoreService_Expecter) Write(ctx interface{}, in interface{}, opts ...interface{}) *StoreService_Write_Call { - return &StoreService_Write_Call{Call: _e.mock.On("Write", - append([]interface{}{ctx, in}, opts...)...)} -} - -func (_c *StoreService_Write_Call) Run(run func(ctx context.Context, in *v0.WriteRequest, opts ...client.CallOption)) *StoreService_Write_Call { - _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]client.CallOption, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(client.CallOption) - } - } - run(args[0].(context.Context), args[1].(*v0.WriteRequest), variadicArgs...) - }) - return _c -} - -func (_c *StoreService_Write_Call) Return(_a0 *v0.WriteResponse, _a1 error) *StoreService_Write_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *StoreService_Write_Call) RunAndReturn(run func(context.Context, *v0.WriteRequest, ...client.CallOption) (*v0.WriteResponse, error)) *StoreService_Write_Call { - _c.Call.Return(run) - return _c -} - -// NewStoreService creates a new instance of StoreService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewStoreService(t interface { - mock.TestingT - Cleanup(func()) -}) *StoreService { - mock := &StoreService{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/protogen/gen/ocis/services/store/v0/store.pb.go b/protogen/gen/ocis/services/store/v0/store.pb.go deleted file mode 100644 index d38008316b..0000000000 --- a/protogen/gen/ocis/services/store/v0/store.pb.go +++ /dev/null @@ -1,929 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.1 -// protoc (unknown) -// source: ocis/services/store/v0/store.proto - -package v0 - -import ( - _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2/options" - v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type ReadRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - Options *v0.ReadOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` -} - -func (x *ReadRequest) Reset() { - *x = ReadRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ReadRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ReadRequest) ProtoMessage() {} - -func (x *ReadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ReadRequest.ProtoReflect.Descriptor instead. -func (*ReadRequest) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{0} -} - -func (x *ReadRequest) GetKey() string { - if x != nil { - return x.Key - } - return "" -} - -func (x *ReadRequest) GetOptions() *v0.ReadOptions { - if x != nil { - return x.Options - } - return nil -} - -type ReadResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Records []*v0.Record `protobuf:"bytes,1,rep,name=records,proto3" json:"records,omitempty"` -} - -func (x *ReadResponse) Reset() { - *x = ReadResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ReadResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ReadResponse) ProtoMessage() {} - -func (x *ReadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ReadResponse.ProtoReflect.Descriptor instead. -func (*ReadResponse) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{1} -} - -func (x *ReadResponse) GetRecords() []*v0.Record { - if x != nil { - return x.Records - } - return nil -} - -type WriteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Record *v0.Record `protobuf:"bytes,1,opt,name=record,proto3" json:"record,omitempty"` - Options *v0.WriteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` -} - -func (x *WriteRequest) Reset() { - *x = WriteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *WriteRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*WriteRequest) ProtoMessage() {} - -func (x *WriteRequest) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use WriteRequest.ProtoReflect.Descriptor instead. -func (*WriteRequest) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{2} -} - -func (x *WriteRequest) GetRecord() *v0.Record { - if x != nil { - return x.Record - } - return nil -} - -func (x *WriteRequest) GetOptions() *v0.WriteOptions { - if x != nil { - return x.Options - } - return nil -} - -type WriteResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *WriteResponse) Reset() { - *x = WriteResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *WriteResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*WriteResponse) ProtoMessage() {} - -func (x *WriteResponse) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use WriteResponse.ProtoReflect.Descriptor instead. -func (*WriteResponse) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{3} -} - -type DeleteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - Options *v0.DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` -} - -func (x *DeleteRequest) Reset() { - *x = DeleteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DeleteRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DeleteRequest) ProtoMessage() {} - -func (x *DeleteRequest) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DeleteRequest.ProtoReflect.Descriptor instead. -func (*DeleteRequest) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{4} -} - -func (x *DeleteRequest) GetKey() string { - if x != nil { - return x.Key - } - return "" -} - -func (x *DeleteRequest) GetOptions() *v0.DeleteOptions { - if x != nil { - return x.Options - } - return nil -} - -type DeleteResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *DeleteResponse) Reset() { - *x = DeleteResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DeleteResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DeleteResponse) ProtoMessage() {} - -func (x *DeleteResponse) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DeleteResponse.ProtoReflect.Descriptor instead. -func (*DeleteResponse) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{5} -} - -type ListRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Options *v0.ListOptions `protobuf:"bytes,1,opt,name=options,proto3" json:"options,omitempty"` -} - -func (x *ListRequest) Reset() { - *x = ListRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListRequest) ProtoMessage() {} - -func (x *ListRequest) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListRequest.ProtoReflect.Descriptor instead. -func (*ListRequest) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{6} -} - -func (x *ListRequest) GetOptions() *v0.ListOptions { - if x != nil { - return x.Options - } - return nil -} - -type ListResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,2,rep,name=keys,proto3" json:"keys,omitempty"` -} - -func (x *ListResponse) Reset() { - *x = ListResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListResponse) ProtoMessage() {} - -func (x *ListResponse) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListResponse.ProtoReflect.Descriptor instead. -func (*ListResponse) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{7} -} - -func (x *ListResponse) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -type DatabasesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *DatabasesRequest) Reset() { - *x = DatabasesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DatabasesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DatabasesRequest) ProtoMessage() {} - -func (x *DatabasesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DatabasesRequest.ProtoReflect.Descriptor instead. -func (*DatabasesRequest) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{8} -} - -type DatabasesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Databases []string `protobuf:"bytes,1,rep,name=databases,proto3" json:"databases,omitempty"` -} - -func (x *DatabasesResponse) Reset() { - *x = DatabasesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DatabasesResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DatabasesResponse) ProtoMessage() {} - -func (x *DatabasesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DatabasesResponse.ProtoReflect.Descriptor instead. -func (*DatabasesResponse) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{9} -} - -func (x *DatabasesResponse) GetDatabases() []string { - if x != nil { - return x.Databases - } - return nil -} - -type TablesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Database string `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"` -} - -func (x *TablesRequest) Reset() { - *x = TablesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TablesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TablesRequest) ProtoMessage() {} - -func (x *TablesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TablesRequest.ProtoReflect.Descriptor instead. -func (*TablesRequest) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{10} -} - -func (x *TablesRequest) GetDatabase() string { - if x != nil { - return x.Database - } - return "" -} - -type TablesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Tables []string `protobuf:"bytes,1,rep,name=tables,proto3" json:"tables,omitempty"` -} - -func (x *TablesResponse) Reset() { - *x = TablesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TablesResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TablesResponse) ProtoMessage() {} - -func (x *TablesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ocis_services_store_v0_store_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TablesResponse.ProtoReflect.Descriptor instead. -func (*TablesResponse) Descriptor() ([]byte, []int) { - return file_ocis_services_store_v0_store_proto_rawDescGZIP(), []int{11} -} - -func (x *TablesResponse) GetTables() []string { - if x != nil { - return x.Tables - } - return nil -} - -var File_ocis_services_store_v0_store_proto protoreflect.FileDescriptor - -var file_ocis_services_store_v0_store_proto_rawDesc = []byte{ - 0x0a, 0x22, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x1a, 0x22, 0x6f, 0x63, - 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x6f, 0x70, 0x65, - 0x6e, 0x61, 0x70, 0x69, 0x76, 0x32, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x61, - 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0x5e, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x3d, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x61, 0x64, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x48, 0x0a, 0x0c, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x38, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, - 0x64, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x86, 0x01, 0x0a, 0x0c, 0x57, - 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x36, 0x0a, 0x06, 0x72, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x63, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, - 0x6f, 0x72, 0x64, 0x12, 0x3e, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x22, 0x0f, 0x0a, 0x0d, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x62, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x3f, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x30, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x10, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x4c, 0x0a, 0x0b, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x63, 0x69, - 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x30, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x4a, 0x04, 0x08, 0x01, - 0x10, 0x02, 0x22, 0x12, 0x0a, 0x10, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x11, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, - 0x73, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, - 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, - 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x73, 0x22, 0x2b, 0x0a, 0x0d, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x61, - 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, - 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x22, 0x28, 0x0a, 0x0e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, - 0x32, 0xa5, 0x04, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x53, 0x0a, 0x04, 0x52, 0x65, - 0x61, 0x64, 0x12, 0x23, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x61, 0x64, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, - 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x56, 0x0a, 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x24, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x30, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, - 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x59, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x12, 0x25, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x30, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x55, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x23, 0x2e, 0x6f, 0x63, 0x69, - 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x30, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x24, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x62, 0x0a, 0x09, 0x44, 0x61, 0x74, - 0x61, 0x62, 0x61, 0x73, 0x65, 0x73, 0x12, 0x28, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, - 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x29, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, - 0x73, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x59, 0x0a, - 0x06, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x25, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, - 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, - 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x30, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0xd9, 0x02, 0x5a, 0x3b, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, - 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x30, 0x92, 0x41, 0x98, 0x02, 0x12, 0xb3, 0x01, 0x0a, - 0x1d, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, - 0x74, 0x65, 0x20, 0x53, 0x63, 0x61, 0x6c, 0x65, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x47, - 0x0a, 0x0d, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, - 0x20, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, - 0x73, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x2e, 0x63, 0x6f, 0x6d, 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x73, - 0x74, 0x65, 0x72, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, - 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x38, 0x0a, 0x10, 0x44, 0x65, - 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x24, - 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_ocis_services_store_v0_store_proto_rawDescOnce sync.Once - file_ocis_services_store_v0_store_proto_rawDescData = file_ocis_services_store_v0_store_proto_rawDesc -) - -func file_ocis_services_store_v0_store_proto_rawDescGZIP() []byte { - file_ocis_services_store_v0_store_proto_rawDescOnce.Do(func() { - file_ocis_services_store_v0_store_proto_rawDescData = protoimpl.X.CompressGZIP(file_ocis_services_store_v0_store_proto_rawDescData) - }) - return file_ocis_services_store_v0_store_proto_rawDescData -} - -var file_ocis_services_store_v0_store_proto_msgTypes = make([]protoimpl.MessageInfo, 12) -var file_ocis_services_store_v0_store_proto_goTypes = []interface{}{ - (*ReadRequest)(nil), // 0: ocis.services.store.v0.ReadRequest - (*ReadResponse)(nil), // 1: ocis.services.store.v0.ReadResponse - (*WriteRequest)(nil), // 2: ocis.services.store.v0.WriteRequest - (*WriteResponse)(nil), // 3: ocis.services.store.v0.WriteResponse - (*DeleteRequest)(nil), // 4: ocis.services.store.v0.DeleteRequest - (*DeleteResponse)(nil), // 5: ocis.services.store.v0.DeleteResponse - (*ListRequest)(nil), // 6: ocis.services.store.v0.ListRequest - (*ListResponse)(nil), // 7: ocis.services.store.v0.ListResponse - (*DatabasesRequest)(nil), // 8: ocis.services.store.v0.DatabasesRequest - (*DatabasesResponse)(nil), // 9: ocis.services.store.v0.DatabasesResponse - (*TablesRequest)(nil), // 10: ocis.services.store.v0.TablesRequest - (*TablesResponse)(nil), // 11: ocis.services.store.v0.TablesResponse - (*v0.ReadOptions)(nil), // 12: ocis.messages.store.v0.ReadOptions - (*v0.Record)(nil), // 13: ocis.messages.store.v0.Record - (*v0.WriteOptions)(nil), // 14: ocis.messages.store.v0.WriteOptions - (*v0.DeleteOptions)(nil), // 15: ocis.messages.store.v0.DeleteOptions - (*v0.ListOptions)(nil), // 16: ocis.messages.store.v0.ListOptions -} -var file_ocis_services_store_v0_store_proto_depIdxs = []int32{ - 12, // 0: ocis.services.store.v0.ReadRequest.options:type_name -> ocis.messages.store.v0.ReadOptions - 13, // 1: ocis.services.store.v0.ReadResponse.records:type_name -> ocis.messages.store.v0.Record - 13, // 2: ocis.services.store.v0.WriteRequest.record:type_name -> ocis.messages.store.v0.Record - 14, // 3: ocis.services.store.v0.WriteRequest.options:type_name -> ocis.messages.store.v0.WriteOptions - 15, // 4: ocis.services.store.v0.DeleteRequest.options:type_name -> ocis.messages.store.v0.DeleteOptions - 16, // 5: ocis.services.store.v0.ListRequest.options:type_name -> ocis.messages.store.v0.ListOptions - 0, // 6: ocis.services.store.v0.Store.Read:input_type -> ocis.services.store.v0.ReadRequest - 2, // 7: ocis.services.store.v0.Store.Write:input_type -> ocis.services.store.v0.WriteRequest - 4, // 8: ocis.services.store.v0.Store.Delete:input_type -> ocis.services.store.v0.DeleteRequest - 6, // 9: ocis.services.store.v0.Store.List:input_type -> ocis.services.store.v0.ListRequest - 8, // 10: ocis.services.store.v0.Store.Databases:input_type -> ocis.services.store.v0.DatabasesRequest - 10, // 11: ocis.services.store.v0.Store.Tables:input_type -> ocis.services.store.v0.TablesRequest - 1, // 12: ocis.services.store.v0.Store.Read:output_type -> ocis.services.store.v0.ReadResponse - 3, // 13: ocis.services.store.v0.Store.Write:output_type -> ocis.services.store.v0.WriteResponse - 5, // 14: ocis.services.store.v0.Store.Delete:output_type -> ocis.services.store.v0.DeleteResponse - 7, // 15: ocis.services.store.v0.Store.List:output_type -> ocis.services.store.v0.ListResponse - 9, // 16: ocis.services.store.v0.Store.Databases:output_type -> ocis.services.store.v0.DatabasesResponse - 11, // 17: ocis.services.store.v0.Store.Tables:output_type -> ocis.services.store.v0.TablesResponse - 12, // [12:18] is the sub-list for method output_type - 6, // [6:12] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name -} - -func init() { file_ocis_services_store_v0_store_proto_init() } -func file_ocis_services_store_v0_store_proto_init() { - if File_ocis_services_store_v0_store_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_ocis_services_store_v0_store_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WriteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WriteResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DatabasesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DatabasesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TablesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ocis_services_store_v0_store_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TablesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_ocis_services_store_v0_store_proto_rawDesc, - NumEnums: 0, - NumMessages: 12, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_ocis_services_store_v0_store_proto_goTypes, - DependencyIndexes: file_ocis_services_store_v0_store_proto_depIdxs, - MessageInfos: file_ocis_services_store_v0_store_proto_msgTypes, - }.Build() - File_ocis_services_store_v0_store_proto = out.File - file_ocis_services_store_v0_store_proto_rawDesc = nil - file_ocis_services_store_v0_store_proto_goTypes = nil - file_ocis_services_store_v0_store_proto_depIdxs = nil -} diff --git a/protogen/gen/ocis/services/store/v0/store.pb.micro.go b/protogen/gen/ocis/services/store/v0/store.pb.micro.go deleted file mode 100644 index 5312134566..0000000000 --- a/protogen/gen/ocis/services/store/v0/store.pb.micro.go +++ /dev/null @@ -1,254 +0,0 @@ -// Code generated by protoc-gen-micro. DO NOT EDIT. -// source: ocis/services/store/v0/store.proto - -package v0 - -import ( - fmt "fmt" - _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2/options" - _ "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" - proto "google.golang.org/protobuf/proto" - math "math" -) - -import ( - context "context" - api "go-micro.dev/v4/api" - client "go-micro.dev/v4/client" - server "go-micro.dev/v4/server" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// Reference imports to suppress errors if they are not otherwise used. -var _ api.Endpoint -var _ context.Context -var _ client.Option -var _ server.Option - -// Api Endpoints for Store service - -func NewStoreEndpoints() []*api.Endpoint { - return []*api.Endpoint{} -} - -// Client API for Store service - -type StoreService interface { - Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error) - Write(ctx context.Context, in *WriteRequest, opts ...client.CallOption) (*WriteResponse, error) - Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) - List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) - Databases(ctx context.Context, in *DatabasesRequest, opts ...client.CallOption) (*DatabasesResponse, error) - Tables(ctx context.Context, in *TablesRequest, opts ...client.CallOption) (*TablesResponse, error) -} - -type storeService struct { - c client.Client - name string -} - -func NewStoreService(name string, c client.Client) StoreService { - return &storeService{ - c: c, - name: name, - } -} - -func (c *storeService) Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error) { - req := c.c.NewRequest(c.name, "Store.Read", in) - out := new(ReadResponse) - err := c.c.Call(ctx, req, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *storeService) Write(ctx context.Context, in *WriteRequest, opts ...client.CallOption) (*WriteResponse, error) { - req := c.c.NewRequest(c.name, "Store.Write", in) - out := new(WriteResponse) - err := c.c.Call(ctx, req, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *storeService) Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) { - req := c.c.NewRequest(c.name, "Store.Delete", in) - out := new(DeleteResponse) - err := c.c.Call(ctx, req, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *storeService) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) { - req := c.c.NewRequest(c.name, "Store.List", &ListRequest{}) - stream, err := c.c.Stream(ctx, req, opts...) - if err != nil { - return nil, err - } - if err := stream.Send(in); err != nil { - return nil, err - } - return &storeServiceList{stream}, nil -} - -type Store_ListService interface { - Context() context.Context - SendMsg(interface{}) error - RecvMsg(interface{}) error - CloseSend() error - Close() error - Recv() (*ListResponse, error) -} - -type storeServiceList struct { - stream client.Stream -} - -func (x *storeServiceList) CloseSend() error { - return x.stream.CloseSend() -} - -func (x *storeServiceList) Close() error { - return x.stream.Close() -} - -func (x *storeServiceList) Context() context.Context { - return x.stream.Context() -} - -func (x *storeServiceList) SendMsg(m interface{}) error { - return x.stream.Send(m) -} - -func (x *storeServiceList) RecvMsg(m interface{}) error { - return x.stream.Recv(m) -} - -func (x *storeServiceList) Recv() (*ListResponse, error) { - m := new(ListResponse) - err := x.stream.Recv(m) - if err != nil { - return nil, err - } - return m, nil -} - -func (c *storeService) Databases(ctx context.Context, in *DatabasesRequest, opts ...client.CallOption) (*DatabasesResponse, error) { - req := c.c.NewRequest(c.name, "Store.Databases", in) - out := new(DatabasesResponse) - err := c.c.Call(ctx, req, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *storeService) Tables(ctx context.Context, in *TablesRequest, opts ...client.CallOption) (*TablesResponse, error) { - req := c.c.NewRequest(c.name, "Store.Tables", in) - out := new(TablesResponse) - err := c.c.Call(ctx, req, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// Server API for Store service - -type StoreHandler interface { - Read(context.Context, *ReadRequest, *ReadResponse) error - Write(context.Context, *WriteRequest, *WriteResponse) error - Delete(context.Context, *DeleteRequest, *DeleteResponse) error - List(context.Context, *ListRequest, Store_ListStream) error - Databases(context.Context, *DatabasesRequest, *DatabasesResponse) error - Tables(context.Context, *TablesRequest, *TablesResponse) error -} - -func RegisterStoreHandler(s server.Server, hdlr StoreHandler, opts ...server.HandlerOption) error { - type store interface { - Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error - Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error - Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error - List(ctx context.Context, stream server.Stream) error - Databases(ctx context.Context, in *DatabasesRequest, out *DatabasesResponse) error - Tables(ctx context.Context, in *TablesRequest, out *TablesResponse) error - } - type Store struct { - store - } - h := &storeHandler{hdlr} - return s.Handle(s.NewHandler(&Store{h}, opts...)) -} - -type storeHandler struct { - StoreHandler -} - -func (h *storeHandler) Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error { - return h.StoreHandler.Read(ctx, in, out) -} - -func (h *storeHandler) Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error { - return h.StoreHandler.Write(ctx, in, out) -} - -func (h *storeHandler) Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error { - return h.StoreHandler.Delete(ctx, in, out) -} - -func (h *storeHandler) List(ctx context.Context, stream server.Stream) error { - m := new(ListRequest) - if err := stream.Recv(m); err != nil { - return err - } - return h.StoreHandler.List(ctx, m, &storeListStream{stream}) -} - -type Store_ListStream interface { - Context() context.Context - SendMsg(interface{}) error - RecvMsg(interface{}) error - Close() error - Send(*ListResponse) error -} - -type storeListStream struct { - stream server.Stream -} - -func (x *storeListStream) Close() error { - return x.stream.Close() -} - -func (x *storeListStream) Context() context.Context { - return x.stream.Context() -} - -func (x *storeListStream) SendMsg(m interface{}) error { - return x.stream.Send(m) -} - -func (x *storeListStream) RecvMsg(m interface{}) error { - return x.stream.Recv(m) -} - -func (x *storeListStream) Send(m *ListResponse) error { - return x.stream.Send(m) -} - -func (h *storeHandler) Databases(ctx context.Context, in *DatabasesRequest, out *DatabasesResponse) error { - return h.StoreHandler.Databases(ctx, in, out) -} - -func (h *storeHandler) Tables(ctx context.Context, in *TablesRequest, out *TablesResponse) error { - return h.StoreHandler.Tables(ctx, in, out) -} diff --git a/protogen/gen/ocis/services/store/v0/store.swagger.json b/protogen/gen/ocis/services/store/v0/store.swagger.json deleted file mode 100644 index 21a5a67ab6..0000000000 --- a/protogen/gen/ocis/services/store/v0/store.swagger.json +++ /dev/null @@ -1,242 +0,0 @@ -{ - "swagger": "2.0", - "info": { - "title": "ownCloud Infinite Scale store", - "version": "1.0.0", - "contact": { - "name": "ownCloud GmbH", - "url": "https://github.com/owncloud/ocis", - "email": "support@owncloud.com" - }, - "license": { - "name": "Apache-2.0", - "url": "https://github.com/owncloud/ocis/blob/master/LICENSE" - } - }, - "tags": [ - { - "name": "Store" - } - ], - "schemes": [ - "http", - "https" - ], - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "paths": {}, - "definitions": { - "protobufAny": { - "type": "object", - "properties": { - "@type": { - "type": "string" - } - }, - "additionalProperties": {} - }, - "rpcStatus": { - "type": "object", - "properties": { - "code": { - "type": "integer", - "format": "int32" - }, - "message": { - "type": "string" - }, - "details": { - "type": "array", - "items": { - "$ref": "#/definitions/protobufAny" - } - } - } - }, - "v0DatabasesResponse": { - "type": "object", - "properties": { - "databases": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "v0DeleteOptions": { - "type": "object", - "properties": { - "database": { - "type": "string" - }, - "table": { - "type": "string" - } - } - }, - "v0DeleteResponse": { - "type": "object" - }, - "v0Field": { - "type": "object", - "properties": { - "type": { - "type": "string", - "title": "type of value e.g string, int, int64, bool, float64" - }, - "value": { - "type": "string", - "title": "the actual value" - } - } - }, - "v0ListOptions": { - "type": "object", - "properties": { - "database": { - "type": "string" - }, - "table": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "limit": { - "type": "string", - "format": "uint64" - }, - "offset": { - "type": "string", - "format": "uint64" - } - } - }, - "v0ListResponse": { - "type": "object", - "properties": { - "keys": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "v0ReadOptions": { - "type": "object", - "properties": { - "database": { - "type": "string" - }, - "table": { - "type": "string" - }, - "prefix": { - "type": "boolean" - }, - "suffix": { - "type": "boolean" - }, - "limit": { - "type": "string", - "format": "uint64" - }, - "offset": { - "type": "string", - "format": "uint64" - }, - "where": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/v0Field" - } - } - } - }, - "v0ReadResponse": { - "type": "object", - "properties": { - "records": { - "type": "array", - "items": { - "$ref": "#/definitions/v0Record" - } - } - } - }, - "v0Record": { - "type": "object", - "properties": { - "key": { - "type": "string", - "title": "key of the recorda" - }, - "value": { - "type": "string", - "format": "byte", - "title": "value in the record" - }, - "expiry": { - "type": "string", - "format": "int64", - "title": "time.Duration (signed int64 nanoseconds)" - }, - "metadata": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/v0Field" - }, - "title": "the associated metadata" - } - } - }, - "v0TablesResponse": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "v0WriteOptions": { - "type": "object", - "properties": { - "database": { - "type": "string" - }, - "table": { - "type": "string" - }, - "expiry": { - "type": "string", - "format": "int64", - "title": "time.Time" - }, - "ttl": { - "type": "string", - "format": "int64", - "title": "time.Duration" - } - } - }, - "v0WriteResponse": { - "type": "object" - } - }, - "externalDocs": { - "description": "Developer Manual", - "url": "https://owncloud.dev/services/store/" - } -} diff --git a/protogen/gen/ocis/services/thumbnails/v0/mocks/thumbnail_service.go b/protogen/gen/ocis/services/thumbnails/v0/mocks/thumbnail_service.go index 269976d7a5..83aad864ee 100644 --- a/protogen/gen/ocis/services/thumbnails/v0/mocks/thumbnail_service.go +++ b/protogen/gen/ocis/services/thumbnails/v0/mocks/thumbnail_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.40.1. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks From 085a1a4b4000ce0719ee60765af1aa262b40ab7e Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 20 Aug 2024 14:39:30 +0545 Subject: [PATCH 12/71] test: fix the headings --- .../expected-failures-API-on-OCIS-storage.md | 18 +++++----- ...ected-failures-localAPI-on-OCIS-storage.md | 36 ++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/tests/acceptance/expected-failures-API-on-OCIS-storage.md b/tests/acceptance/expected-failures-API-on-OCIS-storage.md index e4147f4e0b..0d133ea7a7 100644 --- a/tests/acceptance/expected-failures-API-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-API-on-OCIS-storage.md @@ -150,13 +150,6 @@ _ocdav: api compatibility, return correct status code_ - [coreApiWebdavUploadTUS/optionsRequest.feature:40](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L40) - [coreApiWebdavUploadTUS/optionsRequest.feature:55](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L55) -### Won't fix - -Not everything needs to be implemented for ocis. While the oc10 testsuite covers these things we are not looking at them right now. - -- _The `OC-LazyOps` header is [no longer supported by the client](https://github.com/owncloud/client/pull/8398), implementing this is not necessary for a first production release. We plan to have an upload state machine to visualize the state of a file, see https://github.com/owncloud/ocis/issues/214_ -- _Blacklisted ignored files are no longer required because ocis can handle `.htaccess` files without security implications introduced by serving user provided files with apache._ - #### [Renaming resource to banned name is allowed in spaces webdav](https://github.com/owncloud/ocis/issues/3099) - [coreApiWebdavMove2/moveFile.feature:143](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L143) @@ -168,7 +161,7 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiTrashbin/trashbinDelete.feature:92](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L92) -### [MOVE a file into same folder with same name returns 404 instead of 403](https://github.com/owncloud/ocis/issues/1976) +#### [MOVE a file into same folder with same name returns 404 instead of 403](https://github.com/owncloud/ocis/issues/1976) - [coreApiWebdavMove2/moveFile.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L100) - [coreApiWebdavMove2/moveFile.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L101) @@ -179,12 +172,19 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiWebdavMove2/moveShareOnOcis.feature:307](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature#L307) - [coreApiWebdavMove2/moveShareOnOcis.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature#L310) -### [COPY file/folder to same name is possible (but 500 code error for folder with spaces path)](https://github.com/owncloud/ocis/issues/8711) +#### [COPY file/folder to same name is possible (but 500 code error for folder with spaces path)](https://github.com/owncloud/ocis/issues/8711) - [coreApiSharePublicLink2/copyFromPublicLink.feature:198](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L198) - [coreApiWebdavProperties/copyFile.feature:1067](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties/copyFile.feature#L1067) - [coreApiWebdavProperties/copyFile.feature:1068](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties/copyFile.feature#L1068) - [coreApiWebdavProperties/copyFile.feature:1069](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties/copyFile.feature#L1069) +### Won't fix + +Not everything needs to be implemented for ocis. While the oc10 testsuite covers these things we are not looking at them right now. + +- _The `OC-LazyOps` header is [no longer supported by the client](https://github.com/owncloud/client/pull/8398), implementing this is not necessary for a first production release. We plan to have an upload state machine to visualize the state of a file, see https://github.com/owncloud/ocis/issues/214_ +- _Blacklisted ignored files are no longer required because ocis can handle `.htaccess` files without security implications introduced by serving user provided files with apache._ + Note: always have an empty line at the end of this file. The bash script that processes this file requires that the last line has a newline on the end. diff --git a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md index d4e1cccba9..bcf8e516a6 100644 --- a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md @@ -13,12 +13,12 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiArchiver/downloadByPath.feature:171](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadByPath.feature#L171) - [apiArchiver/downloadByPath.feature:172](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadByPath.feature#L172) -### [Downloaded /Shares tar contains resource (files|folder) with leading / in Response](https://github.com/owncloud/ocis/issues/4636) +#### [Downloaded /Shares tar contains resource (files|folder) with leading / in Response](https://github.com/owncloud/ocis/issues/4636) - [apiArchiver/downloadById.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadById.feature#L173) - [apiArchiver/downloadById.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadById.feature#L174) -### [Shared mount folder gets deleted when overwritten by a file from personal space](https://github.com/owncloud/ocis/issues/7208) +#### [Shared mount folder gets deleted when overwritten by a file from personal space](https://github.com/owncloud/ocis/issues/7208) - [apiSpacesShares/copySpaces.feature:696](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/copySpaces.feature#L696) - [apiSpacesShares/copySpaces.feature:715](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/copySpaces.feature#L715) @@ -29,7 +29,7 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiSpacesShares/shareUploadTUS.feature:303](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/shareUploadTUS.feature#L303) - [apiSpacesShares/shareUploadTUS.feature:384](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/shareUploadTUS.feature#L384) -### [Settings service user can list other peoples assignments](https://github.com/owncloud/ocis/issues/5032) +#### [Settings service user can list other peoples assignments](https://github.com/owncloud/ocis/issues/5032) - [apiAccountsHashDifficulty/assignRole.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiAccountsHashDifficulty/assignRole.feature#L27) - [apiAccountsHashDifficulty/assignRole.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiAccountsHashDifficulty/assignRole.feature#L28) @@ -83,16 +83,16 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiGraphUserGroup/addUserToGroup.feature:295](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiGraphUserGroup/addUserToGroup.feature#L295) -### [Users are added in a group with wrong host in host-part of user](https://github.com/owncloud/ocis/issues/5871) +#### [Users are added in a group with wrong host in host-part of user](https://github.com/owncloud/ocis/issues/5871) - [apiGraphUserGroup/addUserToGroup.feature:379](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiGraphUserGroup/addUserToGroup.feature#L379) - [apiGraphUserGroup/addUserToGroup.feature:393](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiGraphUserGroup/addUserToGroup.feature#L393) -### [Adding the same user as multiple members in a single request results in listing the same user twice in the group](https://github.com/owncloud/ocis/issues/5855) +#### [Adding the same user as multiple members in a single request results in listing the same user twice in the group](https://github.com/owncloud/ocis/issues/5855) - [apiGraphUserGroup/addUserToGroup.feature:430](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiGraphUserGroup/addUserToGroup.feature#L430) -### [Shared file locking is not possible using different path](https://github.com/owncloud/ocis/issues/7599) +#### [Shared file locking is not possible using different path](https://github.com/owncloud/ocis/issues/7599) - [apiLocks/lockFiles.feature:188](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L188) - [apiLocks/lockFiles.feature:189](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L189) @@ -170,7 +170,7 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiLocks/lockFiles.feature:423](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L423) - [apiLocks/lockFiles.feature:424](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L424) -### [Anonymous users can unlock a file shared to them through a public link if they get the lock token](https://github.com/owncloud/ocis/issues/7761) +#### [Anonymous users can unlock a file shared to them through a public link if they get the lock token](https://github.com/owncloud/ocis/issues/7761) - [apiLocks/unlockFiles.feature:42](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L42) - [apiLocks/unlockFiles.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L43) @@ -179,7 +179,7 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiLocks/unlockFiles.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L46) - [apiLocks/unlockFiles.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L47) -### [Trying to unlock a shared file with sharer's lock token gives 500](https://github.com/owncloud/ocis/issues/7767) +#### [Trying to unlock a shared file with sharer's lock token gives 500](https://github.com/owncloud/ocis/issues/7767) - [apiLocks/unlockFiles.feature:115](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L115) - [apiLocks/unlockFiles.feature:116](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L116) @@ -194,7 +194,7 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiLocks/unlockFiles.feature:147](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L147) - [apiLocks/unlockFiles.feature:148](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L148) -### [Anonymous user trying lock a file shared to them through a public link gives 405](https://github.com/owncloud/ocis/issues/7790) +#### [Anonymous user trying lock a file shared to them through a public link gives 405](https://github.com/owncloud/ocis/issues/7790) - [apiLocks/lockFiles.feature:538](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L538) - [apiLocks/lockFiles.feature:539](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L539) @@ -209,7 +209,7 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiLocks/lockFiles.feature:566](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L566) - [apiLocks/lockFiles.feature:567](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L567) -### [blocksDownload link type is not implemented yet (sharing-ng)](https://github.com/owncloud/ocis/issues/7879) +#### [blocksDownload link type is not implemented yet (sharing-ng)](https://github.com/owncloud/ocis/issues/7879) - [apiSharingNgLinkSharePermission/createLinkShare.feature:72](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L72) - [apiSharingNgLinkSharePermission/createLinkShare.feature:202](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L202) @@ -262,14 +262,14 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiSharingNg1/removeAccessToDrive.feature:206](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNg1/removeAccessToDrive.feature#L206) - [apiSharingNg1/removeAccessToDrive.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNg1/removeAccessToDrive.feature#L236) -### [sharee (editor role) MOVE a file by file-id into shared sub-folder returns 502](https://github.com/owncloud/ocis/issues/7617) +#### [sharee (editor role) MOVE a file by file-id into shared sub-folder returns 502](https://github.com/owncloud/ocis/issues/7617) - [apiSpacesDavOperation/moveByFileId.feature:469](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature#L469) - [apiSpacesDavOperation/moveByFileId.feature:470](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature#L470) - [apiSpacesDavOperation/moveByFileId.feature:732](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature#L732) - [apiSpacesDavOperation/moveByFileId.feature:733](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature#L733) -### [MOVE a file into same folder with same name returns 404 instead of 403](https://github.com/owncloud/ocis/issues/1976) +#### [MOVE a file into same folder with same name returns 404 instead of 403](https://github.com/owncloud/ocis/issues/1976) - [apiSpacesShares/moveSpaces.feature:69](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/moveSpaces.feature#L69) - [apiSpacesShares/moveSpaces.feature:70](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/moveSpaces.feature#L70) @@ -285,19 +285,21 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiSpacesDavOperation/moveByFileId.feature:492](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature#L492) - [apiSpacesDavOperation/moveByFileId.feature:493](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature#L493) -### [OCM. sharing issues](https://github.com/owncloud/ocis/issues/9534) +#### [OCM. sharing issues](https://github.com/owncloud/ocis/issues/9534) - [apiOcm/share.feature:12](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiOcm/share.feature#L12) - [apiOcm/share.feature:91](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiOcm/share.feature#L91) -### [OCM. user cannot see invite description and inviteUser email](https://github.com/owncloud/ocis/issues/9591) +#### [OCM. user cannot see invite description and inviteUser email](https://github.com/owncloud/ocis/issues/9591) - [apiOcm/createInvitation.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiOcm/createInvitation.feature#L63) -### [OCM. admin cannot get federated users if he hasn't connection with them ](https://github.com/owncloud/ocis/issues/9829) +#### [OCM. admin cannot get federated users if he hasn't connection with them ](https://github.com/owncloud/ocis/issues/9829) tests/acceptance/features/apiOcm/searchFederationUsers.feature + - [apiOcm/searchFederationUsers.feature:429](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiOcm/searchFederationUsers.feature#L429) - [apiOcm/searchFederationUsers.feature:601](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiOcm/searchFederationUsers.feature#L601) -- Note: always have an empty line at the end of this file. - The bash script that processes this file requires that the last line has a newline on the end. + +Note: always have an empty line at the end of this file. +The bash script that processes this file requires that the last line has a newline on the end. From d45efeacca74b42250dd52da0e7eb1b3b84a84f7 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 20 Aug 2024 17:57:42 +0545 Subject: [PATCH 13/71] test: fix archive tests --- ...ected-failures-localAPI-on-OCIS-storage.md | 5 -- .../features/apiArchiver/downloadById.feature | 20 ++--- .../features/bootstrap/ArchiverContext.php | 73 ++++++++++++++++--- vendor-bin/behat/composer.json | 46 ++++++------ 4 files changed, 94 insertions(+), 50 deletions(-) diff --git a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md index bcf8e516a6..d545cd63ea 100644 --- a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md @@ -13,11 +13,6 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiArchiver/downloadByPath.feature:171](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadByPath.feature#L171) - [apiArchiver/downloadByPath.feature:172](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadByPath.feature#L172) -#### [Downloaded /Shares tar contains resource (files|folder) with leading / in Response](https://github.com/owncloud/ocis/issues/4636) - -- [apiArchiver/downloadById.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadById.feature#L173) -- [apiArchiver/downloadById.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadById.feature#L174) - #### [Shared mount folder gets deleted when overwritten by a file from personal space](https://github.com/owncloud/ocis/issues/7208) - [apiSpacesShares/copySpaces.feature:696](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/copySpaces.feature#L696) diff --git a/tests/acceptance/features/apiArchiver/downloadById.feature b/tests/acceptance/features/apiArchiver/downloadById.feature index d30dbe92d0..968ee81a75 100644 --- a/tests/acceptance/features/apiArchiver/downloadById.feature +++ b/tests/acceptance/features/apiArchiver/downloadById.feature @@ -13,7 +13,7 @@ Feature: download multiple resources bundled into an archive Scenario Outline: download a single file Given user "Alice" has uploaded file with content "some data" to "/textfile0.txt" - When user "Alice" downloads the archive of "/textfile0.txt" using the resource id and setting these headers + When user "Alice" downloads the archive of "/textfile0.txt" using the resource id and setting these headers: | header | value | | User-Agent | | Then the HTTP status code should be "200" @@ -22,7 +22,7 @@ Feature: download multiple resources bundled into an archive | textfile0.txt | some data | Examples: | user-agent | archive-type | - | Linux | zip | + | Linux | tar | | Windows NT | zip | @@ -30,7 +30,7 @@ Feature: download multiple resources bundled into an archive Given user "Alice" has created folder "my_data" And user "Alice" has uploaded file with content "some data" to "/my_data/textfile0.txt" And user "Alice" has uploaded file with content "more data" to "/my_data/an_other_file.txt" - When user "Alice" downloads the archive of "/my_data" using the resource id and setting these headers + When user "Alice" downloads the archive of "/my_data" using the resource id and setting these headers: | header | value | | User-Agent | | Then the HTTP status code should be "200" @@ -41,7 +41,7 @@ Feature: download multiple resources bundled into an archive Examples: | user-agent | archive-type | | Linux | zip | - | Windows NT | zip | + | Windows NT | tar | Scenario: download multiple files and folders @@ -158,16 +158,16 @@ Feature: download multiple resources bundled into an archive | shareType | user | | permissionsRole | Viewer | And user "Brian" has a share "more_data" synced - When user "Brian" downloads the archive of "/Shares" using the resource id and setting these headers + When user "Brian" downloads the archive of "/Shares" using the resource id and setting these headers: | header | value | | User-Agent | | Then the HTTP status code should be "200" And the downloaded archive should contain these files: - | name | content | - | Shares/textfile0.txt | some data | - | Shares/textfile1.txt | other data | - | Shares/my_data/textfile2.txt | some data | - | Shares/more_data/an_other_file.txt | more data | + | name | content | + | textfile0.txt | some data | + | textfile1.txt | other data | + | my_data/textfile2.txt | some data | + | more_data/an_other_file.txt | more data | Examples: | user-agent | archive-type | | Linux | tar | diff --git a/tests/acceptance/features/bootstrap/ArchiverContext.php b/tests/acceptance/features/bootstrap/ArchiverContext.php index 256ef56e4c..55957f1028 100644 --- a/tests/acceptance/features/bootstrap/ArchiverContext.php +++ b/tests/acceptance/features/bootstrap/ArchiverContext.php @@ -28,6 +28,9 @@ use TestHelpers\HttpRequestHelper; use TestHelpers\SetupHelper; use PHPUnit\Framework\Assert; use Psr\Http\Message\ResponseInterface; +use splitbrain\PHPArchive\Tar; +use splitbrain\PHPArchive\Zip; +use splitbrain\PHPArchive\Archive; require_once 'bootstrap.php'; @@ -62,6 +65,38 @@ class ArchiverContext implements Context { ); } + /** + * @param string $type + * + * @return Archive + */ + public function getArchiveClass(string $type): Archive { + if ($type === 'zip') { + return new Zip(); + } elseif ($type === 'tar') { + return new Tar(); + } else { + throw new Exception('Unknown archive type: ' . $type); + } + } + + /** + * @param string $dir + * + * @return void + */ + public function removeDir(string $dir): void { + $items = \glob("$dir/*"); + foreach ($items as $item) { + if (\is_dir($item)) { + $this->removeDir($item); + } else { + \unlink($item); + } + } + \rmdir($dir); + } + /** * @param string $user * @param string $resource @@ -92,9 +127,10 @@ class ArchiverContext implements Context { } /** - * @When user :user downloads the archive of :resource using the resource :addressType and setting these headers + * @When /^user "([^"]*)" downloads the (zip|tar) archive of "([^"]*)" using the resource (id|ids|path|paths) and setting these headers:$/ * * @param string $user + * @param string $archiveType * @param string $resource * @param string $addressType id|path * @param TableNode $headersTable @@ -104,8 +140,9 @@ class ArchiverContext implements Context { * @throws GuzzleException * @throws Exception */ - public function userDownloadsTheArchive( + public function userDownloadsTheZipOrTarArchiveOfResourceUsingResourceIdOrPathAndSettingTheseHeaders( string $user, + string $archiveType, string $resource, string $addressType, TableNode $headersTable @@ -118,7 +155,7 @@ class ArchiverContext implements Context { foreach ($headersTable as $row) { $headers[$row['header']] = $row ['value']; } - $this->featureContext->setResponse($this->downloadArchive($user, $resource, $addressType, null, $headers)); + $this->featureContext->setResponse($this->downloadArchive($user, $resource, $addressType, $archiveType, null, $headers)); } /** @@ -140,13 +177,14 @@ class ArchiverContext implements Context { string $owner, string $addressType ): void { - $this->featureContext->setResponse($this->downloadArchive($downloader, $resource, $addressType, $owner)); + $this->featureContext->setResponse($this->downloadArchive($downloader, $resource, $addressType, null, $owner)); } /** * @param string $downloader * @param string $resource * @param string $addressType + * @param string|null $archiveType * @param string|null $owner * @param array|null $headers * @@ -158,12 +196,16 @@ class ArchiverContext implements Context { string $downloader, string $resource, string $addressType, + ?string $archiveType = null, ?string $owner = null, ?array $headers = null ): ResponseInterface { $owner = $owner ?? $downloader; $downloader = $this->featureContext->getActualUsername($downloader); $queryString = $this->getArchiverQueryString($owner, $resource, $addressType); + if ($archiveType !== null) { + $queryString .= '&output-format=' . $archiveType; + } return HttpRequestHelper::get( $this->featureContext->getBaseUrl() . '/archiver?' . $queryString, $this->featureContext->getStepLineRef(), @@ -220,28 +262,34 @@ class ArchiverContext implements Context { $this->featureContext->verifyTableNodeColumns($expectedFiles, ['name', 'content']); $contents = $this->featureContext->getResponse()->getBody()->getContents(); $tempFile = \tempnam(\sys_get_temp_dir(), 'OcAcceptanceTests_'); + $tempExtractFolder = $tempFile; \unlink($tempFile); // we only need the name $tempFile = $tempFile . '.' . $type; // it needs the extension \file_put_contents($tempFile, $contents); // open the archive - $archiveData = new RecursiveIteratorIterator( - new PharData($tempFile), - RecursiveIteratorIterator::SELF_FIRST - ); + $tar = $this->getArchiveClass($type); + $tar->open($tempFile); + $archiveData = $tar->contents(); + + // extract the archive + $tar->open($tempFile); + $tar->extract($tempExtractFolder); + $tar->close(); + foreach ($expectedFiles->getHash() as $expectedItem) { $expectedPath = trim($expectedItem['name'], "/"); $found = false; foreach ($archiveData as $info) { // get only the parent folder path for the given item - $actualPath = explode(".$type", $info->getPathname())[1]; - $actualPath = trim($actualPath, "/"); + $actualPath = $info->getPath(); if ($expectedPath === $actualPath) { - if (!$info->isDir()) { + if (!$info->getIsdir()) { + $fileContent = \file_get_contents("$tempExtractFolder/$actualPath"); Assert::assertEquals( $expectedItem['content'], - $info->getContent(), + $fileContent, __METHOD__ . " content of '" . $expectedPath . "' not as expected" ); @@ -255,5 +303,6 @@ class ArchiverContext implements Context { } } \unlink($tempFile); + $this->removeDir($tempExtractFolder); } } diff --git a/vendor-bin/behat/composer.json b/vendor-bin/behat/composer.json index 07344d2cc5..19d2f0a8ef 100644 --- a/vendor-bin/behat/composer.json +++ b/vendor-bin/behat/composer.json @@ -1,27 +1,27 @@ { - "config" : { - "platform": { - "php": "8.2" - }, - "allow-plugins": { - "composer/package-versions-deprecated": true - } + "config": { + "platform": { + "php": "8.2" }, - "require": { - "behat/behat": "^3.13", - "behat/gherkin": "^4.9", - "behat/mink": "1.7.1", - "friends-of-behat/mink-extension": "^2.7", - "ciaranmcnulty/behat-stepthroughextension" : "dev-master", - "rdx/behat-variables": "^1.2", - "sensiolabs/behat-page-object-extension": "^2.3", - "symfony/translation": "^5.4", - "sabre/xml": "^2.2", - "guzzlehttp/guzzle": "^7.7", - "phpunit/phpunit": "^9.6", - "laminas/laminas-ldap": "^2.15", - "ankitpokhrel/tus-php": "^2.3", - "wapmorgan/unified-archive": "^1.1.10", - "swaggest/json-schema": "^0.12.42" + "allow-plugins": { + "composer/package-versions-deprecated": true } + }, + "require": { + "behat/behat": "^3.13", + "behat/gherkin": "^4.9", + "behat/mink": "1.7.1", + "friends-of-behat/mink-extension": "^2.7", + "ciaranmcnulty/behat-stepthroughextension": "dev-master", + "rdx/behat-variables": "^1.2", + "sensiolabs/behat-page-object-extension": "^2.3", + "symfony/translation": "^5.4", + "sabre/xml": "^2.2", + "guzzlehttp/guzzle": "^7.7", + "phpunit/phpunit": "^9.6", + "laminas/laminas-ldap": "^2.15", + "ankitpokhrel/tus-php": "^2.3", + "swaggest/json-schema": "^0.12.42", + "splitbrain/php-archive": "^1.3" } +} From 29b9f9b07de15b9f8abc0b221b208ca96032a503 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 20 Aug 2024 18:12:41 +0545 Subject: [PATCH 14/71] test: fix COPY to mountpoint tests --- ...ected-failures-localAPI-on-OCIS-storage.md | 5 ----- .../apiArchiver/downloadByPath.feature | 6 +++--- .../apiSpacesShares/copySpaces.feature | 20 ++++++++++--------- .../features/bootstrap/ArchiverContext.php | 9 +++++---- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md index d545cd63ea..a30aabf4b7 100644 --- a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md @@ -13,11 +13,6 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiArchiver/downloadByPath.feature:171](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadByPath.feature#L171) - [apiArchiver/downloadByPath.feature:172](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadByPath.feature#L172) -#### [Shared mount folder gets deleted when overwritten by a file from personal space](https://github.com/owncloud/ocis/issues/7208) - -- [apiSpacesShares/copySpaces.feature:696](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/copySpaces.feature#L696) -- [apiSpacesShares/copySpaces.feature:715](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/copySpaces.feature#L715) - #### [PATCH request for TUS upload with wrong checksum gives incorrect response](https://github.com/owncloud/ocis/issues/1755) - [apiSpacesShares/shareUploadTUS.feature:283](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSpacesShares/shareUploadTUS.feature#L283) diff --git a/tests/acceptance/features/apiArchiver/downloadByPath.feature b/tests/acceptance/features/apiArchiver/downloadByPath.feature index a0394f75e3..36936d8ef4 100644 --- a/tests/acceptance/features/apiArchiver/downloadByPath.feature +++ b/tests/acceptance/features/apiArchiver/downloadByPath.feature @@ -13,7 +13,7 @@ Feature: download multiple resources bundled into an archive @issue-4637 Scenario Outline: download a single file Given user "Alice" has uploaded file with content "some data" to "/textfile0.txt" - When user "Alice" downloads the archive of "/home/textfile0.txt" using the resource path and setting these headers + When user "Alice" downloads the archive of "/home/textfile0.txt" using the resource path and setting these headers: | header | value | | User-Agent | | Then the HTTP status code should be "200" @@ -30,7 +30,7 @@ Feature: download multiple resources bundled into an archive Given user "Alice" has created folder "my_data" And user "Alice" has uploaded file with content "some data" to "/my_data/textfile0.txt" And user "Alice" has uploaded file with content "more data" to "/my_data/an_other_file.txt" - When user "Alice" downloads the archive of "/home/my_data" using the resource path and setting these headers + When user "Alice" downloads the archive of "/home/my_data" using the resource path and setting these headers: | header | value | | User-Agent | | Then the HTTP status code should be "200" @@ -156,7 +156,7 @@ Feature: download multiple resources bundled into an archive | shareType | user | | permissionsRole | Viewer | And user "Brian" has a share "more_data" synced - When user "Brian" downloads the archive of "/home/Shares" using the resource path and setting these headers + When user "Brian" downloads the archive of "/home/Shares" using the resource path and setting these headers: | header | value | | User-Agent | | Then the HTTP status code should be "200" diff --git a/tests/acceptance/features/apiSpacesShares/copySpaces.feature b/tests/acceptance/features/apiSpacesShares/copySpaces.feature index 8f54594726..43f219147e 100644 --- a/tests/acceptance/features/apiSpacesShares/copySpaces.feature +++ b/tests/acceptance/features/apiSpacesShares/copySpaces.feature @@ -706,10 +706,12 @@ Feature: copy file | permissionsRole | Editor | And user "Alice" has a share "BRIAN-Folder" synced When user "Alice" copies file "/textfile1.txt" from space "Personal" to "/BRIAN-Folder" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "204" - And for user "Alice" the content of the file "/BRIAN-Folder" of the space "Shares" should be "ownCloud test text file 1" - And as "Alice" file "/textfile1.txt" should exist - And user "Alice" should not have any received shares + Then the HTTP status code should be "400" + And as "Alice" folder "Shares/BRIAN-Folder/sample-folder" should exist + And as "Brian" folder "BRIAN-Folder/sample-folder" should exist + But as "Alice" file "Shares/BRIAN-Folder" should not exist + And as "Alice" file "Shares/textfile1.txt" should not exist + And user "Alice" should have a share "BRIAN-Folder" shared by user "Brian" @issue-7208 Scenario: copy a folder over the top of an existing file received as a user share @@ -725,11 +727,11 @@ Feature: copy file | permissionsRole | File Editor | And user "Alice" has a share "sharedfile1.txt" synced When user "Alice" copies folder "/FOLDER" from space "Personal" to "/sharedfile1.txt" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "204" - And as "Alice" folder "/FOLDER/sample-folder" should exist - And for user "Alice" folder "/sharedfile1.txt" of the space "Shares" should contain these files: - | /sample-folder | - And user "Alice" should not have any received shares + Then the HTTP status code should be "400" + And for user "Alice" the content of the file "sharedfile1.txt" of the space "Shares" should be "file to share" + And for user "Brian" the content of the file "sharedfile1.txt" of the space "Personal" should be "file to share" + But as "Alice" folder "Shares/FOLDER/sample-folder" should not exist + And user "Alice" should have a share "sharedfile1.txt" shared by user "Brian" Scenario: copy a folder into another folder at different level which is received as a user share diff --git a/tests/acceptance/features/bootstrap/ArchiverContext.php b/tests/acceptance/features/bootstrap/ArchiverContext.php index 55957f1028..c31865b6be 100644 --- a/tests/acceptance/features/bootstrap/ArchiverContext.php +++ b/tests/acceptance/features/bootstrap/ArchiverContext.php @@ -86,12 +86,13 @@ class ArchiverContext implements Context { * @return void */ public function removeDir(string $dir): void { - $items = \glob("$dir/*"); + $items = array_diff(scandir($dir), ['.', '..']); foreach ($items as $item) { - if (\is_dir($item)) { - $this->removeDir($item); + $itemPath = $dir . DIRECTORY_SEPARATOR . $item; + if (\is_dir($itemPath)) { + $this->removeDir($itemPath); } else { - \unlink($item); + \unlink($itemPath); } } \rmdir($dir); From b8c4a0f4bef919f143cf8d2420491c36a082c496 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Thu, 22 Aug 2024 11:23:47 +0200 Subject: [PATCH 15/71] add changelog Signed-off-by: Christian Richter --- changelog/unreleased/remove-store-service.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/remove-store-service.md diff --git a/changelog/unreleased/remove-store-service.md b/changelog/unreleased/remove-store-service.md new file mode 100644 index 0000000000..067509c6fb --- /dev/null +++ b/changelog/unreleased/remove-store-service.md @@ -0,0 +1,6 @@ +Change: Remove store service + +We have removed the unused store service. + +https://github.com/owncloud/ocis/pull/9890 +https://github.com/owncloud/ocis/issues/1357 From f2505d97d62c573062dbfb5b7a293850f6e53ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 22 Aug 2024 12:40:01 +0000 Subject: [PATCH 16/71] Automated changelog update [skip ci] --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e855a2c7e6..00891f600d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The following sections list the changes for unreleased. * Bugfix - Set capability response `disable_self_password_change` correctly: [#9853](https://github.com/owncloud/ocis/pull/9853) * Bugfix - Activity Translations: [#9856](https://github.com/owncloud/ocis/pull/9856) * Bugfix - The user attributes `userType` and `memberOf` are readonly: [#9867](https://github.com/owncloud/ocis/pull/9867) +* Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) ## Details @@ -81,6 +82,13 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/9858 https://github.com/owncloud/ocis/pull/9867 +* Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) + + We have removed the unused store service. + + https://github.com/owncloud/ocis/issues/1357 + https://github.com/owncloud/ocis/pull/9890 + # Changelog for [6.3.0] (2024-08-20) The following sections list the changes for 6.3.0. From 233d108dbefd4e2e24fcaa39021dfee8e900dc96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:27:00 +0000 Subject: [PATCH 17/71] Bump go.etcd.io/bbolt from 1.3.10 to 1.3.11 Bumps [go.etcd.io/bbolt](https://github.com/etcd-io/bbolt) from 1.3.10 to 1.3.11. - [Release notes](https://github.com/etcd-io/bbolt/releases) - [Commits](https://github.com/etcd-io/bbolt/compare/v1.3.10...v1.3.11) --- updated-dependencies: - dependency-name: go.etcd.io/bbolt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- vendor/go.etcd.io/bbolt/.go-version | 2 +- vendor/go.etcd.io/bbolt/Makefile | 13 +++++++++++++ vendor/go.etcd.io/bbolt/db.go | 8 +++++--- vendor/go.etcd.io/bbolt/freelist.go | 8 ++++++++ vendor/go.etcd.io/bbolt/tx.go | 7 +++++++ vendor/modules.txt | 4 ++-- 8 files changed, 39 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index fca175c96d..bbeec9d440 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,7 @@ require ( github.com/urfave/cli/v2 v2.27.4 github.com/xhit/go-simple-mail/v2 v2.16.0 go-micro.dev/v4 v4.11.0 - go.etcd.io/bbolt v1.3.10 + go.etcd.io/bbolt v1.3.11 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 go.opentelemetry.io/contrib/zpages v0.53.0 diff --git a/go.sum b/go.sum index 81a75fd482..48c8967751 100644 --- a/go.sum +++ b/go.sum @@ -1218,8 +1218,8 @@ github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ go-micro.dev/v4 v4.11.0 h1:DZ2xcr0pnZJDlp6MJiCLhw4tXRxLw9xrJlPT91kubr0= go-micro.dev/v4 v4.11.0/go.mod h1:eE/tD53n3KbVrzrWxKLxdkGw45Fg1qaNLWjpJMvIUF4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= -go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= diff --git a/vendor/go.etcd.io/bbolt/.go-version b/vendor/go.etcd.io/bbolt/.go-version index f124bfa155..013173af5e 100644 --- a/vendor/go.etcd.io/bbolt/.go-version +++ b/vendor/go.etcd.io/bbolt/.go-version @@ -1 +1 @@ -1.21.9 +1.22.6 diff --git a/vendor/go.etcd.io/bbolt/Makefile b/vendor/go.etcd.io/bbolt/Makefile index 18154c6388..2140779741 100644 --- a/vendor/go.etcd.io/bbolt/Makefile +++ b/vendor/go.etcd.io/bbolt/Makefile @@ -41,6 +41,15 @@ coverage: TEST_FREELIST_TYPE=array go test -v -timeout 30m \ -coverprofile cover-freelist-array.out -covermode atomic +BOLT_CMD=bbolt + +build: + go build -o bin/${BOLT_CMD} ./cmd/${BOLT_CMD} + +.PHONY: clean +clean: # Clean binaries + rm -f ./bin/${BOLT_CMD} + .PHONY: gofail-enable gofail-enable: install-gofail gofail enable . @@ -61,3 +70,7 @@ test-failpoint: @echo "[failpoint] array freelist test" TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint +.PHONY: test-robustness # Running robustness tests requires root permission +test-robustness: + go test -v ${TESTFLAGS} ./tests/dmflakey -test.root + go test -v ${TESTFLAGS} ./tests/robustness -test.root diff --git a/vendor/go.etcd.io/bbolt/db.go b/vendor/go.etcd.io/bbolt/db.go index 4175bdf3dd..822798e41a 100644 --- a/vendor/go.etcd.io/bbolt/db.go +++ b/vendor/go.etcd.io/bbolt/db.go @@ -524,7 +524,7 @@ func (db *DB) munmap() error { // gofail: var unmapError string // return errors.New(unmapError) if err := munmap(db); err != nil { - return fmt.Errorf("unmap error: " + err.Error()) + return fmt.Errorf("unmap error: %v", err.Error()) } return nil @@ -571,7 +571,7 @@ func (db *DB) munlock(fileSize int) error { // gofail: var munlockError string // return errors.New(munlockError) if err := munlock(db, fileSize); err != nil { - return fmt.Errorf("munlock error: " + err.Error()) + return fmt.Errorf("munlock error: %v", err.Error()) } return nil } @@ -580,7 +580,7 @@ func (db *DB) mlock(fileSize int) error { // gofail: var mlockError string // return errors.New(mlockError) if err := mlock(db, fileSize); err != nil { - return fmt.Errorf("mlock error: " + err.Error()) + return fmt.Errorf("mlock error: %v", err.Error()) } return nil } @@ -1159,6 +1159,8 @@ func (db *DB) grow(sz int) error { // https://github.com/boltdb/bolt/issues/284 if !db.NoGrowSync && !db.readOnly { if runtime.GOOS != "windows" { + // gofail: var resizeFileError string + // return errors.New(resizeFileError) if err := db.file.Truncate(int64(sz)); err != nil { return fmt.Errorf("file resize error: %s", err) } diff --git a/vendor/go.etcd.io/bbolt/freelist.go b/vendor/go.etcd.io/bbolt/freelist.go index 61d43f81b4..dffc7bc749 100644 --- a/vendor/go.etcd.io/bbolt/freelist.go +++ b/vendor/go.etcd.io/bbolt/freelist.go @@ -252,6 +252,14 @@ func (f *freelist) rollback(txid txid) { } // Remove pages from pending list and mark as free if allocated by txid. delete(f.pending, txid) + + // Remove pgids which are allocated by this txid + for pgid, tid := range f.allocs { + if tid == txid { + delete(f.allocs, pgid) + } + } + f.mergeSpans(m) } diff --git a/vendor/go.etcd.io/bbolt/tx.go b/vendor/go.etcd.io/bbolt/tx.go index 2fac8c0a78..766395de3b 100644 --- a/vendor/go.etcd.io/bbolt/tx.go +++ b/vendor/go.etcd.io/bbolt/tx.go @@ -1,6 +1,7 @@ package bbolt import ( + "errors" "fmt" "io" "os" @@ -185,6 +186,10 @@ func (tx *Tx) Commit() error { // If the high water mark has moved up then attempt to grow the database. if tx.meta.pgid > opgid { + _ = errors.New("") + // gofail: var lackOfDiskSpace string + // tx.rollback() + // return errors.New(lackOfDiskSpace) if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { tx.rollback() return err @@ -470,6 +475,7 @@ func (tx *Tx) write() error { // Ignore file sync if flag is set on DB. if !tx.db.NoSync || IgnoreNoSync { + // gofail: var beforeSyncDataPages struct{} if err := fdatasync(tx.db); err != nil { return err } @@ -507,6 +513,7 @@ func (tx *Tx) writeMeta() error { return err } if !tx.db.NoSync || IgnoreNoSync { + // gofail: var beforeSyncMetaPage struct{} if err := fdatasync(tx.db); err != nil { return err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 834a70895d..f0cb12bbdf 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1993,8 +1993,8 @@ go-micro.dev/v4/util/ring go-micro.dev/v4/util/signal go-micro.dev/v4/util/socket go-micro.dev/v4/util/tls -# go.etcd.io/bbolt v1.3.10 -## explicit; go 1.21 +# go.etcd.io/bbolt v1.3.11 +## explicit; go 1.22 go.etcd.io/bbolt # go.etcd.io/etcd/api/v3 v3.5.12 ## explicit; go 1.20 From 1c4f53d774e7815b5c5217ddfc996d614c213b41 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 22 Aug 2024 14:47:09 +0200 Subject: [PATCH 18/71] feat(activitylog): default locale Signed-off-by: jkoberg --- .../unreleased/default-locale-activitylog.md | 5 ++++ services/activitylog/README.md | 26 +++++++++++++++++++ services/activitylog/pkg/config/config.go | 3 +++ .../pkg/config/defaults/defaultconfig.go | 3 ++- services/activitylog/pkg/service/http.go | 3 +-- 5 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/default-locale-activitylog.md diff --git a/changelog/unreleased/default-locale-activitylog.md b/changelog/unreleased/default-locale-activitylog.md new file mode 100644 index 0000000000..695dec4ef0 --- /dev/null +++ b/changelog/unreleased/default-locale-activitylog.md @@ -0,0 +1,5 @@ +Enhancement: Allow setting default locale of activitylog + +Allows setting the default locale via `OCIS_DEFAULT_LANGUAGE` envvar + +https://github.com/owncloud/ocis/pull/9892 diff --git a/services/activitylog/README.md b/services/activitylog/README.md index b7238194a8..6c5642e967 100644 --- a/services/activitylog/README.md +++ b/services/activitylog/README.md @@ -13,3 +13,29 @@ Log services like the `activitylog`, `userlog`, `clientlog` and `sse` are respon ## Activitylog Store The `activitylog` stores activities for each resource. It works in conjunction with the `eventhistory` service to keep the data it needs to store to a minimum. + +## Translations + +The `activitylog` service has embedded translations sourced via transifex to provide a basic set of translated languages. These embedded translations are available for all deployment scenarios. In addition, the service supports custom translations, though it is currently not possible to just add custom translations to embedded ones. If custom translations are configured, the embedded ones are not used. To configure custom translations, the `ACTIVITYLOG_TRANSLATION_PATH` environment variable needs to point to a base folder that will contain the translation files. This path must be available from all instances of the userlog service, a shared storage is recommended. Translation files must be of type [.po](https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html#PO-Files) or [.mo](https://www.gnu.org/software/gettext/manual/html_node/Binaries.html). For each language, the filename needs to be `userlog.po` (or `userlog.mo`) and stored in a folder structure defining the language code. In general the path/name pattern for a translation file needs to be: + +```text +{ACTIVITYLOG_TRANSLATION_PATH}/{language-code}/LC_MESSAGES/activitylog.po +``` + +The language code pattern is composed of `language[_territory]` where `language` is the base language and `_territory` is optional and defines a country. + +For example, for the language `de`, one needs to place the corresponding translation files to `{ACTIVITYLOG_TRANSLATION_PATH}/de_DE/LC_MESSAGES/activitylog.po`. + + + +Important: For the time being, the embedded ownCloud Web frontend only supports the main language code but does not handle any territory. When strings are available in the language code `language_territory`, the web frontend does not see it as it only requests `language`. In consequence, any translations made must exist in the requested `language` to avoid a fallback to the default. + +### Translation Rules + +* If a requested language code is not available, the service tries to fall back to the base language if available. For example, if the requested language-code `de_DE` is not available, the service tries to fall back to translations in the `de` folder. +* If the base language `de` is also not available, the service falls back to the system's default English (`en`), +which is the source of the texts provided by the code. + +## Default Language + +The default language can be defined via the `OCIS_DEFAULT_LANGUAGE` environment variable. See the `settings` service for a detailed description. diff --git a/services/activitylog/pkg/config/config.go b/services/activitylog/pkg/config/config.go index 7c84b13178..67f6fe6dd4 100644 --- a/services/activitylog/pkg/config/config.go +++ b/services/activitylog/pkg/config/config.go @@ -26,6 +26,9 @@ type Config struct { HTTP HTTP `yaml:"http"` TokenManager *TokenManager `yaml:"token_manager"` + TranslationPath string `yaml:"translation_path" env:"OCIS_TRANSLATION_PATH;ACTIVITYLOG_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. Note that file and folder naming rules apply, see the documentation for more details." introductionVersion:"6.3"` + DefaultLanguage string `yaml:"default_language" env:"OCIS_DEFAULT_LANGUAGE" desc:"The default language used by services and the WebUI. If not defined, English will be used as default. See the documentation for more details." introductionVersion:"6.3"` + ServiceAccount ServiceAccount `yaml:"service_account"` Context context.Context `yaml:"-"` diff --git a/services/activitylog/pkg/config/defaults/defaultconfig.go b/services/activitylog/pkg/config/defaults/defaultconfig.go index 00103af6be..0f434e5baf 100644 --- a/services/activitylog/pkg/config/defaults/defaultconfig.go +++ b/services/activitylog/pkg/config/defaults/defaultconfig.go @@ -37,7 +37,8 @@ func DefaultConfig() *config.Config { Database: "activitylog", Table: "", }, - RevaGateway: shared.DefaultRevaConfig().Address, + RevaGateway: shared.DefaultRevaConfig().Address, + DefaultLanguage: "en", HTTP: config.HTTP{ Addr: "127.0.0.1:0", Root: "/", diff --git a/services/activitylog/pkg/service/http.go b/services/activitylog/pkg/service/http.go index d47e2e7181..4c2f54064d 100644 --- a/services/activitylog/pkg/service/http.go +++ b/services/activitylog/pkg/service/http.go @@ -164,9 +164,8 @@ func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *h continue } - // FIXME: configurable default locale? loc := l10n.MustGetUserLocale(r.Context(), activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), s.valService) - t := l10n.NewTranslatorFromCommonConfig("en", _domain, "", _localeFS, _localeSubPath) + t := l10n.NewTranslatorFromCommonConfig(s.cfg.DefaultLanguage, _domain, s.cfg.TranslationPath, _localeFS, _localeSubPath) resp.Activities = append(resp.Activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars)) } From db550ed82813b5b308b167e6800f24231bfdf370 Mon Sep 17 00:00:00 2001 From: kobergj Date: Thu, 22 Aug 2024 15:59:35 +0200 Subject: [PATCH 19/71] docs(activitylog): use NEXT version Co-authored-by: Martin --- services/activitylog/README.md | 2 +- services/activitylog/pkg/config/config.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/activitylog/README.md b/services/activitylog/README.md index 6c5642e967..1b446f1b37 100644 --- a/services/activitylog/README.md +++ b/services/activitylog/README.md @@ -16,7 +16,7 @@ The `activitylog` stores activities for each resource. It works in conjunction w ## Translations -The `activitylog` service has embedded translations sourced via transifex to provide a basic set of translated languages. These embedded translations are available for all deployment scenarios. In addition, the service supports custom translations, though it is currently not possible to just add custom translations to embedded ones. If custom translations are configured, the embedded ones are not used. To configure custom translations, the `ACTIVITYLOG_TRANSLATION_PATH` environment variable needs to point to a base folder that will contain the translation files. This path must be available from all instances of the userlog service, a shared storage is recommended. Translation files must be of type [.po](https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html#PO-Files) or [.mo](https://www.gnu.org/software/gettext/manual/html_node/Binaries.html). For each language, the filename needs to be `userlog.po` (or `userlog.mo`) and stored in a folder structure defining the language code. In general the path/name pattern for a translation file needs to be: +The `activitylog` service has embedded translations sourced via transifex to provide a basic set of translated languages. These embedded translations are available for all deployment scenarios. In addition, the service supports custom translations, though it is currently not possible to just add custom translations to embedded ones. If custom translations are configured, the embedded ones are not used. To configure custom translations, the `ACTIVITYLOG_TRANSLATION_PATH` environment variable needs to point to a base folder that will contain the translation files. This path must be available from all instances of the activitylog service, a shared storage is recommended. Translation files must be of type [.po](https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html#PO-Files) or [.mo](https://www.gnu.org/software/gettext/manual/html_node/Binaries.html). For each language, the filename needs to be `activitylog.po` (or `activitylog.mo`) and stored in a folder structure defining the language code. In general the path/name pattern for a translation file needs to be: ```text {ACTIVITYLOG_TRANSLATION_PATH}/{language-code}/LC_MESSAGES/activitylog.po diff --git a/services/activitylog/pkg/config/config.go b/services/activitylog/pkg/config/config.go index 67f6fe6dd4..d4472f0884 100644 --- a/services/activitylog/pkg/config/config.go +++ b/services/activitylog/pkg/config/config.go @@ -26,8 +26,8 @@ type Config struct { HTTP HTTP `yaml:"http"` TokenManager *TokenManager `yaml:"token_manager"` - TranslationPath string `yaml:"translation_path" env:"OCIS_TRANSLATION_PATH;ACTIVITYLOG_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. Note that file and folder naming rules apply, see the documentation for more details." introductionVersion:"6.3"` - DefaultLanguage string `yaml:"default_language" env:"OCIS_DEFAULT_LANGUAGE" desc:"The default language used by services and the WebUI. If not defined, English will be used as default. See the documentation for more details." introductionVersion:"6.3"` + TranslationPath string `yaml:"translation_path" env:"OCIS_TRANSLATION_PATH;ACTIVITYLOG_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. Note that file and folder naming rules apply, see the documentation for more details." introductionVersion:"%%NEXT%%"` + DefaultLanguage string `yaml:"default_language" env:"OCIS_DEFAULT_LANGUAGE" desc:"The default language used by services and the WebUI. If not defined, English will be used as default. See the documentation for more details." introductionVersion:"%%NEXT%%"` ServiceAccount ServiceAccount `yaml:"service_account"` From 7d11d9e05ea85c59da54d601ebc2679a4b6a4e06 Mon Sep 17 00:00:00 2001 From: kobergj Date: Thu, 22 Aug 2024 14:32:02 +0000 Subject: [PATCH 20/71] Automated changelog update [skip ci] --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00891f600d..c9a631e378 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ The following sections list the changes for unreleased. * Bugfix - Activity Translations: [#9856](https://github.com/owncloud/ocis/pull/9856) * Bugfix - The user attributes `userType` and `memberOf` are readonly: [#9867](https://github.com/owncloud/ocis/pull/9867) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) +* Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) ## Details @@ -89,6 +90,12 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/1357 https://github.com/owncloud/ocis/pull/9890 +* Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) + + Allows setting the default locale via `OCIS_DEFAULT_LANGUAGE` envvar + + https://github.com/owncloud/ocis/pull/9892 + # Changelog for [6.3.0] (2024-08-20) The following sections list the changes for 6.3.0. From e548f29b8436c386a441102abca04aab3ff927e2 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 21 Aug 2024 16:41:42 +0200 Subject: [PATCH 21/71] feat(ocis): benchmark revision listing possiblities Signed-off-by: jkoberg --- .../unreleased/improve-revisions-purge.md | 5 + ocis/pkg/command/revisions.go | 71 +++++-- ocis/pkg/revisions/revisions.go | 135 ++++++++----- ocis/pkg/revisions/revisions_test.go | 182 ++++++++++-------- 4 files changed, 250 insertions(+), 143 deletions(-) create mode 100644 changelog/unreleased/improve-revisions-purge.md diff --git a/changelog/unreleased/improve-revisions-purge.md b/changelog/unreleased/improve-revisions-purge.md new file mode 100644 index 0000000000..ac4a5055e6 --- /dev/null +++ b/changelog/unreleased/improve-revisions-purge.md @@ -0,0 +1,5 @@ +Enhancement: Improve revisions purge + +The `revisions purge` command would time out on big spaces. We have improved performance by parallelizing the process. + +https://github.com/owncloud/ocis/pull/9891 diff --git a/ocis/pkg/command/revisions.go b/ocis/pkg/command/revisions.go index 174a9df3a6..569b702382 100644 --- a/ocis/pkg/command/revisions.go +++ b/ocis/pkg/command/revisions.go @@ -5,6 +5,7 @@ import ( "fmt" "path/filepath" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" @@ -19,7 +20,7 @@ import ( var ( // _nodesGlobPattern is the glob pattern to find all nodes - _nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*" + _nodesGlobPattern = "spaces/*/*/nodes/" ) // RevisionsCommand is the entrypoint for the revisions command. @@ -30,7 +31,7 @@ func RevisionsCommand(cfg *config.Config) *cli.Command { Subcommands: []*cli.Command{ PurgeRevisionsCommand(cfg), }, - Before: func(c *cli.Context) error { + Before: func(_ *cli.Context) error { return configlog.ReturnError(parser.ParseConfig(cfg, true)) }, Action: func(_ *cli.Context) error { @@ -74,6 +75,11 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { Aliases: []string{"r"}, Usage: "purge all revisions of this file/space. If not set, all revisions will be purged", }, + &cli.StringFlag{ + Name: "glob-mechanism", + Usage: "the glob mechanism to find all nodes. Can be 'glob', 'list' or 'workers'. 'glob' uses globbing with a single worker. 'workers' spawns multiple go routines, accelatering the command drastically but causing high cpu and ram usage. 'list' looks for references by listing directories with multiple workers. Default is 'glob'", + Value: "glob", + }, }, Action: func(c *cli.Context) error { basePath := c.String("basepath") @@ -108,43 +114,72 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { return err } - p, err := generatePath(basePath, c.String("resource-id")) - if err != nil { - fmt.Printf("❌ Error parsing resourceID: %s", err) - return err + var rid *provider.ResourceId + resid, err := storagespace.ParseID(c.String("resource-id")) + if err == nil { + rid = &resid } - if err := revisions.PurgeRevisions(p, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil { - fmt.Printf("❌ Error purging revisions: %s", err) - return err + mechanism := c.String("glob-mechanism") + if rid.GetOpaqueId() != "" { + mechanism = "glob" } + var ch <-chan string + switch mechanism { + default: + fallthrough + case "glob": + p := generatePath(basePath, rid) + if rid.GetOpaqueId() == "" { + p = filepath.Join(p, "*/*/*/*/*") + } + ch = revisions.Glob(p) + case "workers": + p := generatePath(basePath, rid) + ch = revisions.GlobWorkers(p, "/*", "/*/*/*/*") + case "list": + p := filepath.Join(basePath, "spaces") + if rid != nil { + p = generatePath(basePath, rid) + } + ch = revisions.List(p, 10) + } + + files, blobs, revisions := revisions.PurgeRevisions(ch, bs, c.Bool("dry-run"), c.Bool("verbose")) + printResults(files, blobs, revisions, c.Bool("dry-run")) return nil }, } } -func generatePath(basePath string, resourceID string) (string, error) { - if resourceID == "" { - return filepath.Join(basePath, _nodesGlobPattern), nil +func printResults(countFiles, countBlobs, countRevisions int, dryRun bool) { + switch { + case countFiles == 0 && countRevisions == 0 && countBlobs == 0: + fmt.Println("❎ No revisions found. Storage provider is clean.") + case !dryRun: + fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + default: + fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) } +} - rid, err := storagespace.ParseID(resourceID) - if err != nil { - return "", err +func generatePath(basePath string, rid *provider.ResourceId) string { + if rid == nil { + return filepath.Join(basePath, _nodesGlobPattern) } sid := lookup.Pathify(rid.GetSpaceId(), 1, 2) if sid == "" { - sid = "*/*" + return "" } nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2) if nid == "" { - nid = "*/*/*/*/" + return filepath.Join(basePath, "spaces", sid, "nodes") } - return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*"), nil + return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*") } func init() { diff --git a/ocis/pkg/revisions/revisions.go b/ocis/pkg/revisions/revisions.go index d2d1eac686..bc4dd00a3d 100644 --- a/ocis/pkg/revisions/revisions.go +++ b/ocis/pkg/revisions/revisions.go @@ -7,6 +7,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/shamaton/msgpack/v2" @@ -25,16 +26,12 @@ type DelBlobstore interface { Delete(node *node.Node) error } -// PurgeRevisionsGlob removes all revisions from a storage provider using globbing. -func PurgeRevisionsGlob(pattern string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { - if verbose { - fmt.Println("Looking for nodes in", pattern) - } - +// Glob uses globbing to find all revision nodes in a storage provider. +func Glob(pattern string) <-chan string { ch := make(chan string) go func() { defer close(ch) - nodes, err := filepath.Glob(pattern) + nodes, err := filepath.Glob(filepath.Join(pattern)) if err != nil { fmt.Println("error globbing", pattern, err) return @@ -52,11 +49,51 @@ func PurgeRevisionsGlob(pattern string, bs DelBlobstore, dryRun bool, verbose bo } }() - return purgeRevisions(ch, bs, dryRun, verbose) + return ch } -// PurgeRevisionsWalk removes all revisions from a storage provider using walking. -func PurgeRevisionsWalk(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { +// GlobWorkers uses multiple go routine to glob all revision nodes in a storage provider. +func GlobWorkers(pattern string, depth string, remainder string) <-chan string { + wg := sync.WaitGroup{} + ch := make(chan string) + go func() { + defer close(ch) + nodes, err := filepath.Glob(pattern + depth) + if err != nil { + fmt.Println("error globbing", pattern, err) + return + } + + if len(nodes) == 0 { + fmt.Println("no nodes found. Double check storage path") + return + } + + for _, node := range nodes { + wg.Add(1) + go func(node string) { + defer wg.Done() + nodes, err := filepath.Glob(node + remainder) + if err != nil { + fmt.Println("error globbing", node, err) + return + } + for _, n := range nodes { + if _versionRegex.MatchString(n) { + ch <- n + } + } + }(node) + } + + wg.Wait() + }() + + return ch +} + +// Walk walks the storage provider to find all revision nodes. +func Walk(base string) <-chan string { ch := make(chan string) go func() { defer close(ch) @@ -79,58 +116,25 @@ func PurgeRevisionsWalk(base string, bs DelBlobstore, dryRun bool, verbose bool) } }() - return purgeRevisions(ch, bs, dryRun, verbose) + return ch } -// PurgeRevisionsList removes all revisions from a storage provider using listing. -func PurgeRevisionsList(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { +// List uses directory listing to find all revision nodes in a storage provider. +func List(base string, workers int) <-chan string { ch := make(chan string) go func() { defer close(ch) - if err := listFolder(base, ch); err != nil { + if err := listFolder(base, ch, make(chan struct{}, workers)); err != nil { fmt.Println("error listing", base, err) return } }() - return purgeRevisions(ch, bs, dryRun, verbose) + return ch } -func listFolder(path string, ch chan<- string) error { - children, err := os.ReadDir(path) - if err != nil { - return err - } - - for _, child := range children { - if child.IsDir() { - if err := listFolder(filepath.Join(path, child.Name()), ch); err != nil { - return err - } - } - - if _versionRegex.MatchString(child.Name()) { - ch <- filepath.Join(path, child.Name()) - } - - } - return nil -} - -// PrintResults prints the results -func PrintResults(countFiles, countBlobs, countRevisions int, dryRun bool) error { - switch { - case countFiles == 0 && countRevisions == 0 && countBlobs == 0: - fmt.Println("❎ No revisions found. Storage provider is clean.") - case !dryRun: - fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) - default: - fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) - } - return nil -} - -func purgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) { +// PurgeRevisions removes all revisions from a storage provider. +func PurgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) { countFiles := 0 countBlobs := 0 countRevisions := 0 @@ -201,6 +205,37 @@ func purgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) return countFiles, countBlobs, countRevisions } +func listFolder(path string, ch chan<- string, workers chan struct{}) error { + workers <- struct{}{} + wg := sync.WaitGroup{} + + children, err := os.ReadDir(path) + if err != nil { + <-workers + return err + } + + for _, child := range children { + if child.IsDir() { + wg.Add(1) + go func() { + defer wg.Done() + if err := listFolder(filepath.Join(path, child.Name()), ch, workers); err != nil { + fmt.Println("error listing", path, err) + } + }() + } + + if _versionRegex.MatchString(child.Name()) { + ch <- filepath.Join(path, child.Name()) + } + + } + <-workers + wg.Wait() + return nil +} + func getBlobID(path string) (string, error) { b, err := os.ReadFile(path) if err != nil { diff --git a/ocis/pkg/revisions/revisions_test.go b/ocis/pkg/revisions/revisions_test.go index 477ccbebd9..c120d9ff42 100644 --- a/ocis/pkg/revisions/revisions_test.go +++ b/ocis/pkg/revisions/revisions_test.go @@ -14,7 +14,7 @@ import ( ) var ( - _basePath = "test_temp/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/" + _basePath = "/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/" ) // func TestInit(t *testing.T) { @@ -22,117 +22,149 @@ var ( // defer os.RemoveAll("test_temp") // } -func TestGlob30(t *testing.T) { testGlob(t, 10, 2) } -func TestGlob80(t *testing.T) { testGlob(t, 20, 3) } -func TestGlob250(t *testing.T) { testGlob(t, 50, 4) } -func TestGlob600(t *testing.T) { testGlob(t, 100, 5) } +func TestGlob30(t *testing.T) { test(t, 10, 2, glob) } +func TestGlob80(t *testing.T) { test(t, 20, 3, glob) } +func TestGlob250(t *testing.T) { test(t, 50, 4, glob) } +func TestGlob600(t *testing.T) { test(t, 100, 5, glob) } -func TestWalk30(t *testing.T) { testWalk(t, 10, 2) } -func TestWalk80(t *testing.T) { testWalk(t, 20, 3) } -func TestWalk250(t *testing.T) { testWalk(t, 50, 4) } -func TestWalk600(t *testing.T) { testWalk(t, 100, 5) } +func TestWalk30(t *testing.T) { test(t, 10, 2, walk) } +func TestWalk80(t *testing.T) { test(t, 20, 3, walk) } +func TestWalk250(t *testing.T) { test(t, 50, 4, walk) } +func TestWalk600(t *testing.T) { test(t, 100, 5, walk) } -func TestList30(t *testing.T) { testList(t, 10, 2) } -func TestList80(t *testing.T) { testList(t, 20, 3) } -func TestList250(t *testing.T) { testList(t, 50, 4) } -func TestList600(t *testing.T) { testList(t, 100, 5) } +func TestList30(t *testing.T) { test(t, 10, 2, list2) } +func TestList80(t *testing.T) { test(t, 20, 3, list10) } +func TestList250(t *testing.T) { test(t, 50, 4, list20) } +func TestList600(t *testing.T) { test(t, 100, 5, list2) } -func BenchmarkGlob30(b *testing.B) { benchmarkGlob(b, 10, 2) } -func BenchmarkWalk30(b *testing.B) { benchmarkWalk(b, 10, 2) } -func BenchmarkList30(b *testing.B) { benchmarkList(b, 10, 2) } +func TestGlobWorkers30(t *testing.T) { test(t, 10, 2, globWorkersD1) } +func TestGlobWorkers80(t *testing.T) { test(t, 20, 3, globWorkersD2) } +func TestGlobWorkers250(t *testing.T) { test(t, 50, 4, globWorkersD4) } +func TestGlobWorkers600(t *testing.T) { test(t, 100, 5, globWorkersD2) } -func BenchmarkGlob80(b *testing.B) { benchmarkGlob(b, 20, 3) } -func BenchmarkWalk80(b *testing.B) { benchmarkWalk(b, 20, 3) } -func BenchmarkList80(b *testing.B) { benchmarkList(b, 20, 3) } +func BenchmarkGlob30(b *testing.B) { benchmark(b, 10, 2, glob) } +func BenchmarkWalk30(b *testing.B) { benchmark(b, 10, 2, walk) } +func BenchmarkList30(b *testing.B) { benchmark(b, 10, 2, list2) } +func BenchmarkGlobWorkers30(b *testing.B) { benchmark(b, 10, 2, globWorkersD2) } -func BenchmarkGlob250(b *testing.B) { benchmarkGlob(b, 50, 4) } -func BenchmarkWalk250(b *testing.B) { benchmarkWalk(b, 50, 4) } -func BenchmarkList250(b *testing.B) { benchmarkList(b, 50, 4) } +func BenchmarkGlob80(b *testing.B) { benchmark(b, 20, 3, glob) } +func BenchmarkWalk80(b *testing.B) { benchmark(b, 20, 3, walk) } +func BenchmarkList80(b *testing.B) { benchmark(b, 20, 3, list2) } +func BenchmarkGlobWorkers80(b *testing.B) { benchmark(b, 20, 3, globWorkersD2) } -func BenchmarkGlob600(b *testing.B) { benchmarkGlob(b, 100, 5) } -func BenchmarkWalk600(b *testing.B) { benchmarkWalk(b, 100, 5) } -func BenchmarkList600(b *testing.B) { benchmarkList(b, 100, 5) } +func BenchmarkGlob250(b *testing.B) { benchmark(b, 50, 4, glob) } +func BenchmarkWalk250(b *testing.B) { benchmark(b, 50, 4, walk) } +func BenchmarkList250(b *testing.B) { benchmark(b, 50, 4, list2) } +func BenchmarkGlobWorkers250(b *testing.B) { benchmark(b, 50, 4, globWorkersD2) } -func BenchmarkGlob11000(b *testing.B) { benchmarkGlob(b, 1000, 10) } -func BenchmarkWalk11000(b *testing.B) { benchmarkWalk(b, 1000, 10) } -func BenchmarkList11000(b *testing.B) { benchmarkList(b, 1000, 10) } +func BenchmarkGlobAT600(b *testing.B) { benchmark(b, 100, 5, glob) } +func BenchmarkWalkAT600(b *testing.B) { benchmark(b, 100, 5, walk) } +func BenchmarkList2AT600(b *testing.B) { benchmark(b, 100, 5, list2) } +func BenchmarkList10AT600(b *testing.B) { benchmark(b, 100, 5, list10) } +func BenchmarkList20AT600(b *testing.B) { benchmark(b, 100, 5, list20) } +func BenchmarkGlobWorkersD1AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD1) } +func BenchmarkGlobWorkersD2AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD2) } +func BenchmarkGlobWorkersD4AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD4) } -func BenchmarkGlob110000(b *testing.B) { benchmarkGlob(b, 10000, 10) } -func BenchmarkWalk110000(b *testing.B) { benchmarkWalk(b, 10000, 10) } -func BenchmarkList110000(b *testing.B) { benchmarkList(b, 10000, 10) } +func BenchmarkGlobAT22000(b *testing.B) { benchmark(b, 2000, 10, glob) } +func BenchmarkWalkAT22000(b *testing.B) { benchmark(b, 2000, 10, walk) } +func BenchmarkList2AT22000(b *testing.B) { benchmark(b, 2000, 10, list2) } +func BenchmarkList10AT22000(b *testing.B) { benchmark(b, 2000, 10, list10) } +func BenchmarkList20AT22000(b *testing.B) { benchmark(b, 2000, 10, list20) } +func BenchmarkGlobWorkersD1AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD1) } +func BenchmarkGlobWorkersD2AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD2) } +func BenchmarkGlobWorkersD4AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD4) } -func benchmarkGlob(b *testing.B, numNodes int, numRevisions int) { - initialize(numNodes, numRevisions) - defer os.RemoveAll("test_temp") +func BenchmarkGlob110000(b *testing.B) { benchmark(b, 10000, 10, glob) } +func BenchmarkWalk110000(b *testing.B) { benchmark(b, 10000, 10, walk) } +func BenchmarkList110000(b *testing.B) { benchmark(b, 10000, 10, list2) } +func BenchmarkGlobWorkers110000(b *testing.B) { benchmark(b, 10000, 10, globWorkersD2) } +func benchmark(b *testing.B, numNodes int, numRevisions int, f func(string) <-chan string) { + base := initialize(numNodes, numRevisions) + defer os.RemoveAll(base) + + b.ResetTimer() for i := 0; i < b.N; i++ { - PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false) + ch := f(base) + PurgeRevisions(ch, nil, false, false) } + b.StopTimer() } -func benchmarkWalk(b *testing.B, numNodes int, numRevisions int) { - initialize(numNodes, numRevisions) - defer os.RemoveAll("test_temp") +func test(t *testing.T, numNodes int, numRevisions int, f func(string) <-chan string) { + base := initialize(numNodes, numRevisions) + defer os.RemoveAll(base) - for i := 0; i < b.N; i++ { - PurgeRevisionsWalk(_basePath, nil, false, false) - } -} - -func benchmarkList(b *testing.B, numNodes int, numRevisions int) { - initialize(numNodes, numRevisions) - defer os.RemoveAll("test_temp") - - for i := 0; i < b.N; i++ { - PurgeRevisionsList(_basePath, nil, false, false) - } -} - -func testGlob(t *testing.T, numNodes int, numRevisions int) { - initialize(numNodes, numRevisions) - defer os.RemoveAll("test_temp") - - _, _, revisions := PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false) + ch := f(base) + _, _, revisions := PurgeRevisions(ch, nil, false, false) require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") } -func testWalk(t *testing.T, numNodes int, numRevisions int) { - initialize(numNodes, numRevisions) - defer os.RemoveAll("test_temp") - - _, _, revisions := PurgeRevisionsWalk(_basePath, nil, false, false) - require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +func glob(base string) <-chan string { + return Glob(base + _basePath + "*/*/*/*/*") } -func testList(t *testing.T, numNodes int, numRevisions int) { - initialize(numNodes, numRevisions) - defer os.RemoveAll("test_temp") - - _, _, revisions := PurgeRevisionsList(_basePath, nil, false, false) - require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +func walk(base string) <-chan string { + return Walk(base + _basePath) } -func initialize(numNodes int, numRevisions int) { - // create base path - if err := os.MkdirAll(_basePath, fs.ModePerm); err != nil { +func list2(base string) <-chan string { + return List(base+_basePath, 2) +} + +func list10(base string) <-chan string { + return List(base+_basePath, 10) +} + +func list20(base string) <-chan string { + return List(base+_basePath, 20) +} + +func globWorkersD1(base string) <-chan string { + return GlobWorkers(base+_basePath, "*", "/*/*/*/*") +} + +func globWorkersD2(base string) <-chan string { + return GlobWorkers(base+_basePath, "*/*", "/*/*/*") +} + +func globWorkersD4(base string) <-chan string { + return GlobWorkers(base+_basePath, "*/*/*/*", "/*") +} + +func initialize(numNodes int, numRevisions int) string { + base := "test_temp_" + uuid.New().String() + if err := os.Mkdir(base, os.ModePerm); err != nil { fmt.Println("Error creating test_temp directory", err) + os.RemoveAll(base) + os.Exit(1) + } + + // create base path + if err := os.MkdirAll(base+_basePath, fs.ModePerm); err != nil { + fmt.Println("Error creating base path", err) + os.RemoveAll(base) os.Exit(1) } for i := 0; i < numNodes; i++ { path := lookup.Pathify(uuid.New().String(), 4, 2) dir := filepath.Dir(path) - if err := os.MkdirAll(_basePath+dir, fs.ModePerm); err != nil { + if err := os.MkdirAll(base+_basePath+dir, fs.ModePerm); err != nil { fmt.Println("Error creating test_temp directory", err) + os.RemoveAll(base) os.Exit(1) } - if _, err := os.Create(_basePath + path); err != nil { + if _, err := os.Create(base + _basePath + path); err != nil { fmt.Println("Error creating file", err) + os.RemoveAll(base) os.Exit(1) } for i := 0; i < numRevisions; i++ { - os.Create(_basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z") + os.Create(base + _basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z") } } + return base } From 54c304d3fcef1251a5e51a072c1fcbcb2fa9d88b Mon Sep 17 00:00:00 2001 From: mmattel Date: Thu, 22 Aug 2024 19:19:55 +0200 Subject: [PATCH 22/71] [docs-only] Improve dev docs --- docs/ocis/development/beta-testplan.md | 2 +- docs/ocis/development/build-docs.md | 2 +- docs/ocis/development/build.md | 2 +- .../development/continuous-integration.md | 2 +- docs/ocis/development/debugging.md | 2 +- .../ocis/development/deprecating-variables.md | 36 ------------- docs/ocis/development/envvars.md | 13 +++++ docs/ocis/development/profiling.md | 2 +- docs/ocis/development/testing.md | 2 +- docs/ocis/development/unit-testing/_index.md | 2 +- .../development/unit-testing/testing-pkg.md | 2 +- .../general-info/deprecating-variables.md | 54 +++++++++++++++++++ docs/services/general-info/envvar-scopes.md | 34 +++++++++--- .../general-info/new-service-checklist.md | 3 +- services/activitylog/README.md | 2 +- 15 files changed, 106 insertions(+), 54 deletions(-) delete mode 100644 docs/ocis/development/deprecating-variables.md create mode 100644 docs/ocis/development/envvars.md create mode 100644 docs/services/general-info/deprecating-variables.md diff --git a/docs/ocis/development/beta-testplan.md b/docs/ocis/development/beta-testplan.md index 51fa37368d..b66e908404 100644 --- a/docs/ocis/development/beta-testplan.md +++ b/docs/ocis/development/beta-testplan.md @@ -1,7 +1,7 @@ --- title: "Beta testplan" date: 2022-03-24T00:00:00+00:00 -weight: 37 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: beta-testplan.md diff --git a/docs/ocis/development/build-docs.md b/docs/ocis/development/build-docs.md index dc1196f66c..44624b234c 100644 --- a/docs/ocis/development/build-docs.md +++ b/docs/ocis/development/build-docs.md @@ -1,7 +1,7 @@ --- title: "Documentation" date: 2020-07-27T08:39:38+00:00 -weight: 99 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: build-docs.md diff --git a/docs/ocis/development/build.md b/docs/ocis/development/build.md index d7c1914581..e78409335f 100644 --- a/docs/ocis/development/build.md +++ b/docs/ocis/development/build.md @@ -1,7 +1,7 @@ --- title: "Build" date: 2020-02-27T20:35:00+01:00 -weight: 30 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: build.md diff --git a/docs/ocis/development/continuous-integration.md b/docs/ocis/development/continuous-integration.md index 3c9774f656..b5907d7cdc 100644 --- a/docs/ocis/development/continuous-integration.md +++ b/docs/ocis/development/continuous-integration.md @@ -1,7 +1,7 @@ --- title: "Continuous Integration" date: 2020-10-01T20:35:00+01:00 -weight: 100 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: continuous-integration.md diff --git a/docs/ocis/development/debugging.md b/docs/ocis/development/debugging.md index a9721fb9ff..5d0df756d4 100644 --- a/docs/ocis/development/debugging.md +++ b/docs/ocis/development/debugging.md @@ -1,7 +1,7 @@ --- title: "Debugging" date: 2020-03-19T08:21:00+01:00 -weight: 50 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: debugging.md diff --git a/docs/ocis/development/deprecating-variables.md b/docs/ocis/development/deprecating-variables.md deleted file mode 100644 index e69aa742fc..0000000000 --- a/docs/ocis/development/deprecating-variables.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "Deprecating Variables" -date: 2022-11-29T15:41:00+01:00 -geekdocRepo: https://github.com/owncloud/ocis -geekdocEditPath: edit/master/docs/ocis/development -geekdocFilePath: deprecating-variables.md ---- - -{{< toc >}} - -## Deprecating Environment Variables - -Sometimes it is necessary to deprecate environment to align their naming with conventions. We therefore added annotations to automate the documentation process. It is necessary to know when the variable is going to be deprecated, when it is going to be removed and why. - -### Example - -```golang -// Notifications defines the config options for the notifications service. -type Notifications struct { -RevaGateway string `yaml:"reva_gateway" env:"OCIS_REVA_GATEWAY;REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata" deprecationVersion:"3.0" removalVersion:"4.0.0" deprecationInfo:"REVA_GATEWAY changing name for consistency" deprecationReplacement:"OCIS_REVA_GATEWAY"` -... -} -``` - -There are four different annotation variables that need to be filled: - -| Annotation |Description| Format| -|---|---|---| -| deprecationVersion | The version the variable will be deprecated | semver (e.g. 3.0)| -| removalVersion| The version the variable will be removed from the codebase. Note that according to semver, a removal **MUST NOT** be made in a minor or patch version change, but only in a major release | semver (e.g. 4.0.0) | -| deprecationInfo | Information why the variable is deprecated, must start with the name of the variable in order to avoid confusion, when there are multiple options in the `env:`-field | string (e.g. NATS_NATS_HOST is confusing) | -| deprecationReplacement | The name of the variable that is going to replace the deprecated one.| string (e.g. NATS_HOST_ADDRESS) | - -### What happens next? - -Once a variable has been finally removed, the annotations must be removed again from the code, since they do not serve any purpose from this point. diff --git a/docs/ocis/development/envvars.md b/docs/ocis/development/envvars.md new file mode 100644 index 0000000000..4e7e317438 --- /dev/null +++ b/docs/ocis/development/envvars.md @@ -0,0 +1,13 @@ +--- +title: "Environment Variables" +date: 2024-08-22T15:41:00+01:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/ocis/development +geekdocFilePath: envvars.md +--- + +Environment variables are an essential part configuring services. + +If you are going to create new ones or deprecate existing ones, mandatory read the [Envvar Naming Scope]({{< ref "services/general-info/envvar-scopes.md" >}}) and the +[Deprecating Variables]({{< ref "services/general-info/deprecating-variables.md" >}}) documentation for more details first before doing so. diff --git a/docs/ocis/development/profiling.md b/docs/ocis/development/profiling.md index 9afb747cda..47a19431c9 100644 --- a/docs/ocis/development/profiling.md +++ b/docs/ocis/development/profiling.md @@ -1,7 +1,7 @@ --- title: "Profiling" date: 2021-08-24T12:32:20+01:00 -weight: 56 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: profiling.md diff --git a/docs/ocis/development/testing.md b/docs/ocis/development/testing.md index fe14492bd8..1dac8f3be2 100644 --- a/docs/ocis/development/testing.md +++ b/docs/ocis/development/testing.md @@ -1,7 +1,7 @@ --- title: "Acceptance Testing" date: 2018-05-02T00:00:00+00:00 -weight: 38 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: testing.md diff --git a/docs/ocis/development/unit-testing/_index.md b/docs/ocis/development/unit-testing/_index.md index 952819d3e4..48d7215b50 100644 --- a/docs/ocis/development/unit-testing/_index.md +++ b/docs/ocis/development/unit-testing/_index.md @@ -1,7 +1,7 @@ --- title: "Unit Testing" date: 2024-04-25T00:00:00+00:00 -weight: 5 +weight: 20 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development/unit-testing geekdocFilePath: _index.md diff --git a/docs/ocis/development/unit-testing/testing-pkg.md b/docs/ocis/development/unit-testing/testing-pkg.md index 8c5161005c..81ce85905e 100644 --- a/docs/ocis/development/unit-testing/testing-pkg.md +++ b/docs/ocis/development/unit-testing/testing-pkg.md @@ -1,7 +1,7 @@ --- title: "Standard Library Testing" date: 2024-04-25T00:00:00+00:00 -weight: 15 +weight: 10 geekdocRepo: https://github.com/owncloud/ocis geekdocEditPath: edit/master/docs/ocis/development/unit-testing geekdocFilePath: testing-pkg.md diff --git a/docs/services/general-info/deprecating-variables.md b/docs/services/general-info/deprecating-variables.md new file mode 100644 index 0000000000..eff2f312f4 --- /dev/null +++ b/docs/services/general-info/deprecating-variables.md @@ -0,0 +1,54 @@ +--- +title: "Envvar Deprecation" +date: 2024-08-22T15:41:00+01:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/services/general-info +geekdocFilePath: deprecating-variables.md +--- + +{{< toc >}} + +## Deprecating Environment Variables + +Sometimes it is necessary to deprecate an environment variable to align the naming with conventions or remove it at all. We therefore added annotations to automate the *documentation* process. + +The relevant annotations in the envvar struct tag are: + +* `deprecationVersion`\ + The release an envvar is announced for deprecation. +* `removalVersion`\ + The version it is finally going to be removed is defined via the mandatory placeholder `%%NEXT_PRODUCTION_VERSION%%`, not an actual version number. +* `deprecationInfo`\ + The reason why it got deprecated. +* `deprecationReplacement`\ + Only if it is going to be replaced, not necessary if removed. + +{{< hint warning >}} +During the development cycle, the value for the `removalVersion` must be set to `%%NEXT_PRODUCTION_VERSION%%`. This placeholder will be removed by the real version number during the production releasing process. +{{< /hint >}} + +For the documentation only to show the correct value for the `removalVersion`, our docs helper scripts will automatically generate the correct version to be printed in the documentation. If `%%NEXT_PRODUCTION_VERSION%%` is found in the query, it will be replaced with `next-prod`, else the value found is used. + +### Example + +```golang +// Notifications defines the config options for the notifications service. +type Notifications struct { +RevaGateway string `yaml:"reva_gateway" env:"OCIS_REVA_GATEWAY;REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata" deprecationVersion:"3.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"REVA_GATEWAY changing name for consistency" deprecationReplacement:"OCIS_REVA_GATEWAY"` +... +} +``` + +There are four different annotation variables that need to be filled: + +| Annotation |Description| Format| +|---|---|---| +| deprecationVersion | The version the variable will be deprecated | semver (e.g. 3.0)| +| removalVersion| The version the variable will be removed from the codebase. Note that according to semver, a removal **MUST NOT** be made in a minor or patch version change, but only in a major release | `%%NEXT_PRODUCTION_VERSION%%` | +| deprecationInfo | Information why the variable is deprecated, must start with the name of the variable in order to avoid confusion, when there are multiple options in the `env:`-field | string (e.g. NATS_NATS_HOST is confusing) | +| deprecationReplacement | The name of the variable that is going to replace the deprecated one.| string (e.g. NATS_HOST_ADDRESS) | + +### What Happens Next? + +Once a variable has been finally been removed, the annotations must be removed again from the code, since they don't serve any purpose. diff --git a/docs/services/general-info/envvar-scopes.md b/docs/services/general-info/envvar-scopes.md index ffe7f1acb0..3c9068867e 100644 --- a/docs/services/general-info/envvar-scopes.md +++ b/docs/services/general-info/envvar-scopes.md @@ -1,5 +1,5 @@ --- -title: Envvar Naming Scope +title: Envvar Naming Scopes date: 2023-03-23T00:00:00+00:00 weight: 20 geekdocRepo: https://github.com/owncloud/ocis @@ -8,6 +8,8 @@ geekdocFilePath: envvar-scopes.md geekdocCollapseSection: true --- +{{< toc >}} + The scope of an environment variable can be derived from its name. Therefore, it is important to follow the correct naming scheme to enable easy and proper identification. This is important when either: - a new local envvar is introduced. @@ -20,7 +22,7 @@ The scope of an environment variable can be derived from its name. Therefore, it - Mandatory when used in a service, a global envvar must have a local counterpart. - Variables that do not belong to any service are by definition global. -## Name Scope +## Naming Scope ### Local Envvars @@ -34,14 +36,32 @@ Note that this envvar is the global representation of the local example from abo To get a list of global envvars used in all services, see the [Global Environment Variables](https://doc.owncloud.com/ocis/next/deployment/services/env-vars-special-scope.html#global-environment-variables) table in the ocis admin documentation. -### Lifecycle +## Lifecycle of Envvars -In the struct tag values of our config data types, we are using three key/value pairs to document the lifecycle of a config variable: `introductionVersion`, `deprecationVersion` and `removalVersion`. During the development cycle, a new value should set to `%%NEXT%%` as long as no release is scheduled. During the release process, the placeholder will be replaced with the actual version number. Our docs helper scripts will then automatically generate the correct documentation based on the version number. +The envvar struct tag contains at maximum the following key/value pairs to document the lifecycle of a config variable: -## Deprecations +* `introductionVersion` +* `deprecationVersion` +* `removalVersion` +* `deprecationInfo` +* `deprecationReplacement` -All environment variable types that are used in a service follow the same [deprecation rules]({{< ref "ocis/development/deprecating-variables/_index.md" >}}) independent of their scope. +### Introduce new Envvars -## Separating Envvars +If a new envvar is introduced, only the `introductionVersion` is required. + +{{< hint warning >}} +During the development cycle, the value for the `introductionVersion` must be set to `%%NEXT%%`. This placeholder will be removed by the real version number during the production releasing process. +{{< /hint >}} + +For the documentation only to show the correct value for the `IV` (introduction version), our docs helper scripts will automatically generate the correct version to be printed in the documentation. If `%%NEXT%%` is found in the query, it will be replaced with `next`, else the value found is used. + +During the releasing process for a production release, the placeholder `%%NEXT%%` has to be replaced with the new production version number like `%%NEXT%%` → `7.0.0`. + +### Deprecate Existing Envvars + +See the [deprecation rules]({{< ref "./deprecating-variables.md" >}}) documentation fo rmore details. + +## Separating Multiple Envvars When multiple envvars are defined for one purpose like a global and local one, use `;` (semicolon) to properly separate the envvars in go code. Though it is possible to separate with `,` (comma) according go rules, the current implementation of the docs generation process only recognizes semicolons as separator. diff --git a/docs/services/general-info/new-service-checklist.md b/docs/services/general-info/new-service-checklist.md index 613985cfae..2640ca8c14 100644 --- a/docs/services/general-info/new-service-checklist.md +++ b/docs/services/general-info/new-service-checklist.md @@ -32,7 +32,7 @@ Use this checklist with copy/paste in your PR - right from the beginning. It ren - Add the service config to `ocis-pkg/config/defaultconfig.go` - [ ] If the service is using service accounts, add it to `ocis/pkg/init/init.go` - [ ] Add the service to `.drone.star` to enable CI. -- [ ] Inform doc team in an _early stage_ to review the readme AND the environment variables created. +- [ ] Inform doc team in an *early stage* to review the readme AND the environment variables created. - The description must reflect the behaviour AND usually has a positive code quality impact. - [ ] Create proper description strings for envvars - see other services for examples, especially when it comes to multiple values. This must include: - base description, set of available values, description of each value. @@ -44,4 +44,5 @@ Use this checklist with copy/paste in your PR - right from the beginning. It ren - [ ] Make sure to have a function `FullDefaultConfig()` in `pkg/config/defaults/defaultconfig.go` of your service. It is needed to create the documentation. - [ ] Add metrics to the code to enable monitoring. See the proxy service for implementation details. - Plus add documentation about monitoring in the README.md file +- [ ] When the service requires translations that have to be covered by the service and are not sourced by web, see the [add translation]({{< ref "./add-translations.md" >}}) documentation for more details. ``` diff --git a/services/activitylog/README.md b/services/activitylog/README.md index b7238194a8..8918b3e995 100644 --- a/services/activitylog/README.md +++ b/services/activitylog/README.md @@ -1,4 +1,4 @@ -# Activitylog Service +# Activitylog The `activitylog` service is responsible for storing events (activities) per resource. From e679c01dfcdd70d392ed5f91c57677400f2e5447 Mon Sep 17 00:00:00 2001 From: Martin Date: Fri, 23 Aug 2024 08:44:46 +0200 Subject: [PATCH 23/71] Apply suggestions from code review Co-authored-by: Phil Davis --- docs/ocis/development/envvars.md | 4 ++-- docs/services/general-info/deprecating-variables.md | 8 ++++---- docs/services/general-info/envvar-scopes.md | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/ocis/development/envvars.md b/docs/ocis/development/envvars.md index 4e7e317438..c401b370d8 100644 --- a/docs/ocis/development/envvars.md +++ b/docs/ocis/development/envvars.md @@ -7,7 +7,7 @@ geekdocEditPath: edit/master/docs/ocis/development geekdocFilePath: envvars.md --- -Environment variables are an essential part configuring services. +Environment variables are an essential part of configuring services. -If you are going to create new ones or deprecate existing ones, mandatory read the [Envvar Naming Scope]({{< ref "services/general-info/envvar-scopes.md" >}}) and the +If you are going to create new ones or deprecate existing ones, you must read the [Envvar Naming Scope]({{< ref "services/general-info/envvar-scopes.md" >}}) and the [Deprecating Variables]({{< ref "services/general-info/deprecating-variables.md" >}}) documentation for more details first before doing so. diff --git a/docs/services/general-info/deprecating-variables.md b/docs/services/general-info/deprecating-variables.md index eff2f312f4..bc77eebe9f 100644 --- a/docs/services/general-info/deprecating-variables.md +++ b/docs/services/general-info/deprecating-variables.md @@ -11,7 +11,7 @@ geekdocFilePath: deprecating-variables.md ## Deprecating Environment Variables -Sometimes it is necessary to deprecate an environment variable to align the naming with conventions or remove it at all. We therefore added annotations to automate the *documentation* process. +Sometimes it is necessary to deprecate an environment variable to align the naming with conventions or remove it completely. We therefore added annotations to automate the *documentation* process. The relevant annotations in the envvar struct tag are: @@ -20,15 +20,15 @@ The relevant annotations in the envvar struct tag are: * `removalVersion`\ The version it is finally going to be removed is defined via the mandatory placeholder `%%NEXT_PRODUCTION_VERSION%%`, not an actual version number. * `deprecationInfo`\ - The reason why it got deprecated. + The reason why it was deprecated. * `deprecationReplacement`\ Only if it is going to be replaced, not necessary if removed. {{< hint warning >}} -During the development cycle, the value for the `removalVersion` must be set to `%%NEXT_PRODUCTION_VERSION%%`. This placeholder will be removed by the real version number during the production releasing process. +During the development cycle, the value for the `removalVersion` must be set to `%%NEXT_PRODUCTION_VERSION%%`. This placeholder will be replaced by the real version number during the production releasing process. {{< /hint >}} -For the documentation only to show the correct value for the `removalVersion`, our docs helper scripts will automatically generate the correct version to be printed in the documentation. If `%%NEXT_PRODUCTION_VERSION%%` is found in the query, it will be replaced with `next-prod`, else the value found is used. +For the documentation to show the correct value for the `removalVersion`, our docs helper scripts will automatically generate the correct version to be printed in the documentation. If `%%NEXT_PRODUCTION_VERSION%%` is found in the query, it will be replaced with `next-prod`, else the value found is used. ### Example diff --git a/docs/services/general-info/envvar-scopes.md b/docs/services/general-info/envvar-scopes.md index 3c9068867e..f043cf36d9 100644 --- a/docs/services/general-info/envvar-scopes.md +++ b/docs/services/general-info/envvar-scopes.md @@ -54,13 +54,13 @@ If a new envvar is introduced, only the `introductionVersion` is required. During the development cycle, the value for the `introductionVersion` must be set to `%%NEXT%%`. This placeholder will be removed by the real version number during the production releasing process. {{< /hint >}} -For the documentation only to show the correct value for the `IV` (introduction version), our docs helper scripts will automatically generate the correct version to be printed in the documentation. If `%%NEXT%%` is found in the query, it will be replaced with `next`, else the value found is used. +For the documentation to show the correct value for the `IV` (introduction version), our docs helper scripts will automatically generate the correct version to be printed in the documentation. If `%%NEXT%%` is found in the query, it will be replaced with `next`, else the value found is used. During the releasing process for a production release, the placeholder `%%NEXT%%` has to be replaced with the new production version number like `%%NEXT%%` → `7.0.0`. ### Deprecate Existing Envvars -See the [deprecation rules]({{< ref "./deprecating-variables.md" >}}) documentation fo rmore details. +See the [deprecation rules]({{< ref "./deprecating-variables.md" >}}) documentation for more details. ## Separating Multiple Envvars From 3a6030796829d375d2f4536d9774a04d7b8b4298 Mon Sep 17 00:00:00 2001 From: mmattel Date: Fri, 23 Aug 2024 09:05:10 +0200 Subject: [PATCH 24/71] [docs-only][chore] Update envvars yaml --- docs/helpers/env_vars.yaml | 260 +++++++++++++++++++------------------ 1 file changed, 137 insertions(+), 123 deletions(-) diff --git a/docs/helpers/env_vars.yaml b/docs/helpers/env_vars.yaml index 5ef64a6d3f..8c3f1f49de 100644 --- a/docs/helpers/env_vars.yaml +++ b/docs/helpers/env_vars.yaml @@ -274,6 +274,17 @@ ACTIVITYLOG_TRACING_TYPE: deprecationVersion: "" removalVersion: "" deprecationInfo: "" +ACTIVITYLOG_TRANSLATION_PATH: + name: OCIS_TRANSLATION_PATH;ACTIVITYLOG_TRANSLATION_PATH + defaultValue: "" + type: string + description: (optional) Set this to a path with custom translations to overwrite + the builtin translations. Note that file and folder naming rules apply, see the + documentation for more details. + introductionVersion: '%%NEXT%%' + deprecationVersion: "" + removalVersion: "" + deprecationInfo: "" ACTIVITYOG_SERVICE_ACCOUNT_SECRET: name: OCIS_SERVICE_ACCOUNT_SECRET;ACTIVITYOG_SERVICE_ACCOUNT_SECRET defaultValue: "" @@ -7711,28 +7722,28 @@ OCIS_ASYNC_UPLOADS: removalVersion: "" deprecationInfo: "" OCIS_CACHE_AUTH_PASSWORD: - name: OCIS_CACHE_AUTH_PASSWORD;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD + name: OCIS_CACHE_AUTH_PASSWORD;FRONTEND_OCS_STAT_CACHE_AUTH_PASSWORD defaultValue: "" type: string - description: The password to authenticate with the store. Only applies when store - type 'nats-js-kv' is configured. + description: The password to use for authentication. Only applies when using the + 'nats-js-kv' store type. introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_CACHE_AUTH_USERNAME: - name: OCIS_CACHE_AUTH_USERNAME;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME + name: OCIS_CACHE_AUTH_USERNAME;FRONTEND_OCS_STAT_CACHE_AUTH_USERNAME defaultValue: "" type: string - description: The username to authenticate with the store. Only applies when store - type 'nats-js-kv' is configured. + description: The username to use for authentication. Only applies when using the + 'nats-js-kv' store type. introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_CACHE_DATABASE: name: OCIS_CACHE_DATABASE - defaultValue: cache-userinfo + defaultValue: cache-stat type: string description: The database name the configured store should use. introductionVersion: pre5.0 @@ -7740,61 +7751,60 @@ OCIS_CACHE_DATABASE: removalVersion: "" deprecationInfo: "" OCIS_CACHE_DISABLE_PERSISTENCE: - name: OCIS_CACHE_DISABLE_PERSISTENCE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_DISABLE_PERSISTENCE - defaultValue: "true" + name: OCIS_CACHE_DISABLE_PERSISTENCE;FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE + defaultValue: "false" type: bool - description: Disables persistence of the store. Only applies when store type 'nats-js-kv' - is configured. Defaults to true. + description: Disable persistence of the cache. Only applies when using the 'nats-js-kv' + store type. Defaults to false. introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_CACHE_SIZE: - name: OCIS_CACHE_SIZE;PROXY_OIDC_USERINFO_CACHE_SIZE + name: OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE defaultValue: "0" type: int - description: The maximum quantity of items in the user info cache. Only applies - when store type 'ocmem' is configured. Defaults to 512 which is derived from the - ocmem package though not explicitly set as default. + description: Max number of entries to hold in the cache. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_CACHE_STORE: - name: OCIS_CACHE_STORE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE - defaultValue: nats-js-kv + name: OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE + defaultValue: memory type: string - description: 'The type of the signing key store. Supported values are: ''redis-sentinel'', - ''nats-js-kv'' and ''ocisstoreservice'' (deprecated). See the text description - for details.' - introductionVersion: "5.0" + description: 'The type of the cache store. Supported values are: ''memory'', ''redis-sentinel'', + ''nats-js-kv'', ''noop''. See the text description for details.' + introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_CACHE_STORE_NODES: - name: OCIS_CACHE_STORE_NODES;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES + name: OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES defaultValue: '[127.0.0.1:9233]' type: '[]string' - description: A list of nodes to access the configured store. Note that the behaviour - how nodes are used is dependent on the library of the configured store. See the - Environment Variable Types description for more details. - introductionVersion: "5.0" + description: A list of nodes to access the configured store. This has no effect + when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes + are used is dependent on the library of the configured store. See the Environment + Variable Types description for more details. + introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_CACHE_TTL: - name: OCIS_CACHE_TTL;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL - defaultValue: 12h0m0s + name: OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL + defaultValue: 5m0s type: Duration - description: Default time to live for signing keys. See the Environment Variable - Types description for more details. - introductionVersion: "5.0" + description: Default time to live for user info in the cache. Only applied when + access tokens has no expiration. See the Environment Variable Types description + for more details. + introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_CORS_ALLOW_CREDENTIALS: - name: OCIS_CORS_ALLOW_CREDENTIALS;USERLOG_CORS_ALLOW_CREDENTIALS - defaultValue: "true" + name: OCIS_CORS_ALLOW_CREDENTIALS;FRONTEND_CORS_ALLOW_CREDENTIALS + defaultValue: "false" type: bool description: 'Allow credentials for CORS.See following chapter for more details: *Access-Control-Allow-Credentials* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials.' @@ -7803,9 +7813,11 @@ OCIS_CORS_ALLOW_CREDENTIALS: removalVersion: "" deprecationInfo: "" OCIS_CORS_ALLOW_HEADERS: - name: OCIS_CORS_ALLOW_HEADERS;USERLOG_CORS_ALLOW_HEADERS - defaultValue: '[Authorization Origin Content-Type Accept X-Requested-With X-Request-Id - Ocs-Apirequest]' + name: OCIS_CORS_ALLOW_HEADERS;FRONTEND_CORS_ALLOW_HEADERS + defaultValue: '[Origin Accept Content-Type Depth Authorization Ocs-Apirequest If-None-Match + If-Match Destination Overwrite X-Request-Id X-Requested-With Tus-Resumable Tus-Checksum-Algorithm + Upload-Concat Upload-Length Upload-Metadata Upload-Defer-Length Upload-Expires + Upload-Checksum Upload-Offset X-HTTP-Method-Override Cache-Control]' type: '[]string' description: 'A list of allowed CORS headers. See following chapter for more details: *Access-Control-Request-Headers* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Headers. @@ -7815,8 +7827,9 @@ OCIS_CORS_ALLOW_HEADERS: removalVersion: "" deprecationInfo: "" OCIS_CORS_ALLOW_METHODS: - name: OCIS_CORS_ALLOW_METHODS;USERLOG_CORS_ALLOW_METHODS - defaultValue: '[GET]' + name: OCIS_CORS_ALLOW_METHODS;FRONTEND_CORS_ALLOW_METHODS + defaultValue: '[OPTIONS HEAD GET PUT POST PATCH DELETE MKCOL PROPFIND PROPPATCH + MOVE COPY REPORT SEARCH]' type: '[]string' description: 'A list of allowed CORS methods. See following chapter for more details: *Access-Control-Request-Method* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Method. @@ -7826,8 +7839,8 @@ OCIS_CORS_ALLOW_METHODS: removalVersion: "" deprecationInfo: "" OCIS_CORS_ALLOW_ORIGINS: - name: OCIS_CORS_ALLOW_ORIGINS;USERLOG_CORS_ALLOW_ORIGINS - defaultValue: '[*]' + name: OCIS_CORS_ALLOW_ORIGINS;FRONTEND_CORS_ALLOW_ORIGINS + defaultValue: '[https://localhost:9200]' type: '[]string' description: 'A list of allowed CORS origins. See following chapter for more details: *Access-Control-Allow-Origin* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. @@ -7933,7 +7946,7 @@ OCIS_DISABLE_VERSIONING: removalVersion: "" deprecationInfo: "" OCIS_EDITION: - name: OCIS_EDITION;OCDAV_EDITION + name: OCIS_EDITION;FRONTEND_EDITION defaultValue: Community type: string description: Edition of oCIS. Used for branding purposes. @@ -7951,10 +7964,10 @@ OCIS_EMAIL_TEMPLATE_PATH: removalVersion: "" deprecationInfo: "" OCIS_ENABLE_OCM: - name: OCIS_ENABLE_OCM;GRAPH_INCLUDE_OCM_SHAREES + name: OCIS_ENABLE_OCM;FRONTEND_OCS_INCLUDE_OCM_SHAREES defaultValue: "false" type: bool - description: Include OCM sharees when listing users. + description: Include OCM sharees when listing sharees. introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" @@ -7970,7 +7983,7 @@ OCIS_ENABLE_RESHARING: removalVersion: "" deprecationInfo: Resharing will be removed in the future. OCIS_EVENTS_AUTH_PASSWORD: - name: OCIS_EVENTS_AUTH_PASSWORD;USERLOG_EVENTS_AUTH_PASSWORD + name: OCIS_EVENTS_AUTH_PASSWORD;FRONTEND_EVENTS_AUTH_PASSWORD defaultValue: "" type: string description: The password to authenticate with the events broker. The events broker @@ -7980,7 +7993,7 @@ OCIS_EVENTS_AUTH_PASSWORD: removalVersion: "" deprecationInfo: "" OCIS_EVENTS_AUTH_USERNAME: - name: OCIS_EVENTS_AUTH_USERNAME;USERLOG_EVENTS_AUTH_USERNAME + name: OCIS_EVENTS_AUTH_USERNAME;FRONTEND_EVENTS_AUTH_USERNAME defaultValue: "" type: string description: The username to authenticate with the events broker. The events broker @@ -7990,18 +8003,18 @@ OCIS_EVENTS_AUTH_USERNAME: removalVersion: "" deprecationInfo: "" OCIS_EVENTS_CLUSTER: - name: OCIS_EVENTS_CLUSTER;USERLOG_EVENTS_CLUSTER + name: OCIS_EVENTS_CLUSTER;FRONTEND_EVENTS_CLUSTER defaultValue: ocis-cluster type: string description: The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system. - introductionVersion: pre5.0 + introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_EVENTS_ENABLE_TLS: - name: OCIS_EVENTS_ENABLE_TLS;USERLOG_EVENTS_ENABLE_TLS + name: OCIS_EVENTS_ENABLE_TLS;NATS_EVENTS_ENABLE_TLS defaultValue: "false" type: bool description: Enable TLS for the connection to the events broker. The events broker @@ -8011,31 +8024,31 @@ OCIS_EVENTS_ENABLE_TLS: removalVersion: "" deprecationInfo: "" OCIS_EVENTS_ENDPOINT: - name: OCIS_EVENTS_ENDPOINT;USERLOG_EVENTS_ENDPOINT + name: OCIS_EVENTS_ENDPOINT;FRONTEND_EVENTS_ENDPOINT defaultValue: 127.0.0.1:9233 type: string description: The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. - introductionVersion: pre5.0 + introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE: - name: OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE;USERLOG_EVENTS_TLS_ROOT_CA_CERTIFICATE + name: OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE;ANTIVIRUS_EVENTS_TLS_ROOT_CA_CERTIFICATE defaultValue: "" type: string description: The root CA certificate used to validate the server's TLS certificate. - If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false. + If provided ANTIVIRUS_EVENTS_TLS_INSECURE will be seen as false. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_GATEWAY_GRPC_ADDR: - name: OCIS_GATEWAY_GRPC_ADDR;GATEWAY_GRPC_ADDR + name: OCIS_GATEWAY_GRPC_ADDR;STORAGE_USERS_GATEWAY_GRPC_ADDR defaultValue: 127.0.0.1:9142 type: string - description: The bind address of the GRPC service. - introductionVersion: pre5.0 + description: The bind address of the gateway GRPC address. + introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" deprecationInfo: "" @@ -8094,16 +8107,17 @@ OCIS_HTTP_TLS_KEY: removalVersion: "" deprecationInfo: "" OCIS_INSECURE: - name: OCIS_INSECURE;USERLOG_EVENTS_TLS_INSECURE + name: OCIS_INSECURE;NATS_TLS_SKIP_VERIFY_CLIENT_CERT defaultValue: "false" type: bool - description: Whether to verify the server TLS certificates. + description: Whether the NATS server should skip the client certificate verification + during the TLS handshake. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_JWT_SECRET: - name: OCIS_JWT_SECRET;USERLOG_JWT_SECRET + name: OCIS_JWT_SECRET;GROUPS_JWT_SECRET defaultValue: "" type: string description: The secret to mint and validate jwt tokens. @@ -8112,7 +8126,7 @@ OCIS_JWT_SECRET: removalVersion: "" deprecationInfo: "" OCIS_KEYCLOAK_BASE_PATH: - name: OCIS_KEYCLOAK_BASE_PATH;INVITATIONS_KEYCLOAK_BASE_PATH + name: OCIS_KEYCLOAK_BASE_PATH;GRAPH_KEYCLOAK_BASE_PATH defaultValue: "" type: string description: The URL to access keycloak. @@ -8121,16 +8135,16 @@ OCIS_KEYCLOAK_BASE_PATH: removalVersion: "" deprecationInfo: "" OCIS_KEYCLOAK_CLIENT_ID: - name: OCIS_KEYCLOAK_CLIENT_ID;INVITATIONS_KEYCLOAK_CLIENT_ID + name: OCIS_KEYCLOAK_CLIENT_ID;GRAPH_KEYCLOAK_CLIENT_ID defaultValue: "" type: string - description: The client ID to authenticate with keycloak. + description: The client id to authenticate with keycloak. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_KEYCLOAK_CLIENT_REALM: - name: OCIS_KEYCLOAK_CLIENT_REALM;INVITATIONS_KEYCLOAK_CLIENT_REALM + name: OCIS_KEYCLOAK_CLIENT_REALM;GRAPH_KEYCLOAK_CLIENT_REALM defaultValue: "" type: string description: The realm the client is defined in. @@ -8139,7 +8153,7 @@ OCIS_KEYCLOAK_CLIENT_REALM: removalVersion: "" deprecationInfo: "" OCIS_KEYCLOAK_CLIENT_SECRET: - name: OCIS_KEYCLOAK_CLIENT_SECRET;INVITATIONS_KEYCLOAK_CLIENT_SECRET + name: OCIS_KEYCLOAK_CLIENT_SECRET;GRAPH_KEYCLOAK_CLIENT_SECRET defaultValue: "" type: string description: The client secret to use in authentication. @@ -8148,7 +8162,7 @@ OCIS_KEYCLOAK_CLIENT_SECRET: removalVersion: "" deprecationInfo: "" OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY: - name: OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY;INVITATIONS_KEYCLOAK_INSECURE_SKIP_VERIFY + name: OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY;GRAPH_KEYCLOAK_INSECURE_SKIP_VERIFY defaultValue: "false" type: bool description: Disable TLS certificate validation for Keycloak connections. Do not @@ -8158,7 +8172,7 @@ OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY: removalVersion: "" deprecationInfo: "" OCIS_KEYCLOAK_USER_REALM: - name: OCIS_KEYCLOAK_USER_REALM;INVITATIONS_KEYCLOAK_USER_REALM + name: OCIS_KEYCLOAK_USER_REALM;GRAPH_KEYCLOAK_USER_REALM defaultValue: "" type: string description: The realm users are defined. @@ -8197,20 +8211,20 @@ OCIS_LDAP_CACERT: removalVersion: "" deprecationInfo: "" OCIS_LDAP_DISABLE_USER_MECHANISM: - name: OCIS_LDAP_DISABLE_USER_MECHANISM;USERS_LDAP_DISABLE_USER_MECHANISM + name: OCIS_LDAP_DISABLE_USER_MECHANISM;GRAPH_DISABLE_USER_MECHANISM defaultValue: attribute type: string - description: An option to control the behavior for disabling users. Valid options + description: An option to control the behavior for disabling users. Supported options are 'none', 'attribute' and 'group'. If set to 'group', disabling a user via API will add the user to the configured group for disabled users, if set to 'attribute' this will be done in the ldap user entry, if set to 'none' the disable request - is not processed. + is not processed. Default is 'attribute'. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_LDAP_DISABLED_USERS_GROUP_DN: - name: OCIS_LDAP_DISABLED_USERS_GROUP_DN;USERS_LDAP_DISABLED_USERS_GROUP_DN + name: OCIS_LDAP_DISABLED_USERS_GROUP_DN;GRAPH_DISABLED_USERS_GROUP_DN defaultValue: cn=DisabledUsersGroup,ou=groups,o=libregraph-idm type: string description: The distinguished name of the group to which added users will be classified @@ -8326,7 +8340,7 @@ OCIS_LDAP_INSECURE: removalVersion: "" deprecationInfo: "" OCIS_LDAP_SERVER_WRITE_ENABLED: - name: OCIS_LDAP_SERVER_WRITE_ENABLED;GRAPH_LDAP_SERVER_WRITE_ENABLED + name: OCIS_LDAP_SERVER_WRITE_ENABLED;FRONTEND_LDAP_SERVER_WRITE_ENABLED defaultValue: "true" type: bool description: Allow creating, modifying and deleting LDAP users via the GRAPH API. @@ -8357,10 +8371,10 @@ OCIS_LDAP_USER_BASE_DN: removalVersion: "" deprecationInfo: "" OCIS_LDAP_USER_ENABLED_ATTRIBUTE: - name: OCIS_LDAP_USER_ENABLED_ATTRIBUTE;USERS_LDAP_USER_ENABLED_ATTRIBUTE + name: OCIS_LDAP_USER_ENABLED_ATTRIBUTE;GRAPH_USER_ENABLED_ATTRIBUTE defaultValue: ownCloudUserEnabled type: string - description: LDAP attribute to use as a flag telling if the user is enabled or disabled. + description: LDAP Attribute to use as a flag telling if the user is enabled or disabled. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" @@ -8424,7 +8438,7 @@ OCIS_LDAP_USER_SCHEMA_MAIL: removalVersion: "" deprecationInfo: "" OCIS_LDAP_USER_SCHEMA_USER_TYPE: - name: OCIS_LDAP_USER_SCHEMA_USER_TYPE;USERS_LDAP_USER_TYPE_ATTRIBUTE + name: OCIS_LDAP_USER_SCHEMA_USER_TYPE;GRAPH_LDAP_USER_TYPE_ATTRIBUTE defaultValue: ownCloudUserType type: string description: LDAP Attribute to distinguish between 'Member' and 'Guest' users. Default @@ -8453,7 +8467,7 @@ OCIS_LDAP_USER_SCOPE: removalVersion: "" deprecationInfo: "" OCIS_LOG_COLOR: - name: OCIS_LOG_COLOR;USERLOG_LOG_COLOR + name: OCIS_LOG_COLOR;NATS_LOG_COLOR defaultValue: "false" type: bool description: Activates colorized log output. @@ -8462,7 +8476,7 @@ OCIS_LOG_COLOR: removalVersion: "" deprecationInfo: "" OCIS_LOG_FILE: - name: OCIS_LOG_FILE;USERLOG_LOG_FILE + name: OCIS_LOG_FILE;NATS_LOG_FILE defaultValue: "" type: string description: The path to the log file. Activates logging to this file if set. @@ -8471,7 +8485,7 @@ OCIS_LOG_FILE: removalVersion: "" deprecationInfo: "" OCIS_LOG_LEVEL: - name: OCIS_LOG_LEVEL;USERLOG_LOG_LEVEL + name: OCIS_LOG_LEVEL;NATS_LOG_LEVEL defaultValue: "" type: string description: 'The log level. Valid values are: ''panic'', ''fatal'', ''error'', @@ -8481,7 +8495,7 @@ OCIS_LOG_LEVEL: removalVersion: "" deprecationInfo: "" OCIS_LOG_PRETTY: - name: OCIS_LOG_PRETTY;USERLOG_LOG_PRETTY + name: OCIS_LOG_PRETTY;NATS_LOG_PRETTY defaultValue: "false" type: bool description: Activates pretty log output. @@ -8490,11 +8504,11 @@ OCIS_LOG_PRETTY: removalVersion: "" deprecationInfo: "" OCIS_MACHINE_AUTH_API_KEY: - name: OCIS_MACHINE_AUTH_API_KEY;PROXY_MACHINE_AUTH_API_KEY + name: OCIS_MACHINE_AUTH_API_KEY;FRONTEND_MACHINE_AUTH_API_KEY defaultValue: "" type: string - description: Machine auth API key used to validate internal requests necessary to - access resources from other services. + description: The machine auth API key used to validate internal requests necessary + to access resources from other services. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" @@ -8511,16 +8525,17 @@ OCIS_OIDC_CLIENT_ID: removalVersion: "" deprecationInfo: "" OCIS_OIDC_ISSUER: - name: OCIS_URL;OCIS_OIDC_ISSUER;PROXY_OIDC_ISSUER + name: OCIS_URL;OCIS_OIDC_ISSUER;GROUPS_IDP_URL defaultValue: https://localhost:9200 type: string - description: URL of the OIDC issuer. It defaults to URL of the builtin IDP. + description: The identity provider value to set in the group IDs of the CS3 group + objects for groups returned by this group provider. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST: - name: OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST;SHARING_PASSWORD_POLICY_BANNED_PASSWORDS_LIST + name: OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST;FRONTEND_PASSWORD_POLICY_BANNED_PASSWORDS_LIST defaultValue: "" type: string description: Path to the 'banned passwords list' file. This only impacts public @@ -8530,7 +8545,7 @@ OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST: removalVersion: "" deprecationInfo: "" OCIS_PASSWORD_POLICY_DISABLED: - name: OCIS_PASSWORD_POLICY_DISABLED;SHARING_PASSWORD_POLICY_DISABLED + name: OCIS_PASSWORD_POLICY_DISABLED;FRONTEND_PASSWORD_POLICY_DISABLED defaultValue: "false" type: bool description: Disable the password policy. Defaults to false if not set. @@ -8539,7 +8554,7 @@ OCIS_PASSWORD_POLICY_DISABLED: removalVersion: "" deprecationInfo: "" OCIS_PASSWORD_POLICY_MIN_CHARACTERS: - name: OCIS_PASSWORD_POLICY_MIN_CHARACTERS;SHARING_PASSWORD_POLICY_MIN_CHARACTERS + name: OCIS_PASSWORD_POLICY_MIN_CHARACTERS;FRONTEND_PASSWORD_POLICY_MIN_CHARACTERS defaultValue: "8" type: int description: Define the minimum password length. Defaults to 8 if not set. @@ -8548,7 +8563,7 @@ OCIS_PASSWORD_POLICY_MIN_CHARACTERS: removalVersion: "" deprecationInfo: "" OCIS_PASSWORD_POLICY_MIN_DIGITS: - name: OCIS_PASSWORD_POLICY_MIN_DIGITS;SHARING_PASSWORD_POLICY_MIN_DIGITS + name: OCIS_PASSWORD_POLICY_MIN_DIGITS;FRONTEND_PASSWORD_POLICY_MIN_DIGITS defaultValue: "1" type: int description: Define the minimum number of digits. Defaults to 1 if not set. @@ -8557,7 +8572,7 @@ OCIS_PASSWORD_POLICY_MIN_DIGITS: removalVersion: "" deprecationInfo: "" OCIS_PASSWORD_POLICY_MIN_LOWERCASE_CHARACTERS: - name: OCIS_PASSWORD_POLICY_MIN_LOWERCASE_CHARACTERS;SHARING_PASSWORD_POLICY_MIN_LOWERCASE_CHARACTERS + name: OCIS_PASSWORD_POLICY_MIN_LOWERCASE_CHARACTERS;FRONTEND_PASSWORD_POLICY_MIN_LOWERCASE_CHARACTERS defaultValue: "1" type: int description: Define the minimum number of uppercase letters. Defaults to 1 if not @@ -8567,7 +8582,7 @@ OCIS_PASSWORD_POLICY_MIN_LOWERCASE_CHARACTERS: removalVersion: "" deprecationInfo: "" OCIS_PASSWORD_POLICY_MIN_SPECIAL_CHARACTERS: - name: OCIS_PASSWORD_POLICY_MIN_SPECIAL_CHARACTERS;SHARING_PASSWORD_POLICY_MIN_SPECIAL_CHARACTERS + name: OCIS_PASSWORD_POLICY_MIN_SPECIAL_CHARACTERS;FRONTEND_PASSWORD_POLICY_MIN_SPECIAL_CHARACTERS defaultValue: "1" type: int description: Define the minimum number of characters from the special characters @@ -8577,7 +8592,7 @@ OCIS_PASSWORD_POLICY_MIN_SPECIAL_CHARACTERS: removalVersion: "" deprecationInfo: "" OCIS_PASSWORD_POLICY_MIN_UPPERCASE_CHARACTERS: - name: OCIS_PASSWORD_POLICY_MIN_UPPERCASE_CHARACTERS;SHARING_PASSWORD_POLICY_MIN_UPPERCASE_CHARACTERS + name: OCIS_PASSWORD_POLICY_MIN_UPPERCASE_CHARACTERS;FRONTEND_PASSWORD_POLICY_MIN_UPPERCASE_CHARACTERS defaultValue: "1" type: int description: Define the minimum number of lowercase letters. Defaults to 1 if not @@ -8587,8 +8602,8 @@ OCIS_PASSWORD_POLICY_MIN_UPPERCASE_CHARACTERS: removalVersion: "" deprecationInfo: "" OCIS_PERSISTENT_STORE: - name: OCIS_PERSISTENT_STORE;USERLOG_STORE - defaultValue: memory + name: OCIS_PERSISTENT_STORE;POSTPROCESSING_STORE + defaultValue: nats-js-kv type: string description: 'The type of the store. Supported values are: ''memory'', ''ocmem'', ''etcd'', ''redis'', ''redis-sentinel'', ''nats-js'', ''noop''. See the text description @@ -8598,7 +8613,7 @@ OCIS_PERSISTENT_STORE: removalVersion: "" deprecationInfo: "" OCIS_PERSISTENT_STORE_AUTH_PASSWORD: - name: OCIS_PERSISTENT_STORE_AUTH_PASSWORD;USERLOG_STORE_AUTH_PASSWORD + name: OCIS_PERSISTENT_STORE_AUTH_PASSWORD;POSTPROCESSING_STORE_AUTH_PASSWORD defaultValue: "" type: string description: The password to authenticate with the store. Only applies when store @@ -8608,7 +8623,7 @@ OCIS_PERSISTENT_STORE_AUTH_PASSWORD: removalVersion: "" deprecationInfo: "" OCIS_PERSISTENT_STORE_AUTH_USERNAME: - name: OCIS_PERSISTENT_STORE_AUTH_USERNAME;USERLOG_STORE_AUTH_USERNAME + name: OCIS_PERSISTENT_STORE_AUTH_USERNAME;POSTPROCESSING_STORE_AUTH_USERNAME defaultValue: "" type: string description: The username to authenticate with the store. Only applies when store @@ -8618,8 +8633,8 @@ OCIS_PERSISTENT_STORE_AUTH_USERNAME: removalVersion: "" deprecationInfo: "" OCIS_PERSISTENT_STORE_NODES: - name: OCIS_PERSISTENT_STORE_NODES;USERLOG_STORE_NODES - defaultValue: '[]' + name: OCIS_PERSISTENT_STORE_NODES;POSTPROCESSING_STORE_NODES + defaultValue: '[127.0.0.1:9233]' type: '[]string' description: A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes @@ -8630,7 +8645,7 @@ OCIS_PERSISTENT_STORE_NODES: removalVersion: "" deprecationInfo: "" OCIS_PERSISTENT_STORE_SIZE: - name: OCIS_PERSISTENT_STORE_SIZE;USERLOG_STORE_SIZE + name: OCIS_PERSISTENT_STORE_SIZE;POSTPROCESSING_STORE_SIZE defaultValue: "0" type: int description: The maximum quantity of items in the store. Only applies when store @@ -8641,11 +8656,11 @@ OCIS_PERSISTENT_STORE_SIZE: removalVersion: "" deprecationInfo: "" OCIS_PERSISTENT_STORE_TTL: - name: OCIS_PERSISTENT_STORE_TTL;USERLOG_STORE_TTL - defaultValue: 336h0m0s + name: OCIS_PERSISTENT_STORE_TTL;POSTPROCESSING_STORE_TTL + defaultValue: 0s type: Duration - description: Time to live for events in the store. Defaults to '336h' (2 weeks). - See the Environment Variable Types description for more details. + description: Time to live for events in the store. See the Environment Variable + Types description for more details. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" @@ -8663,7 +8678,7 @@ OCIS_REVA_GATEWAY: name: OCIS_REVA_GATEWAY defaultValue: com.owncloud.api.gateway type: string - description: CS3 gateway used to look up user metadata + description: The CS3 gateway endpoint. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" @@ -8691,7 +8706,7 @@ OCIS_REVA_GATEWAY_TLS_MODE: removalVersion: "" deprecationInfo: "" OCIS_SERVICE_ACCOUNT_ID: - name: OCIS_SERVICE_ACCOUNT_ID;USERLOG_SERVICE_ACCOUNT_ID + name: OCIS_SERVICE_ACCOUNT_ID;FRONTEND_SERVICE_ACCOUNT_ID defaultValue: "" type: string description: The ID of the service account the service should use. See the 'auth-service' @@ -8701,7 +8716,7 @@ OCIS_SERVICE_ACCOUNT_ID: removalVersion: "" deprecationInfo: "" OCIS_SERVICE_ACCOUNT_SECRET: - name: OCIS_SERVICE_ACCOUNT_SECRET;USERLOG_SERVICE_ACCOUNT_SECRET + name: OCIS_SERVICE_ACCOUNT_SECRET;FRONTEND_SERVICE_ACCOUNT_SECRET defaultValue: "" type: string description: The service account secret. @@ -8710,7 +8725,7 @@ OCIS_SERVICE_ACCOUNT_SECRET: removalVersion: "" deprecationInfo: "" OCIS_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD: - name: OCIS_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD;SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD + name: OCIS_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD;FRONTEND_OCS_PUBLIC_SHARE_MUST_HAVE_PASSWORD defaultValue: "true" type: bool description: Set this to true if you want to enforce passwords on all public shares. @@ -8719,13 +8734,11 @@ OCIS_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD: removalVersion: "" deprecationInfo: "" OCIS_SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD: - name: OCIS_SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD;SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD + name: OCIS_SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD;FRONTEND_OCS_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD defaultValue: "false" type: bool - description: Set this to true if you want to enforce passwords on Uploader, Editor - or Contributor shares. If not using the global OCIS_SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD, - you must define the FRONTEND_OCS_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD in - the frontend service. + description: Set this to true if you want to enforce passwords for writable shares. + Only effective if the setting for 'passwords on all public shares' is set to false. introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" @@ -8773,7 +8786,7 @@ OCIS_SYSTEM_USER_ID: removalVersion: "" deprecationInfo: "" OCIS_SYSTEM_USER_IDP: - name: OCIS_SYSTEM_USER_IDP;SETTINGS_SYSTEM_USER_IDP + name: OCIS_SYSTEM_USER_IDP;SHARING_PUBLIC_CS3_SYSTEM_USER_IDP defaultValue: internal type: string description: IDP of the oCIS STORAGE-SYSTEM system user. @@ -8782,7 +8795,7 @@ OCIS_SYSTEM_USER_IDP: removalVersion: "" deprecationInfo: "" OCIS_TRACING_COLLECTOR: - name: OCIS_TRACING_COLLECTOR;USERLOG_TRACING_COLLECTOR + name: OCIS_TRACING_COLLECTOR;NATS_TRACING_COLLECTOR defaultValue: "" type: string description: The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. @@ -8792,7 +8805,7 @@ OCIS_TRACING_COLLECTOR: removalVersion: "" deprecationInfo: "" OCIS_TRACING_ENABLED: - name: OCIS_TRACING_ENABLED;USERLOG_TRACING_ENABLED + name: OCIS_TRACING_ENABLED;NATS_TRACING_ENABLED defaultValue: "false" type: bool description: Activates tracing. @@ -8801,7 +8814,7 @@ OCIS_TRACING_ENABLED: removalVersion: "" deprecationInfo: "" OCIS_TRACING_ENDPOINT: - name: OCIS_TRACING_ENDPOINT;USERLOG_TRACING_ENDPOINT + name: OCIS_TRACING_ENDPOINT;NATS_TRACING_ENDPOINT defaultValue: "" type: string description: The endpoint of the tracing agent. @@ -8810,7 +8823,7 @@ OCIS_TRACING_ENDPOINT: removalVersion: "" deprecationInfo: "" OCIS_TRACING_TYPE: - name: OCIS_TRACING_TYPE;USERLOG_TRACING_TYPE + name: OCIS_TRACING_TYPE;NATS_TRACING_TYPE defaultValue: "" type: string description: The type of tracing. Defaults to '', which is the same as 'jaeger'. @@ -8823,13 +8836,13 @@ OCIS_TRANSFER_SECRET: name: OCIS_TRANSFER_SECRET defaultValue: "" type: string - description: The storage transfer secret. + description: Transfer secret for signing file up- and download requests. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_TRANSLATION_PATH: - name: OCIS_TRANSLATION_PATH;USERLOG_TRANSLATION_PATH + name: OCIS_TRANSLATION_PATH;NOTIFICATIONS_TRANSLATION_PATH defaultValue: "" type: string description: (optional) Set this to a path with custom translations to overwrite @@ -8840,20 +8853,21 @@ OCIS_TRANSLATION_PATH: removalVersion: "" deprecationInfo: "" OCIS_URL: - name: OCIS_URL;OCIS_OIDC_ISSUER;PROXY_OIDC_ISSUER + name: OCIS_URL;OCIS_OIDC_ISSUER;GROUPS_IDP_URL defaultValue: https://localhost:9200 type: string - description: URL of the OIDC issuer. It defaults to URL of the builtin IDP. + description: The identity provider value to set in the group IDs of the CS3 group + objects for groups returned by this group provider. introductionVersion: pre5.0 deprecationVersion: "" removalVersion: "" deprecationInfo: "" OCIS_WOPI_DISABLE_CHAT: - name: APP_PROVIDER_WOPI_DISABLE_CHAT;OCIS_WOPI_DISABLE_CHAT + name: COLLABORATION_WOPI_DISABLE_CHAT;OCIS_WOPI_DISABLE_CHAT defaultValue: "false" type: bool - description: Disable the chat functionality of the office app. - introductionVersion: pre5.0 + description: Disable chat in the frontend. + introductionVersion: '%%NEXT%%' deprecationVersion: "" removalVersion: "" deprecationInfo: "" @@ -9103,8 +9117,8 @@ OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE: name: OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE defaultValue: /etc/ocis/ocmproviders.json type: string - description: Path to the JSON file where ocm invite data will be stored. If not - defined, the root directory derives from $OCIS_BASE_DATA_PATH:/storage. + description: Path to the JSON file where ocm invite data will be stored. Defaults + to $OCIS_CONFIG_DIR:/ocmproviders.json. introductionVersion: "5.0" deprecationVersion: "" removalVersion: "" From 4458eddb052988c0ad3430a77dfdf740ad0e3b1d Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 20 Aug 2024 16:27:53 +0200 Subject: [PATCH 25/71] feat(reva): Bump Reva Signed-off-by: jkoberg --- changelog/unreleased/bump-reva.md | 5 ++++ go.mod | 2 +- go.sum | 4 +-- .../handlers/apps/sharing/shares/shares.go | 30 +++++++++++++++++-- .../tusd/v2/pkg/handler/unrouted_handler.go | 20 +++++++++++++ vendor/modules.txt | 2 +- 6 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 changelog/unreleased/bump-reva.md diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md new file mode 100644 index 0000000000..ae9219216d --- /dev/null +++ b/changelog/unreleased/bump-reva.md @@ -0,0 +1,5 @@ +Enhancement: Bump reva + +Bumps reva version + +https://github.com/owncloud/ocis/pull/9860 diff --git a/go.mod b/go.mod index bbeec9d440..a797cf5947 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.11.0 github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb - github.com/cs3org/reva/v2 v2.23.0 + github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index 48c8967751..22327967aa 100644 --- a/go.sum +++ b/go.sum @@ -255,8 +255,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb h1:KmYZDReplv/yfwc1LNYpDcVhVujC3Pasv6WjXx1haSU= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb/go.mod h1:yyP8PRo0EZou3nSH7H4qjlzQwaydPeIRNgX50npQHpE= -github.com/cs3org/reva/v2 v2.23.0 h1:tRa+q6usndTQ6LbaxtfEub3UsKVruJ1l7HY6K+ZKS9s= -github.com/cs3org/reva/v2 v2.23.0/go.mod h1:p7CHBXcg6sSqB+0JMNDfC1S7TSh9FghXkw1kTV3KcJI= +github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d h1:fnb2A+ClZjlwMTPCBXQ3+NoSf7e5zBWK1XJsIaj0834= +github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d/go.mod h1:p7CHBXcg6sSqB+0JMNDfC1S7TSh9FghXkw1kTV3KcJI= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 1136ded122..1ec0b325e4 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -282,8 +282,34 @@ func (h *Handler) CreateShare(w http.ResponseWriter, r *http.Request) { reqRole, reqPermissions := r.FormValue("role"), r.FormValue("permissions") switch shareType { case int(conversions.ShareTypeUser), int(conversions.ShareTypeGroup): - // user collaborations default to Manager (=all permissions) - role, val, ocsErr := h.extractPermissions(reqRole, reqPermissions, statRes.Info, conversions.NewManagerRole()) + // NOTE: clients tend to send "31" as permissions but they mean "15". + // This is because it adds the "16" for sharing , but that is now no longer allowed. + // We could now have some fancy mechanism that casts the string to an int, subtracts 16 and casts it back to a string. + // Or we could change the role later and hope everything works out. + // Or: + if reqRole == "" { + switch reqPermissions { + case "31": + reqPermissions = "15" + case "29": + reqPermissions = "13" + case "27": + reqPermissions = "11" + case "23": + reqPermissions = "7" + case "22": + reqPermissions = "6" + case "21": + reqPermissions = "5" + case "19": + reqPermissions = "3" + case "17": + reqPermissions = "1" + } + } + + // user collaborations default to Viewer. Sane Default. + role, val, ocsErr := h.extractPermissions(reqRole, reqPermissions, statRes.Info, conversions.NewViewerRole()) if ocsErr != nil { response.WriteOCSError(w, r, ocsErr.Code, ocsErr.Message, ocsErr.Error) return diff --git a/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go b/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go index 73be46437e..385bf1c3f1 100644 --- a/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go +++ b/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go @@ -199,6 +199,7 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler { if origin := r.Header.Get("Origin"); !cors.Disable && origin != "" { originIsAllowed := cors.AllowOrigin.MatchString(origin) if !originIsAllowed { + fmt.Println("ORIGIN IS NOT ALLOWED", origin) handler.sendError(c, ErrOriginNotAllowed) return } @@ -691,12 +692,14 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) // PatchFile adds a chunk to an upload. This operation is only allowed // if enough space in the upload is left. func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request) { + fmt.Println("PATCH FILE") c := handler.getContext(w, r) isTusV1 := !handler.usesIETFDraft(r) // Check for presence of application/offset+octet-stream if isTusV1 && r.Header.Get("Content-Type") != "application/offset+octet-stream" { + fmt.Println("WRONG CONTENT TYPE") handler.sendError(c, ErrInvalidContentType) return } @@ -704,12 +707,14 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request // Check for presence of a valid Upload-Offset Header offset, err := strconv.ParseInt(r.Header.Get("Upload-Offset"), 10, 64) if err != nil || offset < 0 { + fmt.Println("WRONG OFFSET") handler.sendError(c, ErrInvalidOffset) return } id, err := extractIDFromPath(r.URL.Path) if err != nil { + fmt.Println("WRONG ID") handler.sendError(c, err) return } @@ -718,6 +723,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request if handler.composer.UsesLocker { lock, err := handler.lockUpload(c, id) if err != nil { + fmt.Println("WRONG LOCK") handler.sendError(c, err) return } @@ -727,23 +733,27 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request upload, err := handler.composer.Core.GetUpload(c, id) if err != nil { + fmt.Println("WRONG UPLOAD") handler.sendError(c, err) return } info, err := upload.GetInfo(c) if err != nil { + fmt.Println("WRONG INFO") handler.sendError(c, err) return } // Modifying a final upload is not allowed if info.IsFinal { + fmt.Println("WRONG FINAL") handler.sendError(c, ErrModifyFinal) return } if offset != info.Offset { + fmt.Println("WRONG INFO OFFSET") handler.sendError(c, ErrMismatchOffset) return } @@ -760,27 +770,32 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request // Do not proxy the call to the data store if the upload is already completed if !info.SizeIsDeferred && info.Offset == info.Size { resp.Header["Upload-Offset"] = strconv.FormatInt(offset, 10) + fmt.Println("UPLOAD ALREADY COMPLETED") handler.sendResp(c, resp) return } if r.Header.Get("Upload-Length") != "" { if !handler.composer.UsesLengthDeferrer { + fmt.Println("UPLOAD LENGTH DEFERRER") handler.sendError(c, ErrNotImplemented) return } if !info.SizeIsDeferred { + fmt.Println("UPLOAD LENGTH NOT DEFERED") handler.sendError(c, ErrInvalidUploadLength) return } uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) if err != nil || uploadLength < 0 || uploadLength < info.Offset || (handler.config.MaxSize > 0 && uploadLength > handler.config.MaxSize) { + fmt.Println("UPLOAD LENGTH INVALID") handler.sendError(c, ErrInvalidUploadLength) return } lengthDeclarableUpload := handler.composer.LengthDeferrer.AsLengthDeclarableUpload(upload) if err := lengthDeclarableUpload.DeclareLength(c, uploadLength); err != nil { + fmt.Println("UPLOAD LENGTH DECLARED") handler.sendError(c, err) return } @@ -791,6 +806,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request resp, err = handler.writeChunk(c, resp, upload, info) if err != nil { + fmt.Println("CANT WRITE CHUNK") handler.sendError(c, err) return } @@ -799,6 +815,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request if willCompleteUpload && info.SizeIsDeferred { info, err = upload.GetInfo(c) if err != nil { + fmt.Println("CANT GET INFO") handler.sendError(c, err) return } @@ -807,6 +824,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request lengthDeclarableUpload := handler.composer.LengthDeferrer.AsLengthDeclarableUpload(upload) if err := lengthDeclarableUpload.DeclareLength(c, uploadLength); err != nil { + fmt.Println("CANT UPLOAD LENGTH") handler.sendError(c, err) return } @@ -816,12 +834,14 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request resp, err = handler.finishUploadIfComplete(c, resp, upload, info) if err != nil { + fmt.Println("CANT COMPLETE") handler.sendError(c, err) return } } handler.sendResp(c, resp) + fmt.Println("PATCH COMPLETE") } // writeChunk reads the body from the requests r and appends it to the upload diff --git a/vendor/modules.txt b/vendor/modules.txt index f0cb12bbdf..54638f2bb1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -367,7 +367,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.23.0 +# github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime From 31df0ad513b3c1dae0050626815577cc16793561 Mon Sep 17 00:00:00 2001 From: Martin Date: Fri, 23 Aug 2024 11:04:38 +0200 Subject: [PATCH 26/71] [docs-only] Update add-translations.md Add an item to not forget maintaining the readme after adding the functionality. --- docs/services/general-info/add-translations.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/services/general-info/add-translations.md b/docs/services/general-info/add-translations.md index df2e9075a3..f1367db202 100644 --- a/docs/services/general-info/add-translations.md +++ b/docs/services/general-info/add-translations.md @@ -91,3 +91,6 @@ Translations have a `context` and a `translatable string`. The context is shown l10n-clean: rm -f $(TEMPLATE_FILE); ``` + +* Add Description Text to README\ + Add the full `Translations` and `Default Language` text blocks including their sub sections to the service readme. You can derive from the `activitylog` or `userlog` service for easy copy/paste. From d335d635b34ce6ff51dfe733701229027f199fb3 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 23 Aug 2024 11:08:29 +0200 Subject: [PATCH 27/71] feat(graph): add TRANSLATION_PATH envvar Signed-off-by: jkoberg --- .../unreleased/graph-translation-path.md | 5 ++++ services/graph/README.md | 26 +++++++++++++++++++ services/graph/pkg/config/config.go | 1 + services/graph/pkg/l10n/translation.go | 4 +-- .../graph/pkg/service/v0/spacetemplates.go | 6 ++--- 5 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/graph-translation-path.md diff --git a/changelog/unreleased/graph-translation-path.md b/changelog/unreleased/graph-translation-path.md new file mode 100644 index 0000000000..7413d81f99 --- /dev/null +++ b/changelog/unreleased/graph-translation-path.md @@ -0,0 +1,5 @@ +Enhancement: Graph translation path + +Add `GRAPH_TRANSLATION_PATH` envvar like in other l10n services + +https://github.com/owncloud/ocis/pull/9902 diff --git a/services/graph/README.md b/services/graph/README.md index 394ba12c7a..f711ff06bb 100644 --- a/services/graph/README.md +++ b/services/graph/README.md @@ -63,3 +63,29 @@ The client that is used to authenticate with keycloak has to be able to list use * `view-authorization` Note that these roles are only available to assign if the client is in the `master` realm. + +## Translations + +The `graph` service has embedded translations sourced via transifex to provide a basic set of translated languages. These embedded translations are available for all deployment scenarios. In addition, the service supports custom translations, though it is currently not possible to just add custom translations to embedded ones. If custom translations are configured, the embedded ones are not used. To configure custom translations, the `GRAPH_TRANSLATION_PATH` environment variable needs to point to a base folder that will contain the translation files. This path must be available from all instances of the graph service, a shared storage is recommended. Translation files must be of type [.po](https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html#PO-Files) or [.mo](https://www.gnu.org/software/gettext/manual/html_node/Binaries.html). For each language, the filename needs to be `graph.po` (or `graph.mo`) and stored in a folder structure defining the language code. In general the path/name pattern for a translation file needs to be: + +```text +{GRAPH_TRANSLATION_PATH}/{language-code}/LC_MESSAGES/graph.po +``` + +The language code pattern is composed of `language[_territory]` where `language` is the base language and `_territory` is optional and defines a country. + +For example, for the language `de`, one needs to place the corresponding translation files to `{GRAPH_TRANSLATION_PATH}/de_DE/LC_MESSAGES/graph.po`. + + + +Important: For the time being, the embedded ownCloud Web frontend only supports the main language code but does not handle any territory. When strings are available in the language code `language_territory`, the web frontend does not see it as it only requests `language`. In consequence, any translations made must exist in the requested `language` to avoid a fallback to the default. + +### Translation Rules + +* If a requested language code is not available, the service tries to fall back to the base language if available. For example, if the requested language-code `de_DE` is not available, the service tries to fall back to translations in the `de` folder. +* If the base language `de` is also not available, the service falls back to the system's default English (`en`), +which is the source of the texts provided by the code. + +## Default Language + +The default language can be defined via the `OCIS_DEFAULT_LANGUAGE` environment variable. See the `settings` service for a detailed description. diff --git a/services/graph/pkg/config/config.go b/services/graph/pkg/config/config.go index 31a9ce39b6..ab9fbe5295 100644 --- a/services/graph/pkg/config/config.go +++ b/services/graph/pkg/config/config.go @@ -46,6 +46,7 @@ type Spaces struct { GroupsCacheTTL int `yaml:"groups_cache_ttl" env:"GRAPH_SPACES_GROUPS_CACHE_TTL" desc:"Max TTL in seconds for the spaces groups cache." introductionVersion:"pre5.0"` StorageUsersAddress string `yaml:"storage_users_address" env:"GRAPH_SPACES_STORAGE_USERS_ADDRESS" desc:"The address of the storage-users service." introductionVersion:"5.0"` DefaultLanguage string `yaml:"default_language" env:"OCIS_DEFAULT_LANGUAGE" desc:"The default language used by services and the WebUI. If not defined, English will be used as default. See the documentation for more details." introductionVersion:"5.0"` + TranslationPath string `yaml:"translation_path" env:"OCIS_TRANSLATION_PATH;GRAPH_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. Note that file and folder naming rules apply, see the documentation for more details." introductionVersion:"%%NEXT%%"` } type LDAP struct { diff --git a/services/graph/pkg/l10n/translation.go b/services/graph/pkg/l10n/translation.go index d2f7c0994d..c30e2177c5 100644 --- a/services/graph/pkg/l10n/translation.go +++ b/services/graph/pkg/l10n/translation.go @@ -20,8 +20,8 @@ const ( ) // Translate translates a string based on the locale and default locale -func Translate(content, locale, defaultLocale string) string { - t := l10n.NewTranslatorFromCommonConfig(defaultLocale, _domain, "", _localeFS, _localeSubPath) +func Translate(content, locale, defaultLocale, translationPath string) string { + t := l10n.NewTranslatorFromCommonConfig(defaultLocale, _domain, translationPath, _localeFS, _localeSubPath) return t.Translate(content, locale) } diff --git a/services/graph/pkg/service/v0/spacetemplates.go b/services/graph/pkg/service/v0/spacetemplates.go index 75da3a7286..99c64ec41b 100644 --- a/services/graph/pkg/service/v0/spacetemplates.go +++ b/services/graph/pkg/service/v0/spacetemplates.go @@ -71,7 +71,7 @@ func (g Graph) applyDefaultTemplate(ctx context.Context, gwc gateway.GatewayAPIC opaque = utils.AppendPlainToOpaque(opaque, SpaceImageSpecialFolderName, iid) // upload readme.md - rid, err := readmeUpload(ctx, mdc, locale, g.config.Spaces.DefaultLanguage) + rid, err := readmeUpload(ctx, mdc, locale, g.config.Spaces.DefaultLanguage, g.config.Spaces.TranslationPath) if err != nil { return err } @@ -112,10 +112,10 @@ func imageUpload(ctx context.Context, mdc *metadata.CS3) (string, error) { return res.FileID, nil } -func readmeUpload(ctx context.Context, mdc *metadata.CS3, locale string, defaultLocale string) (string, error) { +func readmeUpload(ctx context.Context, mdc *metadata.CS3, locale string, defaultLocale string, translationPath string) (string, error) { res, err := mdc.Upload(ctx, metadata.UploadRequest{ Path: filepath.Join(_spaceFolderName, _readmeName), - Content: []byte(l10n_pkg.Translate(_readmeText, locale, defaultLocale)), + Content: []byte(l10n_pkg.Translate(_readmeText, locale, defaultLocale, translationPath)), }) if err != nil { return "", err From ba02d480a6be05e4d1b76edd228ad2dd1ef863e3 Mon Sep 17 00:00:00 2001 From: kobergj Date: Fri, 23 Aug 2024 09:11:59 +0000 Subject: [PATCH 28/71] Automated changelog update [skip ci] --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9a631e378..c6458802ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ The following sections list the changes for unreleased. * Bugfix - Activity Translations: [#9856](https://github.com/owncloud/ocis/pull/9856) * Bugfix - The user attributes `userType` and `memberOf` are readonly: [#9867](https://github.com/owncloud/ocis/pull/9867) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) +* Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) * Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) ## Details @@ -90,6 +91,13 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/1357 https://github.com/owncloud/ocis/pull/9890 +* Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) + + The `revisions purge` command would time out on big spaces. We have improved + performance by parallelizing the process. + + https://github.com/owncloud/ocis/pull/9891 + * Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) Allows setting the default locale via `OCIS_DEFAULT_LANGUAGE` envvar From 5d3de7730d2d6c1198a7e500f79ca5f72271872c Mon Sep 17 00:00:00 2001 From: kobergj Date: Fri, 23 Aug 2024 09:32:59 +0000 Subject: [PATCH 29/71] Automated changelog update [skip ci] --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6458802ff..b31acbbb07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ The following sections list the changes for unreleased. * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) * Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) +* Enhancement - Graph translation path: [#9902](https://github.com/owncloud/ocis/pull/9902) ## Details @@ -104,6 +105,12 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/pull/9892 +* Enhancement - Graph translation path: [#9902](https://github.com/owncloud/ocis/pull/9902) + + Add `GRAPH_TRANSLATION_PATH` envvar like in other l10n services + + https://github.com/owncloud/ocis/pull/9902 + # Changelog for [6.3.0] (2024-08-20) The following sections list the changes for 6.3.0. From 6d465575f4883b913935c38b5983f10d3cc4d3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 23 Aug 2024 11:34:40 +0200 Subject: [PATCH 30/71] List OCM permissions as graph drive item permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/list-ocm-permissions.md | 6 + .../service/v0/api_driveitem_permissions.go | 11 ++ services/graph/pkg/service/v0/base.go | 147 ++++++++++++++++++ services/graph/pkg/service/v0/utils.go | 19 +++ 4 files changed, 183 insertions(+) create mode 100644 changelog/unreleased/list-ocm-permissions.md diff --git a/changelog/unreleased/list-ocm-permissions.md b/changelog/unreleased/list-ocm-permissions.md new file mode 100644 index 0000000000..5f5afa7f1c --- /dev/null +++ b/changelog/unreleased/list-ocm-permissions.md @@ -0,0 +1,6 @@ +Bugfix: List OCM permissions as graph drive item permissions + +The libre graph API now returns OCM shares when listing driveItem permissions. + +https://github.com/owncloud/ocis/pull/9905 +https://github.com/owncloud/ocis/issues/9898 diff --git a/services/graph/pkg/service/v0/api_driveitem_permissions.go b/services/graph/pkg/service/v0/api_driveitem_permissions.go index 0c571cbec3..292a098b51 100644 --- a/services/graph/pkg/service/v0/api_driveitem_permissions.go +++ b/services/graph/pkg/service/v0/api_driveitem_permissions.go @@ -382,6 +382,17 @@ func (s DriveItemPermissionsService) ListPermissions(ctx context.Context, itemID if err != nil { return collectionOfPermissions, err } + if s.config.IncludeOCMSharees { + driveItems, err = s.listOCMShares(ctx, []*ocm.ListOCMSharesRequest_Filter{ + { + Type: ocm.ListOCMSharesRequest_Filter_TYPE_RESOURCE_ID, + Term: &ocm.ListOCMSharesRequest_Filter_ResourceId{ResourceId: itemID}, + }, + }, driveItems) + if err != nil { + return collectionOfPermissions, err + } + } } // finally get public shares, which are possible for spaceroots and "normal" resources driveItems, err = s.listPublicShares(ctx, []*link.ListPublicSharesRequest_Filter{ diff --git a/services/graph/pkg/service/v0/base.go b/services/graph/pkg/service/v0/base.go index dff462afdb..6c8a58ecd6 100644 --- a/services/graph/pkg/service/v0/base.go +++ b/services/graph/pkg/service/v0/base.go @@ -275,6 +275,34 @@ func (g BaseGraphService) listUserShares(ctx context.Context, filters []*collabo return driveItems, nil } +func (g BaseGraphService) listOCMShares(ctx context.Context, filters []*ocm.ListOCMSharesRequest_Filter, driveItems driveItemsByResourceID) (driveItemsByResourceID, error) { + gatewayClient, err := g.gatewaySelector.Next() + if err != nil { + g.logger.Error().Err(err).Msg("could not select next gateway client") + return driveItems, errorcode.New(errorcode.GeneralException, err.Error()) + } + + concreteFilters := []*ocm.ListOCMSharesRequest_Filter{} + concreteFilters = append(concreteFilters, filters...) + + lsOCMSharesRequest := ocm.ListOCMSharesRequest{ + Filters: concreteFilters, + } + + lsOCMSharesResponse, err := gatewayClient.ListOCMShares(ctx, &lsOCMSharesRequest) + if err != nil { + return driveItems, errorcode.New(errorcode.GeneralException, err.Error()) + } + if statusCode := lsOCMSharesResponse.GetStatus().GetCode(); statusCode != rpc.Code_CODE_OK { + return driveItems, errorcode.New(cs3StatusToErrCode(statusCode), lsOCMSharesResponse.Status.Message) + } + driveItems, err = g.cs3OCMSharesToDriveItems(ctx, lsOCMSharesResponse.Shares, driveItems) + if err != nil { + return driveItems, errorcode.New(errorcode.GeneralException, err.Error()) + } + return driveItems, nil +} + func (g BaseGraphService) listPublicShares(ctx context.Context, filters []*link.ListPublicSharesRequest_Filter, driveItems driveItemsByResourceID) (driveItemsByResourceID, error) { gatewayClient, err := g.gatewaySelector.Next() @@ -343,6 +371,42 @@ func (g BaseGraphService) cs3UserSharesToDriveItems(ctx context.Context, shares } return driveItems, nil } +func (g BaseGraphService) cs3OCMSharesToDriveItems(ctx context.Context, shares []*ocm.Share, driveItems driveItemsByResourceID) (driveItemsByResourceID, error) { + for _, s := range shares { + g.logger.Debug().Interface("CS3 OCMShare", s).Msg("Got Share") + resIDStr := storagespace.FormatResourceID(s.ResourceId) + item, ok := driveItems[resIDStr] + if !ok { + itemptr, err := g.getDriveItem(ctx, &storageprovider.Reference{ResourceId: s.ResourceId}) + if err != nil { + g.logger.Debug().Err(err).Interface("Share", s.ResourceId).Msg("could not stat ocm share, skipping") + continue + } + item = *itemptr + } + + var condition string + switch { + case item.Folder != nil: + condition = unifiedrole.UnifiedRoleConditionFolderFederatedUser + case item.File != nil: + condition = unifiedrole.UnifiedRoleConditionFileFederatedUser + } + perm, err := g.cs3OCMShareToPermission(ctx, s, condition) + + var errcode errorcode.Error + switch { + case errors.As(err, &errcode) && errcode.GetCode() == errorcode.ItemNotFound: + // The Grantee couldn't be found (user/group does not exist anymore) + continue + case err != nil: + return driveItems, err + } + item.Permissions = append(item.Permissions, *perm) + driveItems[resIDStr] = item + } + return driveItems, nil +} func (g BaseGraphService) cs3UserShareToPermission(ctx context.Context, share *collaboration.Share, roleCondition string) (*libregraph.Permission, error) { perm := libregraph.Permission{} @@ -419,6 +483,89 @@ func (g BaseGraphService) cs3UserShareToPermission(ctx context.Context, share *c } return &perm, nil } +func (g BaseGraphService) cs3OCMShareToPermission(ctx context.Context, share *ocm.Share, roleCondition string) (*libregraph.Permission, error) { + perm := libregraph.Permission{} + perm.SetRoles([]string{}) + if roleCondition != unifiedrole.UnifiedRoleConditionDrive { + perm.SetId(share.GetId().GetOpaqueId()) + } + grantedTo := libregraph.SharePointIdentitySet{} + // hm or use share.GetShareType() to determine the type of share??? + switch share.GetGrantee().GetType() { + case storageprovider.GranteeType_GRANTEE_TYPE_USER: + user, err := cs3UserIdToIdentity(ctx, g.identityCache, share.Grantee.GetUserId()) + switch { + case errors.Is(err, identity.ErrNotFound): + g.logger.Warn().Str("userid", share.Grantee.GetUserId().GetOpaqueId()).Msg("User not found by id") + // User does not seem to exist anymore, don't add a permission for this + return nil, errorcode.New(errorcode.ItemNotFound, "grantee does not exist") + case err != nil: + return nil, errorcode.New(errorcode.GeneralException, err.Error()) + default: + grantedTo.SetUser(user) + if roleCondition == unifiedrole.UnifiedRoleConditionDrive { + perm.SetId("u:" + user.GetId()) + } + } + case storageprovider.GranteeType_GRANTEE_TYPE_GROUP: + group, err := groupIdToIdentity(ctx, g.identityCache, share.Grantee.GetGroupId().GetOpaqueId()) + switch { + case errors.Is(err, identity.ErrNotFound): + g.logger.Warn().Str("groupid", share.Grantee.GetGroupId().GetOpaqueId()).Msg("Group not found by id") + // Group not seem to exist anymore, don't add a permission for this + return nil, errorcode.New(errorcode.ItemNotFound, "grantee does not exist") + case err != nil: + return nil, errorcode.New(errorcode.GeneralException, err.Error()) + default: + grantedTo.SetGroup(group) + if roleCondition == unifiedrole.UnifiedRoleConditionDrive { + perm.SetId("g:" + group.GetId()) + } + } + } + + // set expiration date + if share.GetExpiration() != nil { + perm.SetExpirationDateTime(cs3TimestampToTime(share.GetExpiration())) + } + // set cTime + if share.GetCtime() != nil { + perm.SetCreatedDateTime(cs3TimestampToTime(share.GetCtime())) + } + var permissions *storageprovider.ResourcePermissions + for _, role := range share.GetAccessMethods() { + if role.GetWebdavOptions().GetPermissions() != nil { + permissions = role.GetWebdavOptions().GetPermissions() + } + } + + role := unifiedrole.CS3ResourcePermissionsToUnifiedRole( + permissions, + roleCondition, + ) + if role != nil { + perm.SetRoles([]string{role.GetId()}) + } else { + actions := unifiedrole.CS3ResourcePermissionsToLibregraphActions(permissions) + perm.SetLibreGraphPermissionsActions(actions) + perm.SetRoles(nil) + } + perm.SetGrantedToV2(grantedTo) + if share.GetCreator() != nil { + identity, err := cs3UserIdToIdentity(ctx, g.identityCache, share.GetCreator()) + if err != nil { + return nil, errorcode.New(errorcode.GeneralException, err.Error()) + } + perm.SetInvitation( + libregraph.SharingInvitation{ + InvitedBy: &libregraph.IdentitySet{ + User: &identity, + }, + }, + ) + } + return &perm, nil +} func (g BaseGraphService) cs3PublicSharesToDriveItems(ctx context.Context, shares []*link.PublicShare, driveItems driveItemsByResourceID) (driveItemsByResourceID, error) { for _, s := range shares { diff --git a/services/graph/pkg/service/v0/utils.go b/services/graph/pkg/service/v0/utils.go index 9a1cd60ee9..d08a05567c 100644 --- a/services/graph/pkg/service/v0/utils.go +++ b/services/graph/pkg/service/v0/utils.go @@ -108,6 +108,22 @@ func userIdToIdentity(ctx context.Context, cache identity.IdentityCache, userID user, err := cache.GetUser(ctx, userID) if err == nil { identity.SetDisplayName(user.GetDisplayName()) + identity.SetLibreGraphUserType(user.GetUserType()) + } + return identity, err +} + +// federatedIdToIdentity looks the user for the supplied id using the cache and returns it +// as a libregraph.Identity +func federatedIdToIdentity(ctx context.Context, cache identity.IdentityCache, userID string) (libregraph.Identity, error) { + identity := libregraph.Identity{ + Id: libregraph.PtrString(userID), + LibreGraphUserType: libregraph.PtrString("Federated"), + } + user, err := cache.GetAcceptedUser(ctx, userID) + if err == nil { + identity.SetDisplayName(user.GetDisplayName()) + identity.SetLibreGraphUserType(user.GetUserType()) } return identity, err } @@ -115,6 +131,9 @@ func userIdToIdentity(ctx context.Context, cache identity.IdentityCache, userID // cs3UserIdToIdentity looks up the user for the supplied cs3 userid using the cache and returns it // as a libregraph.Identity. Skips the user lookup if the id type is USER_TYPE_SPACE_OWNER func cs3UserIdToIdentity(ctx context.Context, cache identity.IdentityCache, cs3UserID *cs3User.UserId) (libregraph.Identity, error) { + if cs3UserID.GetType() == cs3User.UserType_USER_TYPE_FEDERATED { + return federatedIdToIdentity(ctx, cache, cs3UserID.GetOpaqueId()) + } if cs3UserID.GetType() != cs3User.UserType_USER_TYPE_SPACE_OWNER { return userIdToIdentity(ctx, cache, cs3UserID.GetOpaqueId()) } From 38a23ee9bb0a44587faa2dae7fcd563ee5ccbe86 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 23 Aug 2024 11:23:41 +0200 Subject: [PATCH 31/71] feat(ocis): update revisions docu Signed-off-by: jkoberg --- ocis/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ocis/README.md b/ocis/README.md index a1ef4bcb63..7003361d0e 100644 --- a/ocis/README.md +++ b/ocis/README.md @@ -89,6 +89,8 @@ Do not remove any revisions but print the revisions that would be removed. Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see the `storage-users` service for more details). * `-v` / `--verbose`\ Prints additional information about the revisions that are removed. +* `--glob-mechanism` (default: `glob`\ +(advanced) Allows specifying the mechanism to use for globbing. Can be `glob`, `list` or `workers`. In most cases the default `glob` does not need to be changed. If large spaces need to be purged, `list` or `workers` can be used to improve performance at the cost of higher cpu and ram usage. `list` will spawn 10 threads that list folder contents in parallel. `workers` will use a special globbing mechanism and multiple threads to achieve the best performance for the highest cost. ### Trash CLI From b53500835c9467433629d5f1f799c026ca28bcdf Mon Sep 17 00:00:00 2001 From: kobergj Date: Fri, 23 Aug 2024 10:03:40 +0000 Subject: [PATCH 32/71] Automated changelog update [skip ci] --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b31acbbb07..b11169f3d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ The following sections list the changes for unreleased. * Bugfix - Activity Translations: [#9856](https://github.com/owncloud/ocis/pull/9856) * Bugfix - The user attributes `userType` and `memberOf` are readonly: [#9867](https://github.com/owncloud/ocis/pull/9867) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) +* Enhancement - Bump reva: [#9860](https://github.com/owncloud/ocis/pull/9860) * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) * Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) * Enhancement - Graph translation path: [#9902](https://github.com/owncloud/ocis/pull/9902) @@ -92,6 +93,12 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/1357 https://github.com/owncloud/ocis/pull/9890 +* Enhancement - Bump reva: [#9860](https://github.com/owncloud/ocis/pull/9860) + + Bumps reva version + + https://github.com/owncloud/ocis/pull/9860 + * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) The `revisions purge` command would time out on big spaces. We have improved From b398cb1b4b742797c29b316ba408c8f0fdc39a54 Mon Sep 17 00:00:00 2001 From: Prajwol Amatya Date: Wed, 21 Aug 2024 16:44:55 +0545 Subject: [PATCH 33/71] add test coverage for /app/ endpoint --- tests/acceptance/config/behat.yml | 7 +++ .../apiAppProvider/appProvider.feature | 43 +++++++++++++++++++ .../features/bootstrap/SpacesContext.php | 22 ++++++++++ 3 files changed, 72 insertions(+) create mode 100644 tests/acceptance/features/apiAppProvider/appProvider.feature diff --git a/tests/acceptance/config/behat.yml b/tests/acceptance/config/behat.yml index a587cc2474..7bcecc6b35 100644 --- a/tests/acceptance/config/behat.yml +++ b/tests/acceptance/config/behat.yml @@ -414,6 +414,13 @@ default: - TagContext: - TrashbinContext: + apiAppProvider: + paths: + - "%paths.base%/../features/apiAppProvider" + context: *common_ldap_suite_context + contexts: + - FeatureContext: *common_feature_context_params + extensions: rdx\behatvars\BehatVariablesExtension: ~ diff --git a/tests/acceptance/features/apiAppProvider/appProvider.feature b/tests/acceptance/features/apiAppProvider/appProvider.feature new file mode 100644 index 0000000000..1c39b9dee2 --- /dev/null +++ b/tests/acceptance/features/apiAppProvider/appProvider.feature @@ -0,0 +1,43 @@ +Feature: App Provider + + + Scenario: open file with Collabora + Given user "Alice" has been created with default attributes and without skeleton files + And user "Alice" has uploaded file "filesForUpload/simple.odt" to "simple.odt" + When user "Alice" opens file "simple.odt" with Collabora service + Then the HTTP status code should be "200" + And the JSON data of the response should match + """ + { + "type": "object", + "required": [ + "app_url", + "method", + "form_parameters" + ], + "properties": { + "app_url": { + "type": "string", + "pattern": "^https:\\/\\/host\\.docker\\.internal:9980\\/browser\\/8ec5fda\\/cool\\.html\\?WOPISrc=https%3A%2F%2Fhost\\.docker\\.internal%3A9300%2Fwopi%2Ffiles%2F[a-fA-F0-9]{64}$" + }, + "method": { + "const": "POST" + }, + "form_parameters": { + "type": "object", + "required": [ + "access_token", + "access_token_ttl" + ], + "properties": { + "access_token": { + "type": "string" + }, + "access_token_ttl": { + "type": "string" + } + } + } + } + } + """ diff --git a/tests/acceptance/features/bootstrap/SpacesContext.php b/tests/acceptance/features/bootstrap/SpacesContext.php index 9cf94d8608..bbf0de5e78 100644 --- a/tests/acceptance/features/bootstrap/SpacesContext.php +++ b/tests/acceptance/features/bootstrap/SpacesContext.php @@ -4131,4 +4131,26 @@ class SpacesContext implements Context { ) ); } + + /** + * @When user :user opens file :filename with Collabora service + * + * @param string $user + * @param string $filename + * + * @return void + */ + public function userOpensFileWithCollaboraService(string $user, string $filename): void + { + $fileId = $this->getFileId($user, "Personal", $filename); + $url = $this->featureContext->getBaseUrl() . "/app/open?file_id=$fileId&app_name=Collabora"; + $this->featureContext->setResponse( + HttpRequestHelper::post( + $url, + $this->featureContext->getStepLineRef(), + $user, + $this->featureContext->getPasswordForUser($user) + ) + ); + } } From 964dfc62f00f4b9419b5b177981c911d306054bf Mon Sep 17 00:00:00 2001 From: Prajwol Amatya Date: Thu, 22 Aug 2024 11:31:12 +0545 Subject: [PATCH 34/71] setup fakeoffice for acceptance test --- .drone.star | 110 ++++++++++++++---- .../apiAppProvider/appProvider.feature | 2 +- .../features/bootstrap/SpacesContext.php | 41 ++++--- 3 files changed, 111 insertions(+), 42 deletions(-) diff --git a/.drone.star b/.drone.star index e278eae0db..1aa5acd83f 100644 --- a/.drone.star +++ b/.drone.star @@ -59,13 +59,13 @@ dirs = { # configuration config = { "cs3ApiTests": { - "skip": False, + "skip": True, }, "wopiValidatorTests": { - "skip": False, + "skip": True, }, "k6LoadTests": { - "skip": False, + "skip": True, }, "localApiTests": { "basic": { @@ -92,19 +92,20 @@ config = { "apiSharingNgLinkShareRoot", "apiActivities", ], - "skip": False, + "skip": True, }, "apiAccountsHashDifficulty": { "suites": [ "apiAccountsHashDifficulty", ], "accounts_hash_difficulty": "default", + "skip": True, }, "apiNotification": { "suites": [ "apiNotification", ], - "skip": False, + "skip": True, "emailNeeded": True, "extraEnvironment": { "EMAIL_HOST": "email", @@ -122,7 +123,7 @@ config = { "suites": [ "apiAntivirus", ], - "skip": False, + "skip": True, "antivirusNeeded": True, "extraServerEnvironment": { "ANTIVIRUS_SCANNER_TYPE": "clamav", @@ -136,14 +137,14 @@ config = { "suites": [ "apiSearchContent", ], - "skip": False, + "skip": True, "tikaNeeded": True, }, "apiOcm": { "suites": [ "apiOcm", ], - "skip": False, + "skip": True, "federationServer": True, "extraServerEnvironment": { "OCIS_ADD_RUN_SERVICES": "ocm", @@ -158,22 +159,36 @@ config = { "suites": [ "cliCommands", ], + "skip": True, + }, + "apiAppProvider": { + "suites": [ + "apiAppProvider", + ], "skip": False, + "collaborationServiceNeeded": True, + "extraCollaborationEnvironment": { + "COLLABORATION_APP_NAME": "FakeOffice", + "COLLABORATION_APP_ADDR": "http://fakeoffice:8080", + }, + "extraServerEnvironment": { + "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", + }, }, }, "apiTests": { "numberOfParts": 10, - "skip": False, + "skip": True, "skipExceptParts": [], }, "e2eTests": { "part": { - "skip": False, + "skip": True, "totalParts": 4, # divide and run all suites in parts (divide pipelines) "xsuites": ["search", "app-provider", "oidc"], # suites to skip }, "search": { - "skip": False, + "skip": True, "suites": ["search"], # suites to run "tikaNeeded": True, }, @@ -189,7 +204,7 @@ config = { "dockerReleases": { "architectures": ["arm64", "amd64"], }, - "litmus": True, + "litmus": False, "codestyle": True, } @@ -280,15 +295,9 @@ def main(ctx): licenseCheck(ctx) test_pipelines = \ - codestyle(ctx) + \ - checkGherkinLint(ctx) + \ - checkTestSuitesInExpectedFailures(ctx) + \ buildWebCache(ctx) + \ getGoBinForTesting(ctx) + \ buildOcisBinaryForTesting(ctx) + \ - checkStarlark() + \ - build_release_helpers + \ - testOcisAndUploadResults(ctx) + \ testPipelines(ctx) build_release_pipelines = \ @@ -302,7 +311,7 @@ def main(ctx): ), ) - pipelines = test_pipelines + build_release_pipelines + pipelines = test_pipelines if ctx.build.event == "cron": pipelines = \ @@ -855,6 +864,8 @@ def localApiTestPipeline(ctx): "antivirusNeeded": False, "tikaNeeded": False, "federationServer": False, + "collaborationServiceNeeded": False, + "extraCollaborationEnvironment": {}, } if "localApiTests" in config: @@ -880,9 +891,10 @@ def localApiTestPipeline(ctx): (waitForClamavService() if params["antivirusNeeded"] else []) + (waitForEmailService() if params["emailNeeded"] else []) + (ocisServer(storage, params["accounts_hash_difficulty"], deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"]) if params["federationServer"] else []) + + (collaborationService(extra_environment = params["extraCollaborationEnvironment"]) if params["collaborationServiceNeeded"] else []) + localApiTests(suite, storage, params["extraEnvironment"]) + logRequests(), - "services": emailService() if params["emailNeeded"] else [] + clamavService() if params["antivirusNeeded"] else [], + "services": emailService() if params["emailNeeded"] else [] + clamavService() if params["antivirusNeeded"] else [] + fakeOffice() if params["collaborationServiceNeeded"] else [], "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx)), "trigger": { "ref": [ @@ -2894,6 +2906,64 @@ def waitForClamavService(): ], }] +def fakeOffice(): + return [ + { + "name": "fakeoffice", + "image": OC_CI_ALPINE, + "detach": True, + "environment": {}, + "commands": [ + "sh %s/tests/config/drone/serve-hosting-discovery.sh" % (dirs["base"]), + ], + }, + { + "name": "wait-for-fakeoffice", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it fakeoffice:8080 -t 300", + ], + }, + ] + +def collaborationService(extra_environment = {}): + environment = { + "MICRO_REGISTRY": "nats-js-kv", + "MICRO_REGISTRY_ADDRESS": "ocis-server:9233", + "COLLABORATION_LOG_LEVEL": "debug", + "COLLABORATION_HTTP_ADDR": "0.0.0.0:9300", + "COLLABORATION_GRPC_ADDR": "0.0.0.0:9301", + # no proof keys available in the FakeOffice + "COLLABORATION_APP_PROOF_DISABLE": "true", + "COLLABORATION_APP_INSECURE": "true", + "COLLABORATION_WOPI_SRC": "http://wopiserver:9300", + "COLLABORATION_WOPI_SECRET": "some-wopi-secret", + "COLLABORATION_CS3API_DATAGATEWAY_INSECURE": "true", + "OCIS_JWT_SECRET": "some-ocis-jwt-secret", + } + + for item in extra_environment: + environment[item] = extra_environment[item] + + return [ + { + "name": "wopiserver", + "image": OC_CI_GOLANG, + "detach": True, + "environment": environment, + "commands": [ + "ocis/bin/ocis collaboration server", + ], + }, + { + "name": "wait-for-wopi-server", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it wopiserver:9300 -t 300", + ], + }, + ] + def tikaService(): return [{ "name": "tika", diff --git a/tests/acceptance/features/apiAppProvider/appProvider.feature b/tests/acceptance/features/apiAppProvider/appProvider.feature index 1c39b9dee2..84515fda8b 100644 --- a/tests/acceptance/features/apiAppProvider/appProvider.feature +++ b/tests/acceptance/features/apiAppProvider/appProvider.feature @@ -4,7 +4,7 @@ Feature: App Provider Scenario: open file with Collabora Given user "Alice" has been created with default attributes and without skeleton files And user "Alice" has uploaded file "filesForUpload/simple.odt" to "simple.odt" - When user "Alice" opens file "simple.odt" with Collabora service + When user "Alice" opens file "simple.odt" with collaboration service Then the HTTP status code should be "200" And the JSON data of the response should match """ diff --git a/tests/acceptance/features/bootstrap/SpacesContext.php b/tests/acceptance/features/bootstrap/SpacesContext.php index bbf0de5e78..3a224d9610 100644 --- a/tests/acceptance/features/bootstrap/SpacesContext.php +++ b/tests/acceptance/features/bootstrap/SpacesContext.php @@ -4132,25 +4132,24 @@ class SpacesContext implements Context { ); } - /** - * @When user :user opens file :filename with Collabora service - * - * @param string $user - * @param string $filename - * - * @return void - */ - public function userOpensFileWithCollaboraService(string $user, string $filename): void - { - $fileId = $this->getFileId($user, "Personal", $filename); - $url = $this->featureContext->getBaseUrl() . "/app/open?file_id=$fileId&app_name=Collabora"; - $this->featureContext->setResponse( - HttpRequestHelper::post( - $url, - $this->featureContext->getStepLineRef(), - $user, - $this->featureContext->getPasswordForUser($user) - ) - ); - } + /** + * @When user :user opens file :filename with collaboration service + * + * @param string $user + * @param string $filename + * + * @return void + */ + public function userOpensFileWithCollaborationService(string $user, string $filename): void { + $fileId = $this->getFileId($user, "Personal", $filename); + $url = $this->featureContext->getBaseUrl() . "/app/open?file_id=$fileId&app_name=FakeOffice"; + $this->featureContext->setResponse( + HttpRequestHelper::post( + $url, + $this->featureContext->getStepLineRef(), + $user, + $this->featureContext->getPasswordForUser($user) + ) + ); + } } From f79f8bb17deb305bd883e4b68ae31213b2843b61 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Thu, 22 Aug 2024 16:50:31 +0545 Subject: [PATCH 35/71] ci: check requests --- .drone.star | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.drone.star b/.drone.star index 1aa5acd83f..2a48cd30ac 100644 --- a/.drone.star +++ b/.drone.star @@ -2962,6 +2962,24 @@ def collaborationService(extra_environment = {}): "wait-for -it wopiserver:9300 -t 300", ], }, + { + "name": "prepare-test-file", + "image": OC_CI_ALPINE, + "environment": {}, + "commands": [ + "curl -v -X PUT 'https://ocis-server:9200/remote.php/webdav/test.wopitest' -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin -D headers.txt", + "cat headers.txt", + "export FILE_ID=$(cat headers.txt | sed -n -e 's/^.*Oc-Fileid: //p')", + "export URL=\"https://ocis-server:9200/app/open?app_name=FakeOffice&file_id=$FILE_ID\"", + "export URL=$(echo $URL | tr -d '[:cntrl:]')", + "curl -v -X POST \"$URL\" -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin > open.json", + "cat open.json", + "cat open.json | jq .form_parameters.access_token | tr -d '\"' > accesstoken", + "cat open.json | jq .form_parameters.access_token_ttl | tr -d '\"' > accesstokenttl", + "echo -n 'http://wopiserver:9300/wopi/files/' > wopisrc", + "cat open.json | jq .app_url | sed -n -e 's/^.*files%2F//p' | tr -d '\"' >> wopisrc", + ], + }, ] def tikaService(): From 330952733b989df16251010743b86d4a2a20e09f Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Thu, 22 Aug 2024 17:45:20 +0545 Subject: [PATCH 36/71] test: allow FakeOffice to work with other file extensions --- .drone.star | 21 --------------------- tests/config/drone/hosting-discovery.xml | 18 +++++++++++------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/.drone.star b/.drone.star index 2a48cd30ac..935694f4fd 100644 --- a/.drone.star +++ b/.drone.star @@ -295,8 +295,6 @@ def main(ctx): licenseCheck(ctx) test_pipelines = \ - buildWebCache(ctx) + \ - getGoBinForTesting(ctx) + \ buildOcisBinaryForTesting(ctx) + \ testPipelines(ctx) @@ -2933,7 +2931,6 @@ def collaborationService(extra_environment = {}): "COLLABORATION_LOG_LEVEL": "debug", "COLLABORATION_HTTP_ADDR": "0.0.0.0:9300", "COLLABORATION_GRPC_ADDR": "0.0.0.0:9301", - # no proof keys available in the FakeOffice "COLLABORATION_APP_PROOF_DISABLE": "true", "COLLABORATION_APP_INSECURE": "true", "COLLABORATION_WOPI_SRC": "http://wopiserver:9300", @@ -2962,24 +2959,6 @@ def collaborationService(extra_environment = {}): "wait-for -it wopiserver:9300 -t 300", ], }, - { - "name": "prepare-test-file", - "image": OC_CI_ALPINE, - "environment": {}, - "commands": [ - "curl -v -X PUT 'https://ocis-server:9200/remote.php/webdav/test.wopitest' -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin -D headers.txt", - "cat headers.txt", - "export FILE_ID=$(cat headers.txt | sed -n -e 's/^.*Oc-Fileid: //p')", - "export URL=\"https://ocis-server:9200/app/open?app_name=FakeOffice&file_id=$FILE_ID\"", - "export URL=$(echo $URL | tr -d '[:cntrl:]')", - "curl -v -X POST \"$URL\" -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin > open.json", - "cat open.json", - "cat open.json | jq .form_parameters.access_token | tr -d '\"' > accesstoken", - "cat open.json | jq .form_parameters.access_token_ttl | tr -d '\"' > accesstokenttl", - "echo -n 'http://wopiserver:9300/wopi/files/' > wopisrc", - "cat open.json | jq .app_url | sed -n -e 's/^.*files%2F//p' | tr -d '\"' >> wopisrc", - ], - }, ] def tikaService(): diff --git a/tests/config/drone/hosting-discovery.xml b/tests/config/drone/hosting-discovery.xml index f663da91da..9dbe99100b 100644 --- a/tests/config/drone/hosting-discovery.xml +++ b/tests/config/drone/hosting-discovery.xml @@ -1,8 +1,12 @@ - - - - - - - + + + + + + + + + + + \ No newline at end of file From c0173c3ebf8b6f0791a6ce9f510152f0eae7e9e5 Mon Sep 17 00:00:00 2001 From: Prajwol Amatya Date: Fri, 23 Aug 2024 11:36:51 +0545 Subject: [PATCH 37/71] add test to open file with collaboration app --- .../apiAppProvider/appProvider.feature | 14 ++++++++++--- .../features/bootstrap/SpacesContext.php | 21 ------------------- .../acceptance/features/bootstrap/WebDav.php | 3 ++- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/tests/acceptance/features/apiAppProvider/appProvider.feature b/tests/acceptance/features/apiAppProvider/appProvider.feature index 84515fda8b..5d8f180930 100644 --- a/tests/acceptance/features/apiAppProvider/appProvider.feature +++ b/tests/acceptance/features/apiAppProvider/appProvider.feature @@ -1,10 +1,14 @@ Feature: App Provider + As a user + I want to access files with collaboration service apps + So that I can get the content of the file - Scenario: open file with Collabora + Scenario Outline: open file with .odt extension with collaboration app Given user "Alice" has been created with default attributes and without skeleton files And user "Alice" has uploaded file "filesForUpload/simple.odt" to "simple.odt" - When user "Alice" opens file "simple.odt" with collaboration service + And we save it into "FILEID" + When user "Alice" sends HTTP method "POST" to URL "" Then the HTTP status code should be "200" And the JSON data of the response should match """ @@ -18,7 +22,7 @@ Feature: App Provider "properties": { "app_url": { "type": "string", - "pattern": "^https:\\/\\/host\\.docker\\.internal:9980\\/browser\\/8ec5fda\\/cool\\.html\\?WOPISrc=https%3A%2F%2Fhost\\.docker\\.internal%3A9300%2Fwopi%2Ffiles%2F[a-fA-F0-9]{64}$" + "pattern": "^https:\\/\\/fakeoffice\\.owncloud\\.test\\/not\\/relevant\\?WOPISrc=http%3A%2F%2Fwopiserver%3A9300%2Fwopi%2Ffiles%2F[a-fA-F0-9]{64}$" }, "method": { "const": "POST" @@ -41,3 +45,7 @@ Feature: App Provider } } """ + Examples: + | endpoint | + | /app/open?file_id=<>&app_name=FakeOffice | + | /app/open?file_id=<> | diff --git a/tests/acceptance/features/bootstrap/SpacesContext.php b/tests/acceptance/features/bootstrap/SpacesContext.php index 3a224d9610..9cf94d8608 100644 --- a/tests/acceptance/features/bootstrap/SpacesContext.php +++ b/tests/acceptance/features/bootstrap/SpacesContext.php @@ -4131,25 +4131,4 @@ class SpacesContext implements Context { ) ); } - - /** - * @When user :user opens file :filename with collaboration service - * - * @param string $user - * @param string $filename - * - * @return void - */ - public function userOpensFileWithCollaborationService(string $user, string $filename): void { - $fileId = $this->getFileId($user, "Personal", $filename); - $url = $this->featureContext->getBaseUrl() . "/app/open?file_id=$fileId&app_name=FakeOffice"; - $this->featureContext->setResponse( - HttpRequestHelper::post( - $url, - $this->featureContext->getStepLineRef(), - $user, - $this->featureContext->getPasswordForUser($user) - ) - ); - } } diff --git a/tests/acceptance/features/bootstrap/WebDav.php b/tests/acceptance/features/bootstrap/WebDav.php index be822278bf..70f2ac38fb 100644 --- a/tests/acceptance/features/bootstrap/WebDav.php +++ b/tests/acceptance/features/bootstrap/WebDav.php @@ -1745,13 +1745,14 @@ trait WebDav { * * @return void */ - public function userHasUploadedAFileTo(string $user, string $source, string $destination):void { + public function userHasUploadedAFileTo(string $user, string $source, string $destination):array { $response = $this->uploadFile($user, $source, $destination, true); $this->theHTTPStatusCodeShouldBe( ["201", "204"], "HTTP status code was not 201 or 204 while trying to upload file '$source' to '$destination' for user '$user'", $response ); + return $response->getHeader('oc-fileid'); } /** From 4489ea637bcae02a42994fd64e47f2daed26d91b Mon Sep 17 00:00:00 2001 From: Prajwol Amatya Date: Fri, 23 Aug 2024 11:57:49 +0545 Subject: [PATCH 38/71] add app registry config --- .drone.star | 37 +++++++++++-------- .../apiAppProvider/appProvider.feature | 4 +- tests/config/drone/app-registry.yaml | 9 +++++ 3 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 tests/config/drone/app-registry.yaml diff --git a/.drone.star b/.drone.star index 935694f4fd..bb4287ee79 100644 --- a/.drone.star +++ b/.drone.star @@ -59,13 +59,13 @@ dirs = { # configuration config = { "cs3ApiTests": { - "skip": True, + "skip": False, }, "wopiValidatorTests": { - "skip": True, + "skip": False, }, "k6LoadTests": { - "skip": True, + "skip": False, }, "localApiTests": { "basic": { @@ -92,20 +92,19 @@ config = { "apiSharingNgLinkShareRoot", "apiActivities", ], - "skip": True, + "skip": False, }, "apiAccountsHashDifficulty": { "suites": [ "apiAccountsHashDifficulty", ], "accounts_hash_difficulty": "default", - "skip": True, }, "apiNotification": { "suites": [ "apiNotification", ], - "skip": True, + "skip": False, "emailNeeded": True, "extraEnvironment": { "EMAIL_HOST": "email", @@ -123,7 +122,7 @@ config = { "suites": [ "apiAntivirus", ], - "skip": True, + "skip": False, "antivirusNeeded": True, "extraServerEnvironment": { "ANTIVIRUS_SCANNER_TYPE": "clamav", @@ -137,14 +136,14 @@ config = { "suites": [ "apiSearchContent", ], - "skip": True, + "skip": False, "tikaNeeded": True, }, "apiOcm": { "suites": [ "apiOcm", ], - "skip": True, + "skip": False, "federationServer": True, "extraServerEnvironment": { "OCIS_ADD_RUN_SERVICES": "ocm", @@ -159,7 +158,6 @@ config = { "suites": [ "cliCommands", ], - "skip": True, }, "apiAppProvider": { "suites": [ @@ -178,17 +176,17 @@ config = { }, "apiTests": { "numberOfParts": 10, - "skip": True, + "skip": False, "skipExceptParts": [], }, "e2eTests": { "part": { - "skip": True, + "skip": False, "totalParts": 4, # divide and run all suites in parts (divide pipelines) "xsuites": ["search", "app-provider", "oidc"], # suites to skip }, "search": { - "skip": True, + "skip": False, "suites": ["search"], # suites to run "tikaNeeded": True, }, @@ -204,7 +202,7 @@ config = { "dockerReleases": { "architectures": ["arm64", "amd64"], }, - "litmus": False, + "litmus": True, "codestyle": True, } @@ -295,7 +293,15 @@ def main(ctx): licenseCheck(ctx) test_pipelines = \ + codestyle(ctx) + \ + checkGherkinLint(ctx) + \ + checkTestSuitesInExpectedFailures(ctx) + \ + buildWebCache(ctx) + \ + getGoBinForTesting(ctx) + \ buildOcisBinaryForTesting(ctx) + \ + checkStarlark() + \ + build_release_helpers + \ + testOcisAndUploadResults(ctx) + \ testPipelines(ctx) build_release_pipelines = \ @@ -309,7 +315,7 @@ def main(ctx): ), ) - pipelines = test_pipelines + pipelines = test_pipelines + build_release_pipelines if ctx.build.event == "cron": pipelines = \ @@ -2186,6 +2192,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "commands": [ "%s init --insecure true" % ocis_bin, "cat $OCIS_CONFIG_DIR/ocis.yaml", + "cp tests/config/drone/app-registry.yaml /root/.ocis/config/app-registry.yaml", ] + (wrapper_commands), "volumes": volumes, "depends_on": depends_on, diff --git a/tests/acceptance/features/apiAppProvider/appProvider.feature b/tests/acceptance/features/apiAppProvider/appProvider.feature index 5d8f180930..840b874bc4 100644 --- a/tests/acceptance/features/apiAppProvider/appProvider.feature +++ b/tests/acceptance/features/apiAppProvider/appProvider.feature @@ -8,7 +8,7 @@ Feature: App Provider Given user "Alice" has been created with default attributes and without skeleton files And user "Alice" has uploaded file "filesForUpload/simple.odt" to "simple.odt" And we save it into "FILEID" - When user "Alice" sends HTTP method "POST" to URL "" + When user "Alice" sends HTTP method "POST" to URL "" Then the HTTP status code should be "200" And the JSON data of the response should match """ @@ -46,6 +46,6 @@ Feature: App Provider } """ Examples: - | endpoint | + | app-endpoint | | /app/open?file_id=<>&app_name=FakeOffice | | /app/open?file_id=<> | diff --git a/tests/config/drone/app-registry.yaml b/tests/config/drone/app-registry.yaml new file mode 100644 index 0000000000..674829a89c --- /dev/null +++ b/tests/config/drone/app-registry.yaml @@ -0,0 +1,9 @@ +app_registry: + mimetypes: + - mime_type: application/vnd.oasis.opendocument.text + extension: odt + name: FakeOffice + description: FakeOffice text document + icon: "" + default_app: FakeOffice + allow_creation: true From 3e9a6056fa6b2baaf347d132c5ed51ad00c00deb Mon Sep 17 00:00:00 2001 From: mmattel Date: Fri, 23 Aug 2024 14:35:18 +0200 Subject: [PATCH 39/71] [docs-only] Set the correct removal version --- services/notifications/pkg/config/config.go | 2 +- services/web/pkg/config/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/notifications/pkg/config/config.go b/services/notifications/pkg/config/config.go index 3e2ed476ba..55d7d7e647 100644 --- a/services/notifications/pkg/config/config.go +++ b/services/notifications/pkg/config/config.go @@ -46,7 +46,7 @@ type SMTP struct { Password string `yaml:"smtp_password" env:"NOTIFICATIONS_SMTP_PASSWORD" desc:"Password for the SMTP host to connect to." introductionVersion:"pre5.0"` Insecure bool `yaml:"insecure" env:"NOTIFICATIONS_SMTP_INSECURE" desc:"Allow insecure connections to the SMTP server." introductionVersion:"pre5.0"` Authentication string `yaml:"smtp_authentication" env:"NOTIFICATIONS_SMTP_AUTHENTICATION" desc:"Authentication method for the SMTP communication. Possible values are 'login', 'plain', 'crammd5', 'none' or 'auto'. If set to 'auto' or unset, the authentication method is automatically negotiated with the server." introductionVersion:"pre5.0"` - Encryption string `yaml:"smtp_encryption" env:"NOTIFICATIONS_SMTP_ENCRYPTION" desc:"Encryption method for the SMTP communication. Possible values are 'starttls', 'ssl', 'ssltls', 'tls' and 'none'." introductionVersion:"pre5.0" deprecationVersion:"5.0.0" removalVersion:"7.0.0" deprecationInfo:"The NOTIFICATIONS_SMTP_ENCRYPTION values 'ssl' and 'tls' are deprecated and will be removed in the future." deprecationReplacement:"Use 'starttls' instead of 'tls' and 'ssltls' instead of 'ssl'."` + Encryption string `yaml:"smtp_encryption" env:"NOTIFICATIONS_SMTP_ENCRYPTION" desc:"Encryption method for the SMTP communication. Possible values are 'starttls', 'ssl', 'ssltls', 'tls' and 'none'." introductionVersion:"pre5.0" deprecationVersion:"5.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The NOTIFICATIONS_SMTP_ENCRYPTION values 'ssl' and 'tls' are deprecated and will be removed in the future." deprecationReplacement:"Use 'starttls' instead of 'tls' and 'ssltls' instead of 'ssl'."` } // Events combines the configuration options for the event bus. diff --git a/services/web/pkg/config/config.go b/services/web/pkg/config/config.go index 601a7853ff..c0c6f1b96c 100644 --- a/services/web/pkg/config/config.go +++ b/services/web/pkg/config/config.go @@ -31,7 +31,7 @@ type Config struct { // Asset defines the available asset configuration. type Asset struct { - DeprecatedPath string `yaml:"path" env:"WEB_ASSET_PATH" desc:"Serve ownCloud Web assets from a path on the filesystem instead of the builtin assets." introductionVersion:"pre5.0" deprecationVersion:"5.1.0" removalVersion:"7.0.0" deprecationInfo:"The WEB_ASSET_PATH is deprecated and will be removed in the future." deprecationReplacement:"Use WEB_ASSET_CORE_PATH instead."` + DeprecatedPath string `yaml:"path" env:"WEB_ASSET_PATH" desc:"Serve ownCloud Web assets from a path on the filesystem instead of the builtin assets." introductionVersion:"pre5.0" deprecationVersion:"5.1.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The WEB_ASSET_PATH is deprecated and will be removed in the future." deprecationReplacement:"Use WEB_ASSET_CORE_PATH instead."` CorePath string `yaml:"core_path" env:"WEB_ASSET_CORE_PATH" desc:"Serve ownCloud Web assets from a path on the filesystem instead of the builtin assets. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/web/assets/core" introductionVersion:"6.0.0"` ThemesPath string `yaml:"themes_path" env:"OCIS_ASSET_THEMES_PATH;WEB_ASSET_THEMES_PATH" desc:"Serve ownCloud themes from a path on the filesystem instead of the builtin assets. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/web/assets/themes" introductionVersion:"6.0.0"` AppsPath string `yaml:"apps_path" env:"WEB_ASSET_APPS_PATH" desc:"Serve ownCloud Web apps assets from a path on the filesystem instead of the builtin assets. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/web/assets/apps" introductionVersion:"6.0.0"` From 6f6e604f650ab657e59591821fdbee681abfb423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 23 Aug 2024 12:42:16 +0000 Subject: [PATCH 40/71] Automated changelog update [skip ci] --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b11169f3d6..ad93df7c9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The following sections list the changes for unreleased. * Bugfix - Set capability response `disable_self_password_change` correctly: [#9853](https://github.com/owncloud/ocis/pull/9853) * Bugfix - Activity Translations: [#9856](https://github.com/owncloud/ocis/pull/9856) * Bugfix - The user attributes `userType` and `memberOf` are readonly: [#9867](https://github.com/owncloud/ocis/pull/9867) +* Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) * Enhancement - Bump reva: [#9860](https://github.com/owncloud/ocis/pull/9860) * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) @@ -86,6 +87,13 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/9858 https://github.com/owncloud/ocis/pull/9867 +* Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) + + The libre graph API now returns OCM shares when listing driveItem permissions. + + https://github.com/owncloud/ocis/issues/9898 + https://github.com/owncloud/ocis/pull/9905 + * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) We have removed the unused store service. From 5698bf2d433452f2547eac55e7be3dc05e829bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 21 Aug 2024 11:01:05 +0200 Subject: [PATCH 41/71] use key to get specific trash item MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/use-key-to-get-specific-trash-item.md | 5 +++++ services/activitylog/pkg/service/response.go | 1 + services/clientlog/pkg/service/service.go | 1 + services/storage-users/pkg/command/trash_bin.go | 2 +- 4 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/use-key-to-get-specific-trash-item.md diff --git a/changelog/unreleased/use-key-to-get-specific-trash-item.md b/changelog/unreleased/use-key-to-get-specific-trash-item.md new file mode 100644 index 0000000000..9be2dd2bc8 --- /dev/null +++ b/changelog/unreleased/use-key-to-get-specific-trash-item.md @@ -0,0 +1,5 @@ +Bugfix: Use key to get specific trash item + +The activitylog and clientlog services now only fetch the specific trash item instead of getting all items in trash and filtering them on their side. This reduces the load on the storage users service because it no longer has to assemble a full trash listing. + +https://github.com/owncloud/ocis/pull/9879 diff --git a/services/activitylog/pkg/service/response.go b/services/activitylog/pkg/service/response.go index 154a396687..c03dc9f169 100644 --- a/services/activitylog/pkg/service/response.go +++ b/services/activitylog/pkg/service/response.go @@ -99,6 +99,7 @@ func WithTrashedResource(ref *provider.Reference, rid *provider.ResourceId) Acti resp, err := gwc.ListRecycle(ctx, &provider.ListRecycleRequest{ Ref: ref, + Key: rid.GetOpaqueId(), }) if err != nil { return err diff --git a/services/clientlog/pkg/service/service.go b/services/clientlog/pkg/service/service.go index 766d6013cd..849c78ce8e 100644 --- a/services/clientlog/pkg/service/service.go +++ b/services/clientlog/pkg/service/service.go @@ -221,6 +221,7 @@ func processShareEvent(ctx context.Context, ref *provider.Reference, gwc gateway func processItemTrashedEvent(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient, initiatorid string, itemID *provider.ResourceId) ([]string, FileEvent, error) { resp, err := gwc.ListRecycle(ctx, &provider.ListRecycleRequest{ Ref: ref, + Key: itemID.GetOpaqueId(), }) if err != nil { return nil, FileEvent{}, err diff --git a/services/storage-users/pkg/command/trash_bin.go b/services/storage-users/pkg/command/trash_bin.go index dcdb079594..f641c8b608 100644 --- a/services/storage-users/pkg/command/trash_bin.go +++ b/services/storage-users/pkg/command/trash_bin.go @@ -345,7 +345,7 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { func listRecycle(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference) (*provider.ListRecycleResponse, error) { _retrievingErrorMsg := "trash-bin items retrieving error" - res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: ""}) if err != nil { return nil, fmt.Errorf("%s %w", _retrievingErrorMsg, err) } From 23adc13156d3f34b0083baabef4525a5b39a94fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 23 Aug 2024 14:36:03 +0200 Subject: [PATCH 42/71] bump reva MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/bump-reva.md | 1 + go.mod | 2 +- go.sum | 4 +- ...ected-failures-localAPI-on-OCIS-storage.md | 18 --- .../storageprovider/storageprovider.go | 33 +++-- .../http/services/owncloud/ocdav/dav.go | 2 +- .../http/services/owncloud/ocdav/put.go | 2 - .../http/services/owncloud/ocdav/spaces.go | 25 ++-- .../http/services/owncloud/ocdav/trashbin.go | 127 +++++++++--------- .../fs/nextcloud/nextcloud_server_mock.go | 10 +- .../pkg/storage/utils/decomposedfs/recycle.go | 51 +++---- .../storage/utils/decomposedfs/tree/tree.go | 6 +- .../tusd/v2/pkg/handler/unrouted_handler.go | 20 --- vendor/modules.txt | 2 +- 14 files changed, 146 insertions(+), 157 deletions(-) diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md index ae9219216d..eb8a031538 100644 --- a/changelog/unreleased/bump-reva.md +++ b/changelog/unreleased/bump-reva.md @@ -2,4 +2,5 @@ Enhancement: Bump reva Bumps reva version +https://github.com/owncloud/ocis/pull/9879 https://github.com/owncloud/ocis/pull/9860 diff --git a/go.mod b/go.mod index a797cf5947..c654237a0c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.11.0 github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb - github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d + github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7 github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index 22327967aa..c352ef0abc 100644 --- a/go.sum +++ b/go.sum @@ -255,8 +255,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb h1:KmYZDReplv/yfwc1LNYpDcVhVujC3Pasv6WjXx1haSU= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb/go.mod h1:yyP8PRo0EZou3nSH7H4qjlzQwaydPeIRNgX50npQHpE= -github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d h1:fnb2A+ClZjlwMTPCBXQ3+NoSf7e5zBWK1XJsIaj0834= -github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d/go.mod h1:p7CHBXcg6sSqB+0JMNDfC1S7TSh9FghXkw1kTV3KcJI= +github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7 h1:q5U8sebSA3VqeLuf8Xhg1bVRxc8oJuRjQCjkl8xQPaI= +github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7/go.mod h1:p7CHBXcg6sSqB+0JMNDfC1S7TSh9FghXkw1kTV3KcJI= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md index a30aabf4b7..53b15568a5 100644 --- a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md @@ -126,30 +126,12 @@ The expected failures in this file are from features in the owncloud/ocis repo. #### [Trying to upload to a locked file gives 500](https://github.com/owncloud/ocis/issues/7638) -- [apiLocks/lockFiles.feature:330](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L330) -- [apiLocks/lockFiles.feature:331](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L331) -- [apiLocks/lockFiles.feature:332](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L332) -- [apiLocks/lockFiles.feature:333](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L333) -- [apiLocks/lockFiles.feature:334](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L334) -- [apiLocks/lockFiles.feature:335](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L335) -- [apiLocks/unlockFiles.feature:87](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L87) -- [apiLocks/unlockFiles.feature:88](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L88) -- [apiLocks/unlockFiles.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L89) -- [apiLocks/unlockFiles.feature:90](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L90) -- [apiLocks/unlockFiles.feature:91](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L91) -- [apiLocks/unlockFiles.feature:92](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/unlockFiles.feature#L92) - [apiLocks/lockFiles.feature:445](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L445) - [apiLocks/lockFiles.feature:446](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L446) - [apiLocks/lockFiles.feature:447](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L447) - [apiLocks/lockFiles.feature:448](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L448) - [apiLocks/lockFiles.feature:449](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L449) - [apiLocks/lockFiles.feature:450](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L450) -- [apiLocks/lockFiles.feature:489](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L489) -- [apiLocks/lockFiles.feature:490](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L490) -- [apiLocks/lockFiles.feature:491](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L491) -- [apiLocks/lockFiles.feature:492](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L492) -- [apiLocks/lockFiles.feature:493](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L493) -- [apiLocks/lockFiles.feature:494](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiLocks/lockFiles.feature#L494) #### [Folders can be locked and locking works partially](https://github.com/owncloud/ocis/issues/7641) diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go index 763148e8e7..872105788e 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go @@ -27,6 +27,7 @@ import ( "path" "sort" "strconv" + "strings" "time" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -436,6 +437,8 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate st = status.NewInsufficientStorage(ctx, err, "insufficient storage") case errtypes.PreconditionFailed: st = status.NewFailedPrecondition(ctx, err, "failed precondition") + case errtypes.Locked: + st = status.NewLocked(ctx, "locked") default: st = status.NewInternal(ctx, "error getting upload id: "+err.Error()) } @@ -880,8 +883,11 @@ func (s *Service) ListRecycleStream(req *provider.ListRecycleStreamRequest, ss p ctx := ss.Context() log := appctx.GetLogger(ctx) - key, itemPath := router.ShiftPath(req.Key) - items, err := s.Storage.ListRecycle(ctx, req.Ref, key, itemPath) + // if no slash is present in the key, do not pass a relative path to the storage + // when a path is passed to the storage, it will list the contents of the directory + key, relativePath := splitKeyAndPath(req.GetKey()) + items, err := s.Storage.ListRecycle(ctx, req.Ref, key, relativePath) + if err != nil { var st *rpc.Status switch err.(type) { @@ -924,8 +930,10 @@ func (s *Service) ListRecycleStream(req *provider.ListRecycleStreamRequest, ss p } func (s *Service) ListRecycle(ctx context.Context, req *provider.ListRecycleRequest) (*provider.ListRecycleResponse, error) { - key, itemPath := router.ShiftPath(req.Key) - items, err := s.Storage.ListRecycle(ctx, req.Ref, key, itemPath) + // if no slash is present in the key, do not pass a relative path to the storage + // when a path is passed to the storage, it will list the contents of the directory + key, relativePath := splitKeyAndPath(req.GetKey()) + items, err := s.Storage.ListRecycle(ctx, req.Ref, key, relativePath) if err != nil { var st *rpc.Status switch err.(type) { @@ -962,8 +970,8 @@ func (s *Service) RestoreRecycleItem(ctx context.Context, req *provider.RestoreR ctx = ctxpkg.ContextSetLockID(ctx, req.LockId) // TODO(labkode): CRITICAL: fill recycle info with storage provider. - key, itemPath := router.ShiftPath(req.Key) - err := s.Storage.RestoreRecycleItem(ctx, req.Ref, key, itemPath, req.RestoreRef) + key, relativePath := splitKeyAndPath(req.GetKey()) + err := s.Storage.RestoreRecycleItem(ctx, req.Ref, key, relativePath, req.RestoreRef) res := &provider.RestoreRecycleItemResponse{ Status: status.NewStatusFromErrType(ctx, "restore recycle item", err), @@ -980,9 +988,9 @@ func (s *Service) PurgeRecycle(ctx context.Context, req *provider.PurgeRecycleRe } // if a key was sent as opaque id purge only that item - key, itemPath := router.ShiftPath(req.Key) + key, relativePath := splitKeyAndPath(req.GetKey()) if key != "" { - if err := s.Storage.PurgeRecycleItem(ctx, req.Ref, key, itemPath); err != nil { + if err := s.Storage.PurgeRecycleItem(ctx, req.Ref, key, relativePath); err != nil { st := status.NewStatusFromErrType(ctx, "error purging recycle item", err) appctx.GetLogger(ctx). Error(). @@ -1313,3 +1321,12 @@ func canLockPublicShare(ctx context.Context) bool { psr := utils.ReadPlainFromOpaque(u.Opaque, "public-share-role") return psr == "" || psr == conversions.RoleEditor } + +// splitKeyAndPath splits a key into a root and a relative path +func splitKeyAndPath(key string) (string, string) { + root, relativePath := router.ShiftPath(key) + if relativePath == "/" && !strings.HasSuffix(key, "/") { + relativePath = "" + } + return root, relativePath +} diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/dav.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/dav.go index ba82e15439..7aff215a34 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/dav.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/dav.go @@ -291,7 +291,7 @@ func (h *DavHandler) Handler(s *svc) http.Handler { http.Redirect(w, r, rUrl, http.StatusTemporaryRedirect) return } - log.Debug().Str("token", token).Interface("status", res.Status).Msg("resource id not found") + log.Debug().Str("token", token).Interface("status", psRes.Status).Msg("resource id not found") w.WriteHeader(http.StatusNotFound) return } diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go index b89e835d8f..e2955b7366 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go @@ -325,8 +325,6 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ w.WriteHeader(http.StatusPreconditionFailed) case rpc.Code_CODE_FAILED_PRECONDITION: w.WriteHeader(http.StatusConflict) - case rpc.Code_CODE_NOT_FOUND: - w.WriteHeader(http.StatusNotFound) default: errors.HandleErrorStatus(&log, w, uRes.Status) } diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/spaces.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/spaces.go index 37797d60ad..2439a98251 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/spaces.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/spaces.go @@ -21,6 +21,7 @@ package ocdav import ( "net/http" "path" + "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/config" @@ -132,8 +133,7 @@ func (h *SpacesHandler) handleSpacesTrashbin(w http.ResponseWriter, r *http.Requ ctx := r.Context() log := appctx.GetLogger(ctx) - var spaceID string - spaceID, r.URL.Path = router.ShiftPath(r.URL.Path) + spaceID, key := splitSpaceAndKey(r.URL.Path) if spaceID == "" { // listing is disabled, no auth will change that w.WriteHeader(http.StatusMethodNotAllowed) @@ -146,12 +146,9 @@ func (h *SpacesHandler) handleSpacesTrashbin(w http.ResponseWriter, r *http.Requ return } - var key string - key, r.URL.Path = router.ShiftPath(r.URL.Path) - switch r.Method { case MethodPropfind: - trashbinHandler.listTrashbin(w, r, s, &ref, path.Join(_trashbinPath, spaceID), key, r.URL.Path) + trashbinHandler.listTrashbin(w, r, s, &ref, path.Join(_trashbinPath, spaceID), key) case MethodMove: if key == "" { http.Error(w, "501 Not implemented", http.StatusNotImplemented) @@ -167,15 +164,25 @@ func (h *SpacesHandler) handleSpacesTrashbin(w http.ResponseWriter, r *http.Requ w.WriteHeader(http.StatusBadRequest) return } - log.Debug().Str("key", key).Str("path", r.URL.Path).Str("dst", dst).Msg("spaces restore") + log.Debug().Str("key", key).Str("dst", dst).Msg("spaces restore") dstRef := proto.Clone(&ref).(*provider.Reference) dstRef.Path = utils.MakeRelativePath(dst) - trashbinHandler.restore(w, r, s, &ref, dstRef, key, r.URL.Path) + trashbinHandler.restore(w, r, s, &ref, dstRef, key) case http.MethodDelete: - trashbinHandler.delete(w, r, s, &ref, key, r.URL.Path) + trashbinHandler.delete(w, r, s, &ref, key) default: http.Error(w, "501 Not implemented", http.StatusNotImplemented) } } + +func splitSpaceAndKey(p string) (space, key string) { + p = strings.TrimPrefix(p, "/") + parts := strings.SplitN(p, "/", 2) + space = parts[0] + if len(parts) > 1 { + key = parts[1] + } + return +} diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/trashbin.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/trashbin.go index ef742c54df..07fa6c3743 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/trashbin.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/trashbin.go @@ -43,7 +43,6 @@ import ( "github.com/cs3org/reva/v2/pkg/appctx" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" rstatus "github.com/cs3org/reva/v2/pkg/rgrpc/status" - "github.com/cs3org/reva/v2/pkg/rhttp/router" "github.com/cs3org/reva/v2/pkg/utils" semconv "go.opentelemetry.io/otel/semconv/v1.20.0" ) @@ -74,7 +73,7 @@ func (h *TrashbinHandler) Handler(s *svc) http.Handler { } var username string - username, r.URL.Path = router.ShiftPath(r.URL.Path) + username, r.URL.Path = splitSpaceAndKey(r.URL.Path) if username == "" { // listing is disabled, no auth will change that w.WriteHeader(http.StatusMethodNotAllowed) @@ -131,13 +130,12 @@ func (h *TrashbinHandler) Handler(s *svc) http.Handler { } ref := spacelookup.MakeRelativeReference(space, ".", false) - // key will be a base64 encoded cs3 path, it uniquely identifies a trash item & storage - var key string - key, r.URL.Path = router.ShiftPath(r.URL.Path) + // key will be a base64 encoded cs3 path, it uniquely identifies a trash item with an opaque id and an optional path + key := r.URL.Path switch r.Method { case MethodPropfind: - h.listTrashbin(w, r, s, ref, user.Username, key, r.URL.Path) + h.listTrashbin(w, r, s, ref, user.Username, key) case MethodMove: if key == "" { http.Error(w, "501 Not implemented", http.StatusNotImplemented) @@ -172,50 +170,55 @@ func (h *TrashbinHandler) Handler(s *svc) http.Handler { dstRef := spacelookup.MakeRelativeReference(space, p, false) log.Debug().Str("key", key).Str("dst", dst).Msg("restore") - h.restore(w, r, s, ref, dstRef, key, r.URL.Path) + h.restore(w, r, s, ref, dstRef, key) case http.MethodDelete: - h.delete(w, r, s, ref, key, r.URL.Path) + h.delete(w, r, s, ref, key) default: http.Error(w, "501 Not implemented", http.StatusNotImplemented) } }) } -func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s *svc, ref *provider.Reference, refBase, key, itemPath string) { +func (h *TrashbinHandler) getDepth(r *http.Request) (net.Depth, error) { + dh := r.Header.Get(net.HeaderDepth) + depth, err := net.ParseDepth(dh) + if err != nil || depth == net.DepthInfinity && !h.allowPropfindDepthInfinitiy { + return "", errors.ErrInvalidDepth + } + return depth, nil +} + +func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s *svc, ref *provider.Reference, refBase, key string) { ctx, span := appctx.GetTracerProvider(r.Context()).Tracer(tracerName).Start(r.Context(), "list_trashbin") defer span.End() sublog := appctx.GetLogger(ctx).With().Logger() - dh := r.Header.Get(net.HeaderDepth) - depth, err := net.ParseDepth(dh) + depth, err := h.getDepth(r) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, "Invalid Depth header value") span.SetAttributes(semconv.HTTPStatusCodeKey.Int(http.StatusBadRequest)) - sublog.Debug().Str("depth", dh).Msg(err.Error()) + sublog.Debug().Str("depth", r.Header.Get(net.HeaderDepth)).Msg(err.Error()) w.WriteHeader(http.StatusBadRequest) - m := fmt.Sprintf("Invalid Depth header value: %v", dh) + m := fmt.Sprintf("Invalid Depth header value: %v", r.Header.Get(net.HeaderDepth)) b, err := errors.Marshal(http.StatusBadRequest, m, "", "") errors.HandleWebdavError(&sublog, w, b, err) return } - if depth == net.DepthInfinity && !h.allowPropfindDepthInfinitiy { - span.RecordError(errors.ErrInvalidDepth) - span.SetStatus(codes.Error, "DEPTH: infinity is not supported") - span.SetAttributes(semconv.HTTPStatusCodeKey.Int(http.StatusBadRequest)) - sublog.Debug().Str("depth", dh).Msg(errors.ErrInvalidDepth.Error()) - w.WriteHeader(http.StatusBadRequest) - m := fmt.Sprintf("Invalid Depth header value: %v", dh) - b, err := errors.Marshal(http.StatusBadRequest, m, "", "") - errors.HandleWebdavError(&sublog, w, b, err) + pf, status, err := propfind.ReadPropfind(r.Body) + if err != nil { + sublog.Debug().Err(err).Msg("error reading propfind request") + w.WriteHeader(status) return } - if depth == net.DepthZero { - rootHref := path.Join(refBase, key, itemPath) - propRes, err := h.formatTrashPropfind(ctx, s, ref.ResourceId.SpaceId, refBase, rootHref, nil, nil) + if key == "" && depth == net.DepthZero { + // we are listing the trash root, but without children + // so we just fake a root element without actually querying the gateway + rootHref := path.Join(refBase, key) + propRes, err := h.formatTrashPropfind(ctx, s, ref.ResourceId.SpaceId, refBase, rootHref, &pf, nil, true) if err != nil { sublog.Error().Err(err).Msg("error formatting propfind") w.WriteHeader(http.StatusInternalServerError) @@ -232,11 +235,9 @@ func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s return } - pf, status, err := propfind.ReadPropfind(r.Body) - if err != nil { - sublog.Debug().Err(err).Msg("error reading propfind request") - w.WriteHeader(status) - return + if depth == net.DepthOne && key != "" && !strings.HasSuffix(key, "/") { + // when a key is provided and the depth is 1 we need to append a / to the key to list the children + key += "/" } client, err := s.gatewaySelector.Next() @@ -246,7 +247,7 @@ func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s return } // ask gateway for recycle items - getRecycleRes, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: ref, Key: path.Join(key, itemPath)}) + getRecycleRes, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: ref, Key: key}) if err != nil { sublog.Error().Err(err).Msg("error calling ListRecycle") w.WriteHeader(http.StatusInternalServerError) @@ -270,7 +271,7 @@ func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s for i := len(items) - 1; i >= 0; i-- { // for i := range res.Infos { if items[i].Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { - stack = append(stack, items[i].Key) + stack = append(stack, items[i].Key+"/") // fetch children of the item } } @@ -304,8 +305,8 @@ func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s } } - rootHref := path.Join(refBase, key, itemPath) - propRes, err := h.formatTrashPropfind(ctx, s, ref.ResourceId.SpaceId, refBase, rootHref, &pf, items) + rootHref := path.Join(refBase, key) + propRes, err := h.formatTrashPropfind(ctx, s, ref.ResourceId.SpaceId, refBase, rootHref, &pf, items, depth != net.DepthZero) if err != nil { sublog.Error().Err(err).Msg("error formatting propfind") w.WriteHeader(http.StatusInternalServerError) @@ -321,29 +322,30 @@ func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s } } -func (h *TrashbinHandler) formatTrashPropfind(ctx context.Context, s *svc, spaceID, refBase, rootHref string, pf *propfind.XML, items []*provider.RecycleItem) ([]byte, error) { +func (h *TrashbinHandler) formatTrashPropfind(ctx context.Context, s *svc, spaceID, refBase, rootHref string, pf *propfind.XML, items []*provider.RecycleItem, fakeRoot bool) ([]byte, error) { responses := make([]*propfind.ResponseXML, 0, len(items)+1) - // add trashbin dir . entry - responses = append(responses, &propfind.ResponseXML{ - Href: net.EncodePath(path.Join(ctx.Value(net.CtxKeyBaseURI).(string), rootHref) + "/"), // url encode response.Href TODO - Propstat: []propfind.PropstatXML{ - { - Status: "HTTP/1.1 200 OK", - Prop: []prop.PropertyXML{ - prop.Raw("d:resourcetype", ""), + if fakeRoot { + responses = append(responses, &propfind.ResponseXML{ + Href: net.EncodePath(path.Join(ctx.Value(net.CtxKeyBaseURI).(string), rootHref) + "/"), // url encode response.Href TODO + Propstat: []propfind.PropstatXML{ + { + Status: "HTTP/1.1 200 OK", + Prop: []prop.PropertyXML{ + prop.Raw("d:resourcetype", ""), + }, + }, + { + Status: "HTTP/1.1 404 Not Found", + Prop: []prop.PropertyXML{ + prop.NotFound("oc:trashbin-original-filename"), + prop.NotFound("oc:trashbin-original-location"), + prop.NotFound("oc:trashbin-delete-datetime"), + prop.NotFound("d:getcontentlength"), + }, }, }, - { - Status: "HTTP/1.1 404 Not Found", - Prop: []prop.PropertyXML{ - prop.NotFound("oc:trashbin-original-filename"), - prop.NotFound("oc:trashbin-original-location"), - prop.NotFound("oc:trashbin-delete-datetime"), - prop.NotFound("d:getcontentlength"), - }, - }, - }, - }) + }) + } for i := range items { res, err := h.itemToPropResponse(ctx, s, spaceID, refBase, pf, items[i]) @@ -401,7 +403,7 @@ func (h *TrashbinHandler) itemToPropResponse(ctx context.Context, s *svc, spaceI propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:trashbin-delete-datetime", dTime)) if item.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { propstatOK.Prop = append(propstatOK.Prop, prop.Raw("d:resourcetype", "")) - // TODO(jfd): decide if we can and want to list oc:size for folders + propstatOK.Prop = append(propstatOK.Prop, prop.Raw("oc:size", size)) } else { propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:resourcetype", ""), @@ -426,7 +428,7 @@ func (h *TrashbinHandler) itemToPropResponse(ctx context.Context, s *svc, spaceI switch pf.Prop[i].Local { case "oc:size": if item.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { - propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:getcontentlength", size)) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:size", size)) } else { propstatNotFound.Prop = append(propstatNotFound.Prop, prop.NotFound("oc:size")) } @@ -480,7 +482,7 @@ func (h *TrashbinHandler) itemToPropResponse(ctx context.Context, s *svc, spaceI return &response, nil } -func (h *TrashbinHandler) restore(w http.ResponseWriter, r *http.Request, s *svc, ref, dst *provider.Reference, key, itemPath string) { +func (h *TrashbinHandler) restore(w http.ResponseWriter, r *http.Request, s *svc, ref, dst *provider.Reference, key string) { ctx, span := appctx.GetTracerProvider(r.Context()).Tracer(tracerName).Start(r.Context(), "restore") defer span.End() @@ -566,7 +568,7 @@ func (h *TrashbinHandler) restore(w http.ResponseWriter, r *http.Request, s *svc req := &provider.RestoreRecycleItemRequest{ Ref: ref, - Key: path.Join(key, itemPath), + Key: key, RestoreRef: dst, } @@ -608,16 +610,15 @@ func (h *TrashbinHandler) restore(w http.ResponseWriter, r *http.Request, s *svc } // delete has only a key -func (h *TrashbinHandler) delete(w http.ResponseWriter, r *http.Request, s *svc, ref *provider.Reference, key, itemPath string) { +func (h *TrashbinHandler) delete(w http.ResponseWriter, r *http.Request, s *svc, ref *provider.Reference, key string) { ctx, span := appctx.GetTracerProvider(r.Context()).Tracer(tracerName).Start(r.Context(), "erase") defer span.End() - sublog := appctx.GetLogger(ctx).With().Interface("reference", ref).Str("key", key).Str("item_path", itemPath).Logger() + sublog := appctx.GetLogger(ctx).With().Interface("reference", ref).Str("key", key).Logger() - trashPath := path.Join(key, itemPath) req := &provider.PurgeRecycleRequest{ Ref: ref, - Key: trashPath, + Key: key, } client, err := s.gatewaySelector.Next() @@ -638,7 +639,7 @@ func (h *TrashbinHandler) delete(w http.ResponseWriter, r *http.Request, s *svc, case rpc.Code_CODE_NOT_FOUND: sublog.Debug().Interface("status", res.Status).Msg("resource not found") w.WriteHeader(http.StatusConflict) - m := fmt.Sprintf("path %s not found", trashPath) + m := fmt.Sprintf("key %s not found", key) b, err := errors.Marshal(http.StatusConflict, m, "", "") errors.HandleWebdavError(&sublog, w, b, err) case rpc.Code_CODE_PERMISSION_DENIED: diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go index 4998c5a5b7..97507df71b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go @@ -129,8 +129,8 @@ var responses = map[string]Response{ `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListGrants {"path":"/subdir"} GRANT-UPDATED`: {200, `[{"grantee":{"type":1,"Id":{"UserId":{"idp":"some-idp","opaque_id":"some-opaque-id","type":1}}},"permissions":{"add_grant":true,"create_container":true,"delete":true,"get_path":true,"get_quota":true,"initiate_file_download":true,"initiate_file_upload":true,"list_grants":true,"list_container":true,"list_file_versions":true,"list_recycle":true,"move":true,"remove_grant":true,"purge_recycle":true,"restore_file_version":true,"restore_recycle_item":true,"stat":true,"update_grant":true,"deny_grant":true}}]`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListGrants {"path":"/subdir"} GRANT-REMOVED`: {200, `[]`, serverStateEmpty}, - `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListRecycle {"key":"","path":"/"} EMPTY`: {200, `[]`, serverStateEmpty}, - `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListRecycle {"key":"","path":"/"} RECYCLE`: {200, `[{"opaque":{},"key":"some-deleted-version","ref":{"resource_id":{},"path":"/subdir"},"size":12345,"deletion_time":{"seconds":1234567890}}]`, serverStateRecycle}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListRecycle {"key":"","path":""} EMPTY`: {200, `[]`, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListRecycle {"key":"","path":""} RECYCLE`: {200, `[{"opaque":{},"key":"some-deleted-version","ref":{"resource_id":{},"path":"/subdir"},"size":12345,"deletion_time":{"seconds":1234567890}}]`, serverStateRecycle}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListRevisions {"path":"/versionedFile"} EMPTY`: {200, `[{"opaque":{"map":{"some":{"value":"ZGF0YQ=="}}},"key":"version-12","size":1,"mtime":1234567890,"etag":"deadb00f"}]`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/ListRevisions {"path":"/versionedFile"} FILE-RESTORED`: {200, `[{"opaque":{"map":{"some":{"value":"ZGF0YQ=="}}},"key":"version-12","size":1,"mtime":1234567890,"etag":"deadb00f"},{"opaque":{"map":{"different":{"value":"c3R1ZmY="}}},"key":"asdf","size":2,"mtime":1234567890,"etag":"deadbeef"}]`, serverStateFileRestored}, @@ -139,9 +139,9 @@ var responses = map[string]Response{ `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RemoveGrant {"path":"/subdir"} GRANT-ADDED`: {200, ``, serverStateGrantRemoved}, - `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RestoreRecycleItem null`: {200, ``, serverStateSubdir}, - `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RestoreRecycleItem {"key":"some-deleted-version","path":"/","restoreRef":{"path":"/subdirRestored"}}`: {200, ``, serverStateFileRestored}, - `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RestoreRecycleItem {"key":"some-deleted-version","path":"/","restoreRef":null}`: {200, ``, serverStateFileRestored}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RestoreRecycleItem null`: {200, ``, serverStateSubdir}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RestoreRecycleItem {"key":"some-deleted-version","path":"","restoreRef":{"path":"/subdirRestored"}}`: {200, ``, serverStateFileRestored}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RestoreRecycleItem {"key":"some-deleted-version","path":"","restoreRef":null}`: {200, ``, serverStateFileRestored}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/RestoreRevision {"ref":{"path":"/versionedFile"},"key":"version-12"}`: {200, ``, serverStateFileRestored}, diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go index aa0ead067b..fdc99b806b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go @@ -23,7 +23,6 @@ import ( iofs "io/fs" "os" "path/filepath" - "strconv" "strings" "time" @@ -55,6 +54,9 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference if ref == nil || ref.ResourceId == nil || ref.ResourceId.OpaqueId == "" { return nil, errtypes.BadRequest("spaceid required") } + if key == "" && relativePath != "" { + return nil, errtypes.BadRequest("key is required when navigating with a path") + } spaceID := ref.ResourceId.OpaqueId sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Str("key", key).Str("relative_path", relativePath).Logger() @@ -75,7 +77,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference return nil, errtypes.NotFound(key) } - if key == "" && relativePath == "/" { + if key == "" && relativePath == "" { return fs.listTrashRoot(ctx, spaceID) } @@ -113,16 +115,25 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference sublog.Error().Err(err).Msg("could not parse time format, ignoring") } - nodeType := fs.lu.TypeFromPath(ctx, originalPath) - if nodeType != provider.ResourceType_RESOURCE_TYPE_CONTAINER { + var size int64 + if relativePath == "" { // this is the case when we want to directly list a file in the trashbin - blobsize, err := strconv.ParseInt(string(attrs[prefixes.BlobsizeAttr]), 10, 64) - if err != nil { - return items, err + nodeType := fs.lu.TypeFromPath(ctx, originalPath) + switch nodeType { + case provider.ResourceType_RESOURCE_TYPE_FILE: + size, err = fs.lu.ReadBlobSizeAttr(ctx, originalPath) + if err != nil { + return items, err + } + case provider.ResourceType_RESOURCE_TYPE_CONTAINER: + size, err = fs.lu.MetadataBackend().GetInt64(ctx, originalPath, prefixes.TreesizeAttr) + if err != nil { + return items, err + } } item := &provider.RecycleItem{ - Type: nodeType, - Size: uint64(blobsize), + Type: fs.lu.TypeFromPath(ctx, originalPath), + Size: uint64(size), Key: filepath.Join(key, relativePath), DeletionTime: deletionTime, Ref: &provider.Reference{ @@ -134,9 +145,6 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference } // we have to read the names and stat the path to follow the symlinks - if err != nil { - return nil, err - } childrenPath := filepath.Join(originalPath, relativePath) childrenDir, err := os.Open(childrenPath) if err != nil { @@ -154,9 +162,10 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference continue } - size := int64(0) + // reset size + size = 0 - nodeType = fs.lu.TypeFromPath(ctx, resolvedChildPath) + nodeType := fs.lu.TypeFromPath(ctx, resolvedChildPath) switch nodeType { case provider.ResourceType_RESOURCE_TYPE_FILE: size, err = fs.lu.ReadBlobSizeAttr(ctx, resolvedChildPath) @@ -165,12 +174,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference continue } case provider.ResourceType_RESOURCE_TYPE_CONTAINER: - attr, err := fs.lu.MetadataBackend().Get(ctx, resolvedChildPath, prefixes.TreesizeAttr) - if err != nil { - sublog.Error().Err(err).Str("name", name).Msg("invalid tree size, skipping") - continue - } - size, err = strconv.ParseInt(string(attr), 10, 64) + size, err = fs.lu.MetadataBackend().GetInt64(ctx, resolvedChildPath, prefixes.TreesizeAttr) if err != nil { sublog.Error().Err(err).Str("name", name).Msg("invalid tree size, skipping") continue @@ -217,7 +221,7 @@ func readTrashLink(path string) (string, string, string, error) { func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*provider.RecycleItem, error) { log := appctx.GetLogger(ctx) trashRoot := fs.getRecycleRoot(spaceID) - + items := []*provider.RecycleItem{} subTrees, err := filepath.Glob(trashRoot + "/*") if err != nil { return nil, err @@ -256,6 +260,7 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p } for _, itemPath := range matches { + // TODO can we encode this in the path instead of reading the link? nodePath, nodeID, timeSuffix, err := readTrashLink(itemPath) if err != nil { log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Msg("error reading trash link, skipping") @@ -300,6 +305,7 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p } else { log.Error().Str("trashRoot", trashRoot).Str("item", itemPath).Str("spaceid", spaceID).Str("nodeid", nodeID).Str("dtime", timeSuffix).Msg("could not read origin path") } + select { case results <- item: case <-ctx.Done(): @@ -318,7 +324,6 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p }() // Collect results - items := []*provider.RecycleItem{} for ri := range results { items = append(items, ri) } @@ -414,7 +419,7 @@ func (fs *Decomposedfs) EmptyRecycle(ctx context.Context, ref *provider.Referenc return errtypes.BadRequest("spaceid must be set") } - items, err := fs.ListRecycle(ctx, ref, "", "/") + items, err := fs.ListRecycle(ctx, ref, "", "") if err != nil { return err } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go index dd68c1b550..de070c874f 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go @@ -584,10 +584,8 @@ func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPa attrs := node.Attributes{} attrs.SetString(prefixes.NameAttr, targetNode.Name) - if trashPath != "" { - // set ParentidAttr to restorePath's node parent id - attrs.SetString(prefixes.ParentidAttr, targetNode.ParentID) - } + // set ParentidAttr to restorePath's node parent id + attrs.SetString(prefixes.ParentidAttr, targetNode.ParentID) if err = recycleNode.SetXattrsWithContext(ctx, attrs, true); err != nil { return errors.Wrap(err, "Decomposedfs: could not update recycle node") diff --git a/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go b/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go index 385bf1c3f1..73be46437e 100644 --- a/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go +++ b/vendor/github.com/tus/tusd/v2/pkg/handler/unrouted_handler.go @@ -199,7 +199,6 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler { if origin := r.Header.Get("Origin"); !cors.Disable && origin != "" { originIsAllowed := cors.AllowOrigin.MatchString(origin) if !originIsAllowed { - fmt.Println("ORIGIN IS NOT ALLOWED", origin) handler.sendError(c, ErrOriginNotAllowed) return } @@ -692,14 +691,12 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) // PatchFile adds a chunk to an upload. This operation is only allowed // if enough space in the upload is left. func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request) { - fmt.Println("PATCH FILE") c := handler.getContext(w, r) isTusV1 := !handler.usesIETFDraft(r) // Check for presence of application/offset+octet-stream if isTusV1 && r.Header.Get("Content-Type") != "application/offset+octet-stream" { - fmt.Println("WRONG CONTENT TYPE") handler.sendError(c, ErrInvalidContentType) return } @@ -707,14 +704,12 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request // Check for presence of a valid Upload-Offset Header offset, err := strconv.ParseInt(r.Header.Get("Upload-Offset"), 10, 64) if err != nil || offset < 0 { - fmt.Println("WRONG OFFSET") handler.sendError(c, ErrInvalidOffset) return } id, err := extractIDFromPath(r.URL.Path) if err != nil { - fmt.Println("WRONG ID") handler.sendError(c, err) return } @@ -723,7 +718,6 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request if handler.composer.UsesLocker { lock, err := handler.lockUpload(c, id) if err != nil { - fmt.Println("WRONG LOCK") handler.sendError(c, err) return } @@ -733,27 +727,23 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request upload, err := handler.composer.Core.GetUpload(c, id) if err != nil { - fmt.Println("WRONG UPLOAD") handler.sendError(c, err) return } info, err := upload.GetInfo(c) if err != nil { - fmt.Println("WRONG INFO") handler.sendError(c, err) return } // Modifying a final upload is not allowed if info.IsFinal { - fmt.Println("WRONG FINAL") handler.sendError(c, ErrModifyFinal) return } if offset != info.Offset { - fmt.Println("WRONG INFO OFFSET") handler.sendError(c, ErrMismatchOffset) return } @@ -770,32 +760,27 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request // Do not proxy the call to the data store if the upload is already completed if !info.SizeIsDeferred && info.Offset == info.Size { resp.Header["Upload-Offset"] = strconv.FormatInt(offset, 10) - fmt.Println("UPLOAD ALREADY COMPLETED") handler.sendResp(c, resp) return } if r.Header.Get("Upload-Length") != "" { if !handler.composer.UsesLengthDeferrer { - fmt.Println("UPLOAD LENGTH DEFERRER") handler.sendError(c, ErrNotImplemented) return } if !info.SizeIsDeferred { - fmt.Println("UPLOAD LENGTH NOT DEFERED") handler.sendError(c, ErrInvalidUploadLength) return } uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) if err != nil || uploadLength < 0 || uploadLength < info.Offset || (handler.config.MaxSize > 0 && uploadLength > handler.config.MaxSize) { - fmt.Println("UPLOAD LENGTH INVALID") handler.sendError(c, ErrInvalidUploadLength) return } lengthDeclarableUpload := handler.composer.LengthDeferrer.AsLengthDeclarableUpload(upload) if err := lengthDeclarableUpload.DeclareLength(c, uploadLength); err != nil { - fmt.Println("UPLOAD LENGTH DECLARED") handler.sendError(c, err) return } @@ -806,7 +791,6 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request resp, err = handler.writeChunk(c, resp, upload, info) if err != nil { - fmt.Println("CANT WRITE CHUNK") handler.sendError(c, err) return } @@ -815,7 +799,6 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request if willCompleteUpload && info.SizeIsDeferred { info, err = upload.GetInfo(c) if err != nil { - fmt.Println("CANT GET INFO") handler.sendError(c, err) return } @@ -824,7 +807,6 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request lengthDeclarableUpload := handler.composer.LengthDeferrer.AsLengthDeclarableUpload(upload) if err := lengthDeclarableUpload.DeclareLength(c, uploadLength); err != nil { - fmt.Println("CANT UPLOAD LENGTH") handler.sendError(c, err) return } @@ -834,14 +816,12 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request resp, err = handler.finishUploadIfComplete(c, resp, upload, info) if err != nil { - fmt.Println("CANT COMPLETE") handler.sendError(c, err) return } } handler.sendResp(c, resp) - fmt.Println("PATCH COMPLETE") } // writeChunk reads the body from the requests r and appends it to the upload diff --git a/vendor/modules.txt b/vendor/modules.txt index 54638f2bb1..bac8f587e4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -367,7 +367,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.23.1-0.20240823074930-ff4b71b50b7d +# github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7 ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime From 94dd32972b6cea758fe007bb18542dc34606a0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 23 Aug 2024 16:29:24 +0000 Subject: [PATCH 43/71] Automated changelog update [skip ci] --- CHANGELOG.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad93df7c9e..ea4cc2054f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,9 +56,10 @@ The following sections list the changes for unreleased. * Bugfix - Set capability response `disable_self_password_change` correctly: [#9853](https://github.com/owncloud/ocis/pull/9853) * Bugfix - Activity Translations: [#9856](https://github.com/owncloud/ocis/pull/9856) * Bugfix - The user attributes `userType` and `memberOf` are readonly: [#9867](https://github.com/owncloud/ocis/pull/9867) +* Bugfix - Use key to get specific trash item: [#9879](https://github.com/owncloud/ocis/pull/9879) * Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) -* Enhancement - Bump reva: [#9860](https://github.com/owncloud/ocis/pull/9860) +* Enhancement - Bump reva: [#9879](https://github.com/owncloud/ocis/pull/9879) * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) * Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) * Enhancement - Graph translation path: [#9902](https://github.com/owncloud/ocis/pull/9902) @@ -87,6 +88,15 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/9858 https://github.com/owncloud/ocis/pull/9867 +* Bugfix - Use key to get specific trash item: [#9879](https://github.com/owncloud/ocis/pull/9879) + + The activitylog and clientlog services now only fetch the specific trash item + instead of getting all items in trash and filtering them on their side. This + reduces the load on the storage users service because it no longer has to + assemble a full trash listing. + + https://github.com/owncloud/ocis/pull/9879 + * Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) The libre graph API now returns OCM shares when listing driveItem permissions. @@ -101,10 +111,11 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/1357 https://github.com/owncloud/ocis/pull/9890 -* Enhancement - Bump reva: [#9860](https://github.com/owncloud/ocis/pull/9860) +* Enhancement - Bump reva: [#9879](https://github.com/owncloud/ocis/pull/9879) Bumps reva version + https://github.com/owncloud/ocis/pull/9879 https://github.com/owncloud/ocis/pull/9860 * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) From 84e87cd722eaaa83b65b9992254ab2cf8d4dda0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 27 Jun 2024 16:53:00 +0200 Subject: [PATCH 44/71] set the configured protocol transport for service metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/set-service-transport.md | 6 ++++++ ocis-pkg/registry/service.go | 21 ++++++++++++------- services/app-provider/pkg/command/server.go | 2 +- services/app-registry/pkg/command/server.go | 2 +- services/auth-app/pkg/command/server.go | 2 +- services/auth-basic/pkg/command/server.go | 2 +- services/auth-bearer/pkg/command/server.go | 2 +- services/auth-machine/pkg/command/server.go | 2 +- services/auth-service/pkg/command/server.go | 2 +- .../pkg/config/defaults/defaultconfig.go | 1 + services/collaboration/pkg/config/grpc.go | 4 +++- .../collaboration/pkg/helpers/registration.go | 2 +- services/gateway/pkg/command/server.go | 2 +- .../graph/pkg/service/v0/graph_suite_test.go | 2 +- services/groups/pkg/command/server.go | 2 +- .../pkg/service/notification_suite_test.go | 2 +- services/ocm/pkg/command/server.go | 2 +- .../search/pkg/search/search_suite_test.go | 2 +- services/sharing/pkg/command/server.go | 2 +- .../storage-publiclink/pkg/command/server.go | 2 +- services/storage-shares/pkg/command/server.go | 2 +- services/storage-system/pkg/command/server.go | 2 +- services/storage-users/pkg/command/server.go | 2 +- .../storage-users/pkg/task/task_suite_test.go | 2 +- .../userlog/pkg/service/service_suit_test.go | 2 +- services/users/pkg/command/server.go | 2 +- 26 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 changelog/unreleased/set-service-transport.md diff --git a/changelog/unreleased/set-service-transport.md b/changelog/unreleased/set-service-transport.md new file mode 100644 index 0000000000..48b6346c37 --- /dev/null +++ b/changelog/unreleased/set-service-transport.md @@ -0,0 +1,6 @@ +Enhancement: We now set the configured protocol transport for service metadata + +This allows using `dns` or `unix` as the grpc protocol for services. Requires reva changes to have an effect + +https://github.com/owncloud/ocis/pull/9490 +https://github.com/cs3org/reva/pull/4744 \ No newline at end of file diff --git a/ocis-pkg/registry/service.go b/ocis-pkg/registry/service.go index aebccc7520..b6f84c4442 100644 --- a/ocis-pkg/registry/service.go +++ b/ocis-pkg/registry/service.go @@ -8,10 +8,10 @@ import ( mRegistry "go-micro.dev/v4/registry" "go-micro.dev/v4/server" - "go-micro.dev/v4/util/addr" + mAddr "go-micro.dev/v4/util/addr" ) -func BuildGRPCService(serviceID, address string, version string) *mRegistry.Service { +func BuildGRPCService(serviceID, transport, address, version string) *mRegistry.Service { var host string var port int @@ -23,20 +23,25 @@ func BuildGRPCService(serviceID, address string, version string) *mRegistry.Serv host = parts[0] } - addr, err := addr.Extract(host) - if err != nil { - addr = host + addr := host + if transport != "unix" { + var err error + addr, err = mAddr.Extract(host) + if err != nil { + addr = host + } + addr = net.JoinHostPort(addr, strconv.Itoa(port)) } node := &mRegistry.Node{ Id: serviceID + "-" + server.DefaultId, - Address: net.JoinHostPort(addr, fmt.Sprint(port)), + Address: addr, Metadata: make(map[string]string), } node.Metadata["registry"] = GetRegistry().String() node.Metadata["server"] = "grpc" - node.Metadata["transport"] = "grpc" + node.Metadata["transport"] = transport node.Metadata["protocol"] = "grpc" return &mRegistry.Service{ @@ -59,7 +64,7 @@ func BuildHTTPService(serviceID, address string, version string) *mRegistry.Serv host = parts[0] } - addr, err := addr.Extract(host) + addr, err := mAddr.Extract(host) if err != nil { addr = host } diff --git a/services/app-provider/pkg/command/server.go b/services/app-provider/pkg/command/server.go index 450bb2d7ab..6d0a044ae6 100644 --- a/services/app-provider/pkg/command/server.go +++ b/services/app-provider/pkg/command/server.go @@ -81,7 +81,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/app-registry/pkg/command/server.go b/services/app-registry/pkg/command/server.go index b8d1a790bf..0a85ce0346 100644 --- a/services/app-registry/pkg/command/server.go +++ b/services/app-registry/pkg/command/server.go @@ -76,7 +76,7 @@ func Server(cfg *config.Config) *cli.Command { cancel() }) - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/auth-app/pkg/command/server.go b/services/auth-app/pkg/command/server.go index 77cb36431a..f70aa4e8c9 100644 --- a/services/auth-app/pkg/command/server.go +++ b/services/auth-app/pkg/command/server.go @@ -89,7 +89,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/auth-basic/pkg/command/server.go b/services/auth-basic/pkg/command/server.go index a89ccb5326..084fce012f 100644 --- a/services/auth-basic/pkg/command/server.go +++ b/services/auth-basic/pkg/command/server.go @@ -94,7 +94,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/auth-bearer/pkg/command/server.go b/services/auth-bearer/pkg/command/server.go index 1c8e239b3a..00652d312f 100644 --- a/services/auth-bearer/pkg/command/server.go +++ b/services/auth-bearer/pkg/command/server.go @@ -81,7 +81,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/auth-machine/pkg/command/server.go b/services/auth-machine/pkg/command/server.go index 0a4dcbb59b..d90e68b763 100644 --- a/services/auth-machine/pkg/command/server.go +++ b/services/auth-machine/pkg/command/server.go @@ -81,7 +81,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/auth-service/pkg/command/server.go b/services/auth-service/pkg/command/server.go index 58ab0dde1d..2f013c8295 100644 --- a/services/auth-service/pkg/command/server.go +++ b/services/auth-service/pkg/command/server.go @@ -82,7 +82,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/collaboration/pkg/config/defaults/defaultconfig.go b/services/collaboration/pkg/config/defaults/defaultconfig.go index 19e5618f23..7ff01181f8 100644 --- a/services/collaboration/pkg/config/defaults/defaultconfig.go +++ b/services/collaboration/pkg/config/defaults/defaultconfig.go @@ -33,6 +33,7 @@ func DefaultConfig() *config.Config { }, GRPC: config.GRPC{ Addr: "127.0.0.1:9301", + Protocol: "tcp", Namespace: "com.owncloud.api", }, HTTP: config.HTTP{ diff --git a/services/collaboration/pkg/config/grpc.go b/services/collaboration/pkg/config/grpc.go index cf66901b72..d52d16228d 100644 --- a/services/collaboration/pkg/config/grpc.go +++ b/services/collaboration/pkg/config/grpc.go @@ -2,6 +2,8 @@ package config // GRPC defines the available grpc configuration. type GRPC struct { - Addr string `yaml:"addr" env:"COLLABORATION_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"6.0.0"` + Addr string `yaml:"addr" env:"COLLABORATION_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"6.0.0"` + Protocol string `yaml:"protocol" env:"COLLABORATION_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"%%NEXT%%"` + Namespace string `yaml:"-"` } diff --git a/services/collaboration/pkg/helpers/registration.go b/services/collaboration/pkg/helpers/registration.go index 321a1947c1..d9ea9459c2 100644 --- a/services/collaboration/pkg/helpers/registration.go +++ b/services/collaboration/pkg/helpers/registration.go @@ -18,7 +18,7 @@ import ( // There are no explicit requirements for the context, and it will be passed // without changes to the underlying RegisterService method. func RegisterOcisService(ctx context.Context, cfg *config.Config, logger log.Logger) error { - svc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name+"."+cfg.App.Name, cfg.GRPC.Addr, version.GetString()) + svc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name+"."+cfg.App.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) return registry.RegisterService(ctx, svc, logger) } diff --git a/services/gateway/pkg/command/server.go b/services/gateway/pkg/command/server.go index 9f556d556e..01b2742599 100644 --- a/services/gateway/pkg/command/server.go +++ b/services/gateway/pkg/command/server.go @@ -81,7 +81,7 @@ func Server(cfg *config.Config) *cli.Command { cancel() }) - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/graph/pkg/service/v0/graph_suite_test.go b/services/graph/pkg/service/v0/graph_suite_test.go index 8446bc9b69..70dfcce145 100644 --- a/services/graph/pkg/service/v0/graph_suite_test.go +++ b/services/graph/pkg/service/v0/graph_suite_test.go @@ -12,7 +12,7 @@ import ( func init() { r := registry.GetRegistry(registry.Inmemory()) - service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "") + service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", }} diff --git a/services/groups/pkg/command/server.go b/services/groups/pkg/command/server.go index cd6dffcb6d..cdd3db0ec2 100644 --- a/services/groups/pkg/command/server.go +++ b/services/groups/pkg/command/server.go @@ -94,7 +94,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/notifications/pkg/service/notification_suite_test.go b/services/notifications/pkg/service/notification_suite_test.go index e045e43981..7f282054fb 100644 --- a/services/notifications/pkg/service/notification_suite_test.go +++ b/services/notifications/pkg/service/notification_suite_test.go @@ -12,7 +12,7 @@ import ( func init() { r := registry.GetRegistry(registry.Inmemory()) - service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "") + service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", }} diff --git a/services/ocm/pkg/command/server.go b/services/ocm/pkg/command/server.go index 6dc234fda9..2e28cccedc 100644 --- a/services/ocm/pkg/command/server.go +++ b/services/ocm/pkg/command/server.go @@ -82,7 +82,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/search/pkg/search/search_suite_test.go b/services/search/pkg/search/search_suite_test.go index ace85d9294..9422179fc5 100644 --- a/services/search/pkg/search/search_suite_test.go +++ b/services/search/pkg/search/search_suite_test.go @@ -12,7 +12,7 @@ import ( func init() { r := registry.GetRegistry(registry.Inmemory()) - service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "") + service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", }} diff --git a/services/sharing/pkg/command/server.go b/services/sharing/pkg/command/server.go index 5fc2bf58c5..73292fd3c2 100644 --- a/services/sharing/pkg/command/server.go +++ b/services/sharing/pkg/command/server.go @@ -98,7 +98,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/storage-publiclink/pkg/command/server.go b/services/storage-publiclink/pkg/command/server.go index 64551f0366..184254cb9f 100644 --- a/services/storage-publiclink/pkg/command/server.go +++ b/services/storage-publiclink/pkg/command/server.go @@ -81,7 +81,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/storage-shares/pkg/command/server.go b/services/storage-shares/pkg/command/server.go index 4cc258a84a..bdeabf0e2d 100644 --- a/services/storage-shares/pkg/command/server.go +++ b/services/storage-shares/pkg/command/server.go @@ -81,7 +81,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/storage-system/pkg/command/server.go b/services/storage-system/pkg/command/server.go index 369f2a5ad7..5195280e41 100644 --- a/services/storage-system/pkg/command/server.go +++ b/services/storage-system/pkg/command/server.go @@ -81,7 +81,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/storage-users/pkg/command/server.go b/services/storage-users/pkg/command/server.go index c4b4137910..df7fca8a4b 100644 --- a/services/storage-users/pkg/command/server.go +++ b/services/storage-users/pkg/command/server.go @@ -92,7 +92,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } diff --git a/services/storage-users/pkg/task/task_suite_test.go b/services/storage-users/pkg/task/task_suite_test.go index 3837277808..ac0c8a44b9 100644 --- a/services/storage-users/pkg/task/task_suite_test.go +++ b/services/storage-users/pkg/task/task_suite_test.go @@ -12,7 +12,7 @@ import ( func init() { r := registry.GetRegistry(registry.Inmemory()) - service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "") + service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", }} diff --git a/services/userlog/pkg/service/service_suit_test.go b/services/userlog/pkg/service/service_suit_test.go index d049672b76..9da9654d4e 100644 --- a/services/userlog/pkg/service/service_suit_test.go +++ b/services/userlog/pkg/service/service_suit_test.go @@ -12,7 +12,7 @@ import ( func init() { r := registry.GetRegistry(registry.Inmemory()) - service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "") + service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", }} diff --git a/services/users/pkg/command/server.go b/services/users/pkg/command/server.go index 92094a627a..95a899873d 100644 --- a/services/users/pkg/command/server.go +++ b/services/users/pkg/command/server.go @@ -94,7 +94,7 @@ func Server(cfg *config.Config) *cli.Command { sync.Trap(&gr, cancel) } - grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Addr, version.GetString()) + grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString()) if err := registry.RegisterService(ctx, grpcSvc, logger); err != nil { logger.Fatal().Err(err).Msg("failed to register the grpc service") } From 871228ac96d42df8d4da3b838879f6930884b747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 28 Jun 2024 14:13:34 +0200 Subject: [PATCH 45/71] make gateway endpoints configurable again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- ocis/pkg/runtime/service/service.go | 4 +-- services/gateway/pkg/config/config.go | 28 ++++++++++---------- services/storage-system/pkg/config/config.go | 9 ++++--- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 311d6f7c1e..afa6e6657c 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -526,7 +526,7 @@ func pingNats(cfg *ociscfg.Config) error { return err } -func pingGateway(_ *ociscfg.Config) error { +func pingGateway(cfg *ociscfg.Config) error { // init grpc connection _, err := ogrpc.NewClient() if err != nil { @@ -536,7 +536,7 @@ func pingGateway(_ *ociscfg.Config) error { b := backoff.NewExponentialBackOff() o := func() error { n := b.NextBackOff() - _, err := pool.GetGatewayServiceClient("com.owncloud.api.gateway") + _, err := pool.GetGatewayServiceClient(cfg.Reva.Address) if err != nil && n > time.Second { logger.New().Error().Err(err).Msgf("can't connect to gateway service, retrying in %s", n) } diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 1d683de4b1..2c4e90e694 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -31,20 +31,20 @@ type Config struct { FrontendPublicURL string `yaml:"frontend_public_url" env:"OCIS_URL;GATEWAY_FRONTEND_PUBLIC_URL" desc:"The public facing URL of the oCIS frontend." introductionVersion:"pre5.0"` - UsersEndpoint string `yaml:"-"` - GroupsEndpoint string `yaml:"-"` - PermissionsEndpoint string `yaml:"-"` - SharingEndpoint string `yaml:"-"` - AuthAppEndpoint string `yaml:"-"` - AuthBasicEndpoint string `yaml:"-"` - AuthBearerEndpoint string `yaml:"-"` - AuthMachineEndpoint string `yaml:"-"` - AuthServiceEndpoint string `yaml:"-"` - StoragePublicLinkEndpoint string `yaml:"-"` - StorageUsersEndpoint string `yaml:"-"` - StorageSharesEndpoint string `yaml:"-"` - AppRegistryEndpoint string `yaml:"-"` - OCMEndpoint string `yaml:"-"` + UsersEndpoint string `yaml:"users_endpoint" env:"GATEWAY_USERS_ENDPOINT" desc:"The USERS API endpoint." introductionVersion:"%%NEXT%%"` + GroupsEndpoint string `yaml:"groups_endpoint" env:"GATEWAY_GROUPS_ENDPOINT" desc:"The GROUPS API endpoint." introductionVersion:"%%NEXT%%"` + PermissionsEndpoint string `yaml:"permissions_endpoint" env:"GATEWAY_PERMISSIONS_ENDPOINT" desc:"The SETTINGS API endpoint." introductionVersion:"%%NEXT%%"` + SharingEndpoint string `yaml:"sharing_endpoint" env:"GATEWAY_SHARING_ENDPOINT" desc:"The SHARE API endpoint." introductionVersion:"%%NEXT%%"` + AuthAppEndpoint string `yaml:"auth_app_endpoint" env:"GATEWAY_AUTH_APP_ENDPOINT" desc:"The AUTH APP API endpoint." introductionVersion:"%%NEXT%%"` + AuthBasicEndpoint string `yaml:"auth_basic_endpoint" env:"GATEWAY_AUTH_BASIC_ENDPOINT" desc:"The AUTH BASIC API endpoint." introductionVersion:"%%NEXT%%"` + AuthBearerEndpoint string `yaml:"auth_bearer_endpoint" env:"GATEWAY_AUTH_BEARER_ENDPOINT" desc:"The AUTH BEARER API endpoint." introductionVersion:"%%NEXT%%"` + AuthMachineEndpoint string `yaml:"auth_machine_endpoint" env:"GATEWAY_AUTH_MACHINE_ENDPOINT" desc:"The AUTH MACHINE API endpoint." introductionVersion:"%%NEXT%%"` + AuthServiceEndpoint string `yaml:"auth_service_endpoint" env:"GATEWAY_AUTH_SERVICE_ENDPOINT" desc:"The AUTH SERVICE API endpoint." introductionVersion:"%%NEXT%%"` + StoragePublicLinkEndpoint string `yaml:"storage_public_link_endpoint" env:"GATEWAY_STORAGE_PUBLIC_LINK_ENDPOINT" desc:"The STORAGE PUBLICLINK API endpoint." introductionVersion:"%%NEXT%%"` + StorageUsersEndpoint string `yaml:"storage_users_endpoint" env:"GATEWAY_STORAGE_USERS_ENDPOINT" desc:"The STORAGE USERS API endpoint." introductionVersion:"%%NEXT%%"` + StorageSharesEndpoint string `yaml:"storage_shares_endpoint" env:"GATEWAY_STORAGE_SHARES_ENDPOINT" desc:"The STORAGE SHARES API endpoint." introductionVersion:"%%NEXT%%"` + AppRegistryEndpoint string `yaml:"app_registry_endpoint" env:"GATEWAY_APP_REGISTRY_ENDPOINT" desc:"The APP REGISTRY API endpoint." introductionVersion:"%%NEXT%%"` + OCMEndpoint string `yaml:"ocm_endpoint" env:"GATEWAY_OCM_ENDPOINT" desc:"The OCM API endpoint." introductionVersion:"%%NEXT%%"` StorageRegistry StorageRegistry `yaml:"storage_registry"` // TODO: should we even support switching this? diff --git a/services/storage-system/pkg/config/config.go b/services/storage-system/pkg/config/config.go index 360ac9cc78..44a1f8dac4 100644 --- a/services/storage-system/pkg/config/config.go +++ b/services/storage-system/pkg/config/config.go @@ -18,10 +18,11 @@ type Config struct { GRPC GRPCConfig `yaml:"grpc"` HTTP HTTPConfig `yaml:"http"` - TokenManager *TokenManager `yaml:"token_manager"` - Reva *shared.Reva `yaml:"reva"` - SystemUserID string `yaml:"system_user_id" env:"OCIS_SYSTEM_USER_ID" desc:"ID of the oCIS storage-system system user. Admins need to set the ID for the STORAGE-SYSTEM system user in this config option which is then used to reference the user. Any reasonable long string is possible, preferably this would be an UUIDv4 format." introductionVersion:"pre5.0"` - SystemUserAPIKey string `yaml:"system_user_api_key" env:"OCIS_SYSTEM_USER_API_KEY" desc:"API key for the STORAGE-SYSTEM system user." introductionVersion:"pre5.0"` + TokenManager *TokenManager `yaml:"token_manager"` + Reva *shared.Reva `yaml:"reva"` + + SystemUserID string `yaml:"system_user_id" env:"OCIS_SYSTEM_USER_ID" desc:"ID of the oCIS storage-system system user. Admins need to set the ID for the STORAGE-SYSTEM system user in this config option which is then used to reference the user. Any reasonable long string is possible, preferably this would be an UUIDv4 format." introductionVersion:"pre5.0"` + SystemUserAPIKey string `yaml:"system_user_api_key" env:"OCIS_SYSTEM_USER_API_KEY" desc:"API key for the STORAGE-SYSTEM system user." introductionVersion:"pre5.0"` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_SYSTEM_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token." introductionVersion:"pre5.0"` From 9d40aa8fc28a2b1a8db089c7a6b034a4fd2f5adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 28 Jun 2024 17:25:40 +0200 Subject: [PATCH 46/71] allow configuring default reva address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- ocis-pkg/config/config.go | 1 + ocis-pkg/config/defaultconfig.go | 4 ++++ ocis-pkg/config/parser/parse.go | 6 +++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 84f4119acd..5e469d7bed 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -67,6 +67,7 @@ type Config struct { GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"` GRPCServiceTLS *shared.GRPCServiceTLS `yaml:"grpc_service_tls"` HTTPServiceTLS shared.HTTPServiceTLS `yaml:"http_service_tls"` + Reva *shared.Reva `yaml:"reva"` Mode Mode // DEPRECATED File string diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index bca348b4ef..f25cccd810 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -1,6 +1,7 @@ package config import ( + "github.com/owncloud/ocis/v2/ocis-pkg/shared" activitylog "github.com/owncloud/ocis/v2/services/activitylog/pkg/config/defaults" antivirus "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/defaults" appProvider "github.com/owncloud/ocis/v2/services/app-provider/pkg/config/defaults" @@ -52,6 +53,9 @@ func DefaultConfig() *Config { Port: "9250", Host: "localhost", }, + Reva: &shared.Reva{ + Address: "com.owncloud.api.gateway", + }, Activitylog: activitylog.DefaultConfig(), Antivirus: antivirus.DefaultConfig(), diff --git a/ocis-pkg/config/parser/parse.go b/ocis-pkg/config/parser/parse.go index 27d98f7b1e..30dbacd17b 100644 --- a/ocis-pkg/config/parser/parse.go +++ b/ocis-pkg/config/parser/parse.go @@ -58,7 +58,9 @@ func EnsureDefaults(cfg *config.Config) { if cfg.GRPCServiceTLS == nil { cfg.GRPCServiceTLS = &shared.GRPCServiceTLS{} } - + if cfg.Reva == nil { + cfg.Reva = &shared.Reva{} + } } // EnsureCommons copies applicable parts of the oCIS config into the commons part @@ -111,6 +113,8 @@ func EnsureCommons(cfg *config.Config) { if cfg.OcisURL != "" { cfg.Commons.OcisURL = cfg.OcisURL } + + cfg.Commons.Reva = structs.CopyOrZeroValue(cfg.Reva) } // Validate checks that all required configs are set. If a required config value From b1e393ab50de0c9af7902f79aa88e4803f0dfc7a Mon Sep 17 00:00:00 2001 From: ownClouders Date: Sat, 24 Aug 2024 00:56:14 +0000 Subject: [PATCH 47/71] [tx] updated from transifex --- .../l10n/locale/ru/LC_MESSAGES/activitylog.po | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 services/activitylog/pkg/service/l10n/locale/ru/LC_MESSAGES/activitylog.po diff --git a/services/activitylog/pkg/service/l10n/locale/ru/LC_MESSAGES/activitylog.po b/services/activitylog/pkg/service/l10n/locale/ru/LC_MESSAGES/activitylog.po new file mode 100644 index 0000000000..fffcf4f57c --- /dev/null +++ b/services/activitylog/pkg/service/l10n/locale/ru/LC_MESSAGES/activitylog.po @@ -0,0 +1,62 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +# Translators: +# Viktor Scharf, 2024 +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: EMAIL\n" +"POT-Creation-Date: 2024-08-24 00:53+0000\n" +"PO-Revision-Date: 2024-08-20 10:13+0000\n" +"Last-Translator: Viktor Scharf, 2024\n" +"Language-Team: Russian (https://app.transifex.com/owncloud-org/teams/6149/ru/)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: ru\n" +"Plural-Forms: nplurals=4; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : n%10==0 || (n%10>=5 && n%10<=9) || (n%100>=11 && n%100<=14)? 2 : 3);\n" + +#: pkg/service/response.go:23 +msgid "{user} added {resource} to {space}" +msgstr "{user} добавил {resource} в {space}" + +#: pkg/service/response.go:31 +msgid "{user} added {sharee} as member of {space}" +msgstr "{user} добавил {sharee} как участника {space}" + +#: pkg/service/response.go:24 +msgid "{user} deleted {resource} from {space}" +msgstr "{user} удалил {resource} из {space}" + +#: pkg/service/response.go:25 +msgid "{user} moved {resource} to {space}" +msgstr "{user} переместил {resource} в {space}" + +#: pkg/service/response.go:30 +msgid "{user} removed link to {resource}" +msgstr "{user} удалил ссылку на {resource}" + +#: pkg/service/response.go:28 +msgid "{user} removed {sharee} from {resource}" +msgstr "{user} удалил {sharee} из {resource}" + +#: pkg/service/response.go:32 +msgid "{user} removed {sharee} from {space}" +msgstr "{user} удалил {sharee} из {space}" + +#: pkg/service/response.go:26 +msgid "{user} renamed {oldResource} to {resource}" +msgstr "{user} переименовал {oldResource} в {resource}" + +#: pkg/service/response.go:29 +msgid "{user} shared {resource} via link" +msgstr "{user} поделился {resource} через ссылку" + +#: pkg/service/response.go:27 +msgid "{user} shared {resource} with {sharee}" +msgstr "{user} поделился {resource} с {sharee}" From a9895a9073af0edad06aefaaa905ec4f8ecf6d60 Mon Sep 17 00:00:00 2001 From: Salipa-Gurung Date: Tue, 20 Aug 2024 17:13:37 +0545 Subject: [PATCH 48/71] add test for activity depth filter --- tests/TestHelpers/GraphHelper.php | 10 +- .../features/apiActivities/activities.feature | 95 +++++++++ .../apiActivities/activityFilter.feature | 187 ++++++++++++++++++ .../features/bootstrap/GraphContext.php | 23 +++ 4 files changed, 313 insertions(+), 2 deletions(-) create mode 100644 tests/acceptance/features/apiActivities/activityFilter.feature diff --git a/tests/TestHelpers/GraphHelper.php b/tests/TestHelpers/GraphHelper.php index 07998e209e..9d8b651826 100644 --- a/tests/TestHelpers/GraphHelper.php +++ b/tests/TestHelpers/GraphHelper.php @@ -2326,6 +2326,7 @@ class GraphHelper { * @param string $user * @param string $password * @param string $resourceId + * @param string $depth * * @return ResponseInterface */ @@ -2334,11 +2335,16 @@ class GraphHelper { string $requestId, string $user, string $password, - string $resourceId + string $resourceId, + ?string $depth = null ): ResponseInterface { // 'kql=itemId' filter is required for the current implementation but it might change in future // See: https://github.com/owncloud/ocis/issues/9194 - $fullUrl = self::getBetaFullUrl($baseUrl, "extensions/org.libregraph/activities?kql=itemid%3A$resourceId"); + if ($depth !== null) { + $fullUrl = self::getBetaFullUrl($baseUrl, "extensions/org.libregraph/activities?kql=itemid%3A$resourceId+AND+depth%3A$depth"); + } else { + $fullUrl = self::getBetaFullUrl($baseUrl, "extensions/org.libregraph/activities?kql=itemid%3A$resourceId"); + } return HttpRequestHelper::get( $fullUrl, $requestId, diff --git a/tests/acceptance/features/apiActivities/activities.feature b/tests/acceptance/features/apiActivities/activities.feature index dc930cb3ef..2f94a0bd82 100644 --- a/tests/acceptance/features/apiActivities/activities.feature +++ b/tests/acceptance/features/apiActivities/activities.feature @@ -1326,3 +1326,98 @@ Feature: check activities } } """ + + @issue-9712 + Scenario: check activity with depth filter + Given user "Alice" has created folder "/New Folder" + And user "Alice" has created folder "/New Folder/Sub Folder" + And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile0.txt" + When user "Alice" lists the activities for folder "New Folder" of space "Personal" with depth "1" using the Graph API + Then the HTTP status code should be "200" + And the JSON data of the response should match + """ + { + "type": "object", + "required": ["value"], + "properties": { + "value": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "uniqueItems": true, + "items": { + "oneOf": [ + { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "New Folder" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + }, + { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "Sub Folder" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + } + ] + } + } + } + } + """ diff --git a/tests/acceptance/features/apiActivities/activityFilter.feature b/tests/acceptance/features/apiActivities/activityFilter.feature new file mode 100644 index 0000000000..0e3e95f611 --- /dev/null +++ b/tests/acceptance/features/apiActivities/activityFilter.feature @@ -0,0 +1,187 @@ +@issue-9712 +Feature: activity filter + As a user + I want to filter activities + So that I can track modifications of specific resource + + Background: + Given user "Alice" has been created with default attributes and without skeleton files + + + Scenario: check activity with depth filter + Given user "Alice" has created folder "/New Folder" + And user "Alice" has created folder "/New Folder/Sub Folder" + And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile0.txt" + When user "Alice" lists the activities for folder "New Folder" of space "Personal" with depth "0" using the Graph API + Then the HTTP status code should be "200" + And the JSON data of the response should match + """ + { + "type": "object", + "required": ["value"], + "properties": { + "value": { + "type": "array", + "minItems": 1, + "maxItems": 1, + "items": { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "New Folder" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + } + } + } + } + """ + When user "Alice" lists the activities for folder "New Folder" of space "Personal" with depth "2" using the Graph API + Then the HTTP status code should be "200" + And the JSON data of the response should match + """ + { + "type": "object", + "required": ["value"], + "properties": { + "value": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "uniqueItems": true, + "items": { + "oneOf": [ + { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "New Folder" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + }, + { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "Sub Folder" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + }, + { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "textfile0.txt" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + } + ] + } + } + } + } + """ diff --git a/tests/acceptance/features/bootstrap/GraphContext.php b/tests/acceptance/features/bootstrap/GraphContext.php index 2fbb154458..d55272f98c 100644 --- a/tests/acceptance/features/bootstrap/GraphContext.php +++ b/tests/acceptance/features/bootstrap/GraphContext.php @@ -2839,6 +2839,29 @@ class GraphContext implements Context { $this->featureContext->setResponse($response); } + /** + * @When /^user "([^"]*)" lists the activities for (?:folder|file) "([^"]*)" of space "([^"]*)" with depth "([^"]*)" using the Graph API/ + * + * @param string $user + * @param string $resource + * @param string $spaceName + * @param string $folderDepth + * + * @return void + */ + public function userListsTheActivitiesForFolderOfSpaceWithDepthUsingTheGraphApi(string $user, string $resource, string $spaceName, string $folderDepth): void { + $resourceId = $this->featureContext->spacesContext->getResourceId($user, $spaceName, $resource); + $response = GraphHelper::getActivities( + $this->featureContext->getBaseUrl(), + $this->featureContext->getStepLineRef(), + $user, + $this->featureContext->getPasswordForUser($user), + $resourceId, + $folderDepth + ); + $this->featureContext->setResponse($response); + } + /** * @When the administrator gets federated users using the Graph API * @When user :user tries to get federated users using the Graph API From df34b917ff84a874eb35ac5a1728319db8756036 Mon Sep 17 00:00:00 2001 From: Prajwol Amatya Date: Mon, 26 Aug 2024 10:00:58 +0545 Subject: [PATCH 49/71] update url pattern regex --- tests/acceptance/features/apiAppProvider/appProvider.feature | 4 ++-- tests/acceptance/features/bootstrap/WebDav.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/features/apiAppProvider/appProvider.feature b/tests/acceptance/features/apiAppProvider/appProvider.feature index 840b874bc4..859d9fb74d 100644 --- a/tests/acceptance/features/apiAppProvider/appProvider.feature +++ b/tests/acceptance/features/apiAppProvider/appProvider.feature @@ -1,7 +1,7 @@ Feature: App Provider As a user I want to access files with collaboration service apps - So that I can get the content of the file + So that I can collaborate with other users Scenario Outline: open file with .odt extension with collaboration app @@ -22,7 +22,7 @@ Feature: App Provider "properties": { "app_url": { "type": "string", - "pattern": "^https:\\/\\/fakeoffice\\.owncloud\\.test\\/not\\/relevant\\?WOPISrc=http%3A%2F%2Fwopiserver%3A9300%2Fwopi%2Ffiles%2F[a-fA-F0-9]{64}$" + "pattern": "^https:\\/\\/(.*?)\\?WOPISrc=http%3A%2F%2F(.*?)%3A9300%2Fwopi%2Ffiles%2F[a-fA-F0-9]{64}$" }, "method": { "const": "POST" diff --git a/tests/acceptance/features/bootstrap/WebDav.php b/tests/acceptance/features/bootstrap/WebDav.php index 70f2ac38fb..cbdadefa9f 100644 --- a/tests/acceptance/features/bootstrap/WebDav.php +++ b/tests/acceptance/features/bootstrap/WebDav.php @@ -1743,7 +1743,7 @@ trait WebDav { * @param string $source * @param string $destination * - * @return void + * @return array */ public function userHasUploadedAFileTo(string $user, string $source, string $destination):array { $response = $this->uploadFile($user, $source, $destination, true); From 349c78f020d5c45b41066b16c8bdeb1b3375cbd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 06:52:28 +0000 Subject: [PATCH 50/71] Bump github.com/grpc-ecosystem/grpc-gateway/v2 from 2.21.0 to 2.22.0 Bumps [github.com/grpc-ecosystem/grpc-gateway/v2](https://github.com/grpc-ecosystem/grpc-gateway) from 2.21.0 to 2.22.0. - [Release notes](https://github.com/grpc-ecosystem/grpc-gateway/releases) - [Changelog](https://github.com/grpc-ecosystem/grpc-gateway/blob/main/.goreleaser.yml) - [Commits](https://github.com/grpc-ecosystem/grpc-gateway/compare/v2.21.0...v2.22.0) --- updated-dependencies: - dependency-name: github.com/grpc-ecosystem/grpc-gateway/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 6 ++-- go.sum | 12 +++---- .../grpc-gateway/v2/runtime/errors.go | 16 +++++++-- .../grpc-gateway/v2/runtime/handler.go | 32 +++++++++++------ .../grpc-gateway/v2/runtime/mux.go | 34 +++++++++++++++---- vendor/modules.txt | 12 +++---- 6 files changed, 77 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index c654237a0c..85bb286a8a 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gookit/config/v2 v2.2.5 github.com/gorilla/mux v1.8.1 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 github.com/invopop/validation v0.8.0 github.com/jellydator/ttlcache/v2 v2.11.1 github.com/jellydator/ttlcache/v3 v3.2.0 @@ -110,7 +110,7 @@ require ( golang.org/x/sync v0.8.0 golang.org/x/term v0.23.0 golang.org/x/text v0.17.0 - google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 @@ -348,7 +348,7 @@ require ( golang.org/x/tools v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect diff --git a/go.sum b/go.sum index c352ef0abc..5dea2d0561 100644 --- a/go.sum +++ b/go.sum @@ -610,8 +610,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vb github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 h1:CWyXh/jylQWp2dtiV33mY4iSSp6yf4lmn+c7/tN+ObI= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0/go.mod h1:nCLIt0w3Ept2NwF8ThLmrppXsfT07oC8k0XNDxd8sVU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE= @@ -1668,10 +1668,10 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a h1:YIa/rzVqMEokBkPtydCkx1VLmv3An1Uw7w1P1m6EhOY= -google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a h1:hqK4+jJZXCU4pW7jsAdGOVFIfLHQeV7LaizZKnZ84HI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/errors.go b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/errors.go index 5682998699..01f5734191 100644 --- a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/errors.go +++ b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/errors.go @@ -93,6 +93,7 @@ func HTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.R func DefaultHTTPErrorHandler(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.ResponseWriter, r *http.Request, err error) { // return Internal when Marshal failed const fallback = `{"code": 13, "message": "failed to marshal error message"}` + const fallbackRewriter = `{"code": 13, "message": "failed to rewrite error message"}` var customStatus *HTTPStatusError if errors.As(err, &customStatus) { @@ -100,19 +101,28 @@ func DefaultHTTPErrorHandler(ctx context.Context, mux *ServeMux, marshaler Marsh } s := status.Convert(err) - pb := s.Proto() w.Header().Del("Trailer") w.Header().Del("Transfer-Encoding") - contentType := marshaler.ContentType(pb) + respRw, err := mux.forwardResponseRewriter(ctx, s.Proto()) + if err != nil { + grpclog.Errorf("Failed to rewrite error message %q: %v", s, err) + w.WriteHeader(http.StatusInternalServerError) + if _, err := io.WriteString(w, fallbackRewriter); err != nil { + grpclog.Errorf("Failed to write response: %v", err) + } + return + } + + contentType := marshaler.ContentType(respRw) w.Header().Set("Content-Type", contentType) if s.Code() == codes.Unauthenticated { w.Header().Set("WWW-Authenticate", s.Message()) } - buf, merr := marshaler.Marshal(pb) + buf, merr := marshaler.Marshal(respRw) if merr != nil { grpclog.Errorf("Failed to marshal error message %q: %v", s, merr) w.WriteHeader(http.StatusInternalServerError) diff --git a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/handler.go b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/handler.go index de1eef1f4f..9f50a569e9 100644 --- a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/handler.go +++ b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/handler.go @@ -3,6 +3,7 @@ package runtime import ( "context" "errors" + "fmt" "io" "net/http" "net/textproto" @@ -55,20 +56,27 @@ func ForwardResponseStream(ctx context.Context, mux *ServeMux, marshaler Marshal return } + respRw, err := mux.forwardResponseRewriter(ctx, resp) + if err != nil { + grpclog.Errorf("Rewrite error: %v", err) + handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err, delimiter) + return + } + if !wroteHeader { - w.Header().Set("Content-Type", marshaler.ContentType(resp)) + w.Header().Set("Content-Type", marshaler.ContentType(respRw)) } var buf []byte - httpBody, isHTTPBody := resp.(*httpbody.HttpBody) + httpBody, isHTTPBody := respRw.(*httpbody.HttpBody) switch { - case resp == nil: + case respRw == nil: buf, err = marshaler.Marshal(errorChunk(status.New(codes.Internal, "empty response"))) case isHTTPBody: buf = httpBody.GetData() default: - result := map[string]interface{}{"result": resp} - if rb, ok := resp.(responseBody); ok { + result := map[string]interface{}{"result": respRw} + if rb, ok := respRw.(responseBody); ok { result["result"] = rb.XXX_ResponseBody() } @@ -164,12 +172,17 @@ func ForwardResponseMessage(ctx context.Context, mux *ServeMux, marshaler Marsha HTTPError(ctx, mux, marshaler, w, req, err) return } + respRw, err := mux.forwardResponseRewriter(ctx, resp) + if err != nil { + grpclog.Errorf("Rewrite error: %v", err) + HTTPError(ctx, mux, marshaler, w, req, err) + return + } var buf []byte - var err error - if rb, ok := resp.(responseBody); ok { + if rb, ok := respRw.(responseBody); ok { buf, err = marshaler.Marshal(rb.XXX_ResponseBody()) } else { - buf, err = marshaler.Marshal(resp) + buf, err = marshaler.Marshal(respRw) } if err != nil { grpclog.Errorf("Marshal error: %v", err) @@ -201,8 +214,7 @@ func handleForwardResponseOptions(ctx context.Context, w http.ResponseWriter, re } for _, opt := range opts { if err := opt(ctx, w, resp); err != nil { - grpclog.Errorf("Error handling ForwardResponseOptions: %v", err) - return err + return fmt.Errorf("error handling ForwardResponseOptions: %w", err) } } return nil diff --git a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go index dbb5ca42b4..60c2065ddc 100644 --- a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go +++ b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go @@ -60,6 +60,7 @@ type ServeMux struct { handlers map[string][]handler middlewares []Middleware forwardResponseOptions []func(context.Context, http.ResponseWriter, proto.Message) error + forwardResponseRewriter ForwardResponseRewriter marshalers marshalerRegistry incomingHeaderMatcher HeaderMatcherFunc outgoingHeaderMatcher HeaderMatcherFunc @@ -75,6 +76,24 @@ type ServeMux struct { // ServeMuxOption is an option that can be given to a ServeMux on construction. type ServeMuxOption func(*ServeMux) +// ForwardResponseRewriter is the signature of a function that is capable of rewriting messages +// before they are forwarded in a unary, stream, or error response. +type ForwardResponseRewriter func(ctx context.Context, response proto.Message) (any, error) + +// WithForwardResponseRewriter returns a ServeMuxOption that allows for implementers to insert logic +// that can rewrite the final response before it is forwarded. +// +// The response rewriter function is called during unary message forwarding, stream message +// forwarding and when errors are being forwarded. +// +// NOTE: Using this option will likely make what is generated by `protoc-gen-openapiv2` incorrect. +// Since this option involves making runtime changes to the response shape or type. +func WithForwardResponseRewriter(fwdResponseRewriter ForwardResponseRewriter) ServeMuxOption { + return func(sm *ServeMux) { + sm.forwardResponseRewriter = fwdResponseRewriter + } +} + // WithForwardResponseOption returns a ServeMuxOption representing the forwardResponseOption. // // forwardResponseOption is an option that will be called on the relevant context.Context, @@ -292,13 +311,14 @@ func WithHealthzEndpoint(healthCheckClient grpc_health_v1.HealthClient) ServeMux // NewServeMux returns a new ServeMux whose internal mapping is empty. func NewServeMux(opts ...ServeMuxOption) *ServeMux { serveMux := &ServeMux{ - handlers: make(map[string][]handler), - forwardResponseOptions: make([]func(context.Context, http.ResponseWriter, proto.Message) error, 0), - marshalers: makeMarshalerMIMERegistry(), - errorHandler: DefaultHTTPErrorHandler, - streamErrorHandler: DefaultStreamErrorHandler, - routingErrorHandler: DefaultRoutingErrorHandler, - unescapingMode: UnescapingModeDefault, + handlers: make(map[string][]handler), + forwardResponseOptions: make([]func(context.Context, http.ResponseWriter, proto.Message) error, 0), + forwardResponseRewriter: func(ctx context.Context, response proto.Message) (any, error) { return response, nil }, + marshalers: makeMarshalerMIMERegistry(), + errorHandler: DefaultHTTPErrorHandler, + streamErrorHandler: DefaultStreamErrorHandler, + routingErrorHandler: DefaultRoutingErrorHandler, + unescapingMode: UnescapingModeDefault, } for _, opt := range opts { diff --git a/vendor/modules.txt b/vendor/modules.txt index bac8f587e4..376d0c8c39 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1169,8 +1169,8 @@ github.com/gorilla/schema ## explicit; go 1.14 github.com/grpc-ecosystem/go-grpc-middleware github.com/grpc-ecosystem/go-grpc-middleware/recovery -# github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 -## explicit; go 1.20 +# github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 +## explicit; go 1.21 github.com/grpc-ecosystem/grpc-gateway/v2/internal/httprule github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2/options github.com/grpc-ecosystem/grpc-gateway/v2/runtime @@ -2286,13 +2286,13 @@ golang.org/x/xerrors/internal # google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de ## explicit; go 1.19 google.golang.org/genproto/protobuf/field_mask -# google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a -## explicit; go 1.20 +# google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 +## explicit; go 1.21 google.golang.org/genproto/googleapis/api google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/api/httpbody -# google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a -## explicit; go 1.20 +# google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 +## explicit; go 1.21 google.golang.org/genproto/googleapis/rpc/errdetails google.golang.org/genproto/googleapis/rpc/status # google.golang.org/grpc v1.65.0 From 027fe55a19a6cbccb938d48100ad2aafee7971d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 06:30:30 +0000 Subject: [PATCH 51/71] Bump github.com/onsi/ginkgo/v2 from 2.20.0 to 2.20.1 Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.20.0 to 2.20.1. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.20.0...v2.20.1) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md | 5 +++++ vendor/github.com/onsi/ginkgo/v2/types/version.go | 2 +- vendor/modules.txt | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 85bb286a8a..8e745fb65b 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/oklog/run v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/ginkgo/v2 v2.20.0 + github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 github.com/open-policy-agent/opa v0.67.1 github.com/orcaman/concurrent-map v1.0.0 diff --git a/go.sum b/go.sum index 5dea2d0561..80e745ce62 100644 --- a/go.sum +++ b/go.sum @@ -921,8 +921,8 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= -github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= +github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= diff --git a/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md b/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md index 56fa8d24c2..6f105f1ad5 100644 --- a/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md +++ b/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md @@ -1,3 +1,8 @@ +## 2.20.1 + +### Fixes +- make BeSpecEvent duration matcher more forgiving [d6f9640] + ## 2.20.0 ### Features diff --git a/vendor/github.com/onsi/ginkgo/v2/types/version.go b/vendor/github.com/onsi/ginkgo/v2/types/version.go index c6af20b689..58fddc09e2 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/version.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/version.go @@ -1,3 +1,3 @@ package types -const VERSION = "2.20.0" +const VERSION = "2.20.1" diff --git a/vendor/modules.txt b/vendor/modules.txt index 376d0c8c39..f991003070 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1494,7 +1494,7 @@ github.com/onsi/ginkgo/reporters/stenographer github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty github.com/onsi/ginkgo/types -# github.com/onsi/ginkgo/v2 v2.20.0 +# github.com/onsi/ginkgo/v2 v2.20.1 ## explicit; go 1.20 github.com/onsi/ginkgo/v2 github.com/onsi/ginkgo/v2/config From 1f6b8362171582698507a66b3d57409c1bafd05d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 07:01:24 +0000 Subject: [PATCH 52/71] Bump sass-loader from 10.5.2 to 16.0.1 in /services/idp Bumps [sass-loader](https://github.com/webpack-contrib/sass-loader) from 10.5.2 to 16.0.1. - [Release notes](https://github.com/webpack-contrib/sass-loader/releases) - [Changelog](https://github.com/webpack-contrib/sass-loader/blob/master/CHANGELOG.md) - [Commits](https://github.com/webpack-contrib/sass-loader/compare/v10.5.2...v16.0.1) --- updated-dependencies: - dependency-name: sass-loader dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- services/idp/package.json | 2 +- services/idp/pnpm-lock.yaml | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/services/idp/package.json b/services/idp/package.json index 8dd1f21698..a84833fe56 100644 --- a/services/idp/package.json +++ b/services/idp/package.json @@ -146,7 +146,7 @@ "react-dev-utils": "^12.0.1", "resolve": "1.22.8", "resolve-url-loader": "^5.0.0", - "sass-loader": "^10.5.2", + "sass-loader": "^16.0.1", "source-map-explorer": "^2.5.3", "typescript": "^4.9.5", "url-loader": "4.1.1", diff --git a/services/idp/pnpm-lock.yaml b/services/idp/pnpm-lock.yaml index 874001d736..eebe06f4c6 100644 --- a/services/idp/pnpm-lock.yaml +++ b/services/idp/pnpm-lock.yaml @@ -232,8 +232,8 @@ importers: specifier: ^5.0.0 version: 5.0.0 sass-loader: - specifier: ^10.5.2 - version: 10.5.2(webpack@5.93.0) + specifier: ^16.0.1 + version: 16.0.1(webpack@5.93.0) source-map-explorer: specifier: ^2.5.3 version: 2.5.3 @@ -5219,21 +5219,26 @@ packages: sanitize.css@13.0.0: resolution: {integrity: sha512-ZRwKbh/eQ6w9vmTjkuG0Ioi3HBwPFce0O+v//ve+aOq1oeCy7jMV2qzzAlpsNuqpqCBjjriM1lbtZbF/Q8jVyA==} - sass-loader@10.5.2: - resolution: {integrity: sha512-vMUoSNOUKJILHpcNCCyD23X34gve1TS7Rjd9uXHeKqhvBG39x6XbswFDtpbTElj6XdMFezoWhkh5vtKudf2cgQ==} - engines: {node: '>= 10.13.0'} + sass-loader@16.0.1: + resolution: {integrity: sha512-xACl1ToTsKnL9Ce5yYpRxrLj9QUDCnwZNhzpC7tKiFyA8zXsd3Ap+HGVnbCgkdQcm43E+i6oKAWBsvGA6ZoiMw==} + engines: {node: '>= 18.12.0'} peerDependencies: - fibers: '>= 3.1.0' + '@rspack/core': 0.x || 1.x node-sass: ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 || ^9.0.0 sass: ^1.3.0 - webpack: ^4.36.0 || ^5.0.0 + sass-embedded: '*' + webpack: ^5.0.0 peerDependenciesMeta: - fibers: + '@rspack/core': optional: true node-sass: optional: true sass: optional: true + sass-embedded: + optional: true + webpack: + optional: true scheduler@0.20.2: resolution: {integrity: sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ==} @@ -12031,13 +12036,10 @@ snapshots: sanitize.css@13.0.0: {} - sass-loader@10.5.2(webpack@5.93.0): + sass-loader@16.0.1(webpack@5.93.0): dependencies: - klona: 2.0.6 - loader-utils: 2.0.4 neo-async: 2.6.2 - schema-utils: 3.3.0 - semver: 7.6.3 + optionalDependencies: webpack: 5.93.0 scheduler@0.20.2: From 385086b491409257a9bf24cff8eb27df79c68e12 Mon Sep 17 00:00:00 2001 From: Salipa-Gurung Date: Wed, 21 Aug 2024 16:22:59 +0545 Subject: [PATCH 53/71] add test for activity limit filter --- tests/TestHelpers/GraphHelper.php | 13 +- .../features/apiActivities/activities.feature | 97 ++++++++- .../apiActivities/activityFilter.feature | 187 ------------------ .../features/bootstrap/GraphContext.php | 13 +- 4 files changed, 110 insertions(+), 200 deletions(-) delete mode 100644 tests/acceptance/features/apiActivities/activityFilter.feature diff --git a/tests/TestHelpers/GraphHelper.php b/tests/TestHelpers/GraphHelper.php index 9d8b651826..99cad0d8c1 100644 --- a/tests/TestHelpers/GraphHelper.php +++ b/tests/TestHelpers/GraphHelper.php @@ -2326,7 +2326,7 @@ class GraphHelper { * @param string $user * @param string $password * @param string $resourceId - * @param string $depth + * @param array $filterParams * * @return ResponseInterface */ @@ -2336,14 +2336,15 @@ class GraphHelper { string $user, string $password, string $resourceId, - ?string $depth = null + ?array $filterParams = [] ): ResponseInterface { // 'kql=itemId' filter is required for the current implementation but it might change in future // See: https://github.com/owncloud/ocis/issues/9194 - if ($depth !== null) { - $fullUrl = self::getBetaFullUrl($baseUrl, "extensions/org.libregraph/activities?kql=itemid%3A$resourceId+AND+depth%3A$depth"); - } else { - $fullUrl = self::getBetaFullUrl($baseUrl, "extensions/org.libregraph/activities?kql=itemid%3A$resourceId"); + $fullUrl = self::getBetaFullUrl($baseUrl, "extensions/org.libregraph/activities?kql=itemid%3A$resourceId"); + if (!empty($filterParams)) { + foreach ($filterParams as $filter => $value) { + $fullUrl .= "+AND+$filter%3A$value"; + } } return HttpRequestHelper::get( $fullUrl, diff --git a/tests/acceptance/features/apiActivities/activities.feature b/tests/acceptance/features/apiActivities/activities.feature index 2f94a0bd82..1a7af58869 100644 --- a/tests/acceptance/features/apiActivities/activities.feature +++ b/tests/acceptance/features/apiActivities/activities.feature @@ -1327,7 +1327,7 @@ Feature: check activities } """ - @issue-9712 + @issue-9850 Scenario: check activity with depth filter Given user "Alice" has created folder "/New Folder" And user "Alice" has created folder "/New Folder/Sub Folder" @@ -1421,3 +1421,98 @@ Feature: check activities } } """ + + @issue-9880 + Scenario: check activity with limit filter + Given user "Alice" has created folder "/New Folder" + And user "Alice" has created folder "/New Folder/Sub Folder" + And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile0.txt" + When user "Alice" lists the activities for folder "New Folder" of space "Personal" with limit "2" using the Graph API + Then the HTTP status code should be "200" + And the JSON data of the response should match + """ + { + "type": "object", + "required": ["value"], + "properties": { + "value": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "uniqueItems": true, + "items": { + "oneOf": [ + { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "New Folder" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + }, + { + "type": "object", + "required": ["id", "template", "times"], + "properties": { + "template": { + "type": "object", + "required": ["message", "variables"], + "properties": { + "message": { + "const": "{user} added {resource} to {space}" + }, + "variables": { + "type": "object", + "required": ["resource", "space", "user"], + "properties": { + "resource": { + "type": "object", + "required": ["id", "name"], + "properties": { + "name": { + "const": "Sub Folder" + } + } + } + } + } + } + }, + "times": { + "type": "object", + "required": ["recordedTime"] + } + } + } + ] + } + } + } + } + """ diff --git a/tests/acceptance/features/apiActivities/activityFilter.feature b/tests/acceptance/features/apiActivities/activityFilter.feature deleted file mode 100644 index 0e3e95f611..0000000000 --- a/tests/acceptance/features/apiActivities/activityFilter.feature +++ /dev/null @@ -1,187 +0,0 @@ -@issue-9712 -Feature: activity filter - As a user - I want to filter activities - So that I can track modifications of specific resource - - Background: - Given user "Alice" has been created with default attributes and without skeleton files - - - Scenario: check activity with depth filter - Given user "Alice" has created folder "/New Folder" - And user "Alice" has created folder "/New Folder/Sub Folder" - And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile0.txt" - When user "Alice" lists the activities for folder "New Folder" of space "Personal" with depth "0" using the Graph API - Then the HTTP status code should be "200" - And the JSON data of the response should match - """ - { - "type": "object", - "required": ["value"], - "properties": { - "value": { - "type": "array", - "minItems": 1, - "maxItems": 1, - "items": { - "type": "object", - "required": ["id", "template", "times"], - "properties": { - "template": { - "type": "object", - "required": ["message", "variables"], - "properties": { - "message": { - "const": "{user} added {resource} to {space}" - }, - "variables": { - "type": "object", - "required": ["resource", "space", "user"], - "properties": { - "resource": { - "type": "object", - "required": ["id", "name"], - "properties": { - "name": { - "const": "New Folder" - } - } - } - } - } - } - }, - "times": { - "type": "object", - "required": ["recordedTime"] - } - } - } - } - } - } - """ - When user "Alice" lists the activities for folder "New Folder" of space "Personal" with depth "2" using the Graph API - Then the HTTP status code should be "200" - And the JSON data of the response should match - """ - { - "type": "object", - "required": ["value"], - "properties": { - "value": { - "type": "array", - "minItems": 3, - "maxItems": 3, - "uniqueItems": true, - "items": { - "oneOf": [ - { - "type": "object", - "required": ["id", "template", "times"], - "properties": { - "template": { - "type": "object", - "required": ["message", "variables"], - "properties": { - "message": { - "const": "{user} added {resource} to {space}" - }, - "variables": { - "type": "object", - "required": ["resource", "space", "user"], - "properties": { - "resource": { - "type": "object", - "required": ["id", "name"], - "properties": { - "name": { - "const": "New Folder" - } - } - } - } - } - } - }, - "times": { - "type": "object", - "required": ["recordedTime"] - } - } - }, - { - "type": "object", - "required": ["id", "template", "times"], - "properties": { - "template": { - "type": "object", - "required": ["message", "variables"], - "properties": { - "message": { - "const": "{user} added {resource} to {space}" - }, - "variables": { - "type": "object", - "required": ["resource", "space", "user"], - "properties": { - "resource": { - "type": "object", - "required": ["id", "name"], - "properties": { - "name": { - "const": "Sub Folder" - } - } - } - } - } - } - }, - "times": { - "type": "object", - "required": ["recordedTime"] - } - } - }, - { - "type": "object", - "required": ["id", "template", "times"], - "properties": { - "template": { - "type": "object", - "required": ["message", "variables"], - "properties": { - "message": { - "const": "{user} added {resource} to {space}" - }, - "variables": { - "type": "object", - "required": ["resource", "space", "user"], - "properties": { - "resource": { - "type": "object", - "required": ["id", "name"], - "properties": { - "name": { - "const": "textfile0.txt" - } - } - } - } - } - } - }, - "times": { - "type": "object", - "required": ["recordedTime"] - } - } - } - ] - } - } - } - } - """ diff --git a/tests/acceptance/features/bootstrap/GraphContext.php b/tests/acceptance/features/bootstrap/GraphContext.php index d55272f98c..856da67d92 100644 --- a/tests/acceptance/features/bootstrap/GraphContext.php +++ b/tests/acceptance/features/bootstrap/GraphContext.php @@ -2840,16 +2840,17 @@ class GraphContext implements Context { } /** - * @When /^user "([^"]*)" lists the activities for (?:folder|file) "([^"]*)" of space "([^"]*)" with depth "([^"]*)" using the Graph API/ + * @When /^user "([^"]*)" lists the activities for (?:folder|file) "([^"]*)" of space "([^"]*)" with (depth|limit) "([^"]*)" using the Graph API/ * * @param string $user * @param string $resource * @param string $spaceName - * @param string $folderDepth + * @param string $filterType + * @param string $filterValue * * @return void */ - public function userListsTheActivitiesForFolderOfSpaceWithDepthUsingTheGraphApi(string $user, string $resource, string $spaceName, string $folderDepth): void { + public function userListsTheActivitiesForFolderOfSpaceWithDepthOrLimitUsingTheGraphApi(string $user, string $resource, string $spaceName, string $filterType, string $filterValue): void { $resourceId = $this->featureContext->spacesContext->getResourceId($user, $spaceName, $resource); $response = GraphHelper::getActivities( $this->featureContext->getBaseUrl(), @@ -2857,7 +2858,7 @@ class GraphContext implements Context { $user, $this->featureContext->getPasswordForUser($user), $resourceId, - $folderDepth + [$filterType => $filterValue] ); $this->featureContext->setResponse($response); } @@ -2872,7 +2873,7 @@ class GraphContext implements Context { */ public function theUserGetsFederatedUsersUsingTheGraphApi(?string $user = null): void { $credentials = $this->getAdminOrUserCredentials($user); - + $response = GraphHelper::getFederatedUsers( $this->featureContext->getBaseUrl(), $this->featureContext->getStepLineRef(), @@ -2893,7 +2894,7 @@ class GraphContext implements Context { */ public function theUserGetsAllUsersUsingTheGraphApi(?string $user = null): void { $credentials = $this->getAdminOrUserCredentials($user); - + $response = GraphHelper::getAllUsers( $this->featureContext->getBaseUrl(), $this->featureContext->getStepLineRef(), From b92ec36ff174b1c6f63a1b59835010e6b98b5c3f Mon Sep 17 00:00:00 2001 From: Salipa-Gurung Date: Mon, 26 Aug 2024 14:32:38 +0545 Subject: [PATCH 54/71] rename file name in test steps --- .../features/apiActivities/activities.feature | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/acceptance/features/apiActivities/activities.feature b/tests/acceptance/features/apiActivities/activities.feature index 1a7af58869..63f452878e 100644 --- a/tests/acceptance/features/apiActivities/activities.feature +++ b/tests/acceptance/features/apiActivities/activities.feature @@ -8,9 +8,9 @@ Feature: check activities @issue-9712 Scenario: check activities after uploading a file and a folder - Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile0.txt" + Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile.txt" And user "Alice" has created folder "/FOLDER" - When user "Alice" lists the activities for file "textfile0.txt" of space "Personal" using the Graph API + When user "Alice" lists the activities for file "textfile.txt" of space "Personal" using the Graph API Then the HTTP status code should be "200" And the JSON data of the response should match """ @@ -50,7 +50,7 @@ Feature: check activities "pattern": "%file_id_pattern%" }, "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } }, @@ -193,9 +193,9 @@ Feature: check activities @issue-9712 Scenario: check activities after deleting a file and a folder - Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile0.txt" + Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile.txt" And user "Alice" has created folder "/FOLDER" - And user "Alice" has deleted file "textfile0.txt" + And user "Alice" has deleted file "textfile.txt" And user "Alice" has deleted folder "FOLDER" When user "Alice" lists the activities of space "Personal" using the Graph API Then the HTTP status code should be "200" @@ -232,7 +232,7 @@ Feature: check activities "required": ["id", "name"], "properties": { "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } } @@ -300,7 +300,7 @@ Feature: check activities "pattern": "%file_id_pattern%" }, "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } }, @@ -429,12 +429,12 @@ Feature: check activities @issue-9712 Scenario: check move activity for a file and a folder - Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile0.txt" + Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile.txt" And user "Alice" has created folder "/FOLDER" And user "Alice" has created folder "/New Folder" - And user "Alice" has moved file "textfile0.txt" to "New Folder/textfile0.txt" + And user "Alice" has moved file "textfile.txt" to "New Folder/textfile.txt" And user "Alice" has moved folder "FOLDER" to "New Folder/FOLDER" - When user "Alice" lists the activities for file "New Folder/textfile0.txt" of space "Personal" using the Graph API + When user "Alice" lists the activities for file "New Folder/textfile.txt" of space "Personal" using the Graph API Then the HTTP status code should be "200" And the JSON data of the response should match """ @@ -492,7 +492,7 @@ Feature: check activities "pattern": "^%file_id_pattern%$" }, "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } }, @@ -657,9 +657,9 @@ Feature: check activities @issue-9712 Scenario: check rename activity for a file and a folder - Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile0.txt" + Given user "Alice" has uploaded file with content "ownCloud test text file 0" to "/textfile.txt" And user "Alice" has created folder "/FOLDER" - And user "Alice" has moved file "textfile0.txt" to "renamed.txt" + And user "Alice" has moved file "textfile.txt" to "renamed.txt" And user "Alice" has moved folder "/FOLDER" to "RENAMED FOLDER" When user "Alice" lists the activities for file "renamed.txt" of space "Personal" using the Graph API Then the HTTP status code should be "200" @@ -718,7 +718,7 @@ Feature: check activities "const": "" }, "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } }, @@ -885,10 +885,10 @@ Feature: check activities Given user "Alice" has created folder "/New Folder" And user "Alice" has created folder "/New Folder/Folder" And user "Alice" has created folder "/New Folder/Sub Folder" - And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/textfile0.txt" - And user "Alice" has moved file "/New Folder/textfile0.txt" to "/New Folder/Sub Folder/textfile0.txt" + And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/textfile.txt" + And user "Alice" has moved file "/New Folder/textfile.txt" to "/New Folder/Sub Folder/textfile.txt" And user "Alice" has moved folder "/New Folder/Folder" to "/New Folder/Sub Folder/Folder" - And user "Alice" has moved file "/New Folder/Sub Folder/textfile0.txt" to "/New Folder/Sub Folder/renamed.txt" + And user "Alice" has moved file "/New Folder/Sub Folder/textfile.txt" to "/New Folder/Sub Folder/renamed.txt" And user "Alice" has moved folder "/New Folder/Sub Folder/Folder" to "/New Folder/Sub Folder/Renamed Folder" And user "Alice" has deleted file "/New Folder/Sub Folder/renamed.txt" And user "Alice" has deleted folder "/New Folder/Sub Folder/Renamed Folder" @@ -1017,7 +1017,7 @@ Feature: check activities "required": ["id", "name"], "properties": { "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } } @@ -1047,7 +1047,7 @@ Feature: check activities "required": ["id", "name"], "properties": { "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } } @@ -1107,7 +1107,7 @@ Feature: check activities "required": ["id", "name"], "properties": { "name": { - "const": "textfile0.txt" + "const": "textfile.txt" } } }, @@ -1331,7 +1331,7 @@ Feature: check activities Scenario: check activity with depth filter Given user "Alice" has created folder "/New Folder" And user "Alice" has created folder "/New Folder/Sub Folder" - And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile0.txt" + And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile.txt" When user "Alice" lists the activities for folder "New Folder" of space "Personal" with depth "1" using the Graph API Then the HTTP status code should be "200" And the JSON data of the response should match @@ -1426,7 +1426,7 @@ Feature: check activities Scenario: check activity with limit filter Given user "Alice" has created folder "/New Folder" And user "Alice" has created folder "/New Folder/Sub Folder" - And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile0.txt" + And user "Alice" has uploaded file with content "ownCloud test text file 0" to "/New Folder/Sub Folder/textfile.txt" When user "Alice" lists the activities for folder "New Folder" of space "Personal" with limit "2" using the Graph API Then the HTTP status code should be "200" And the JSON data of the response should match From e55219604216ff26fc38b9c0d7c259c9937ac052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 12 Aug 2024 15:00:55 +0200 Subject: [PATCH 55/71] update gateway service endpoint descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer Update services/gateway/README.md Co-authored-by: Martin fix env tag Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/set-service-transport.md | 2 +- services/app-provider/pkg/config/config.go | 2 +- services/app-registry/pkg/config/config.go | 2 +- services/auth-app/pkg/config/config.go | 2 +- services/auth-basic/pkg/config/config.go | 2 +- services/auth-bearer/pkg/config/config.go | 2 +- services/auth-machine/pkg/config/config.go | 2 +- services/auth-service/pkg/config/config.go | 2 +- services/collaboration/pkg/config/grpc.go | 2 +- services/gateway/README.md | 66 +++++++++++++++++++ services/gateway/pkg/config/config.go | 30 ++++----- services/groups/pkg/config/config.go | 2 +- services/ocm/pkg/config/config.go | 2 +- services/sharing/pkg/config/config.go | 2 +- .../storage-publiclink/pkg/config/config.go | 2 +- services/storage-shares/pkg/config/config.go | 2 +- services/storage-system/pkg/config/config.go | 2 +- services/storage-users/pkg/config/config.go | 2 +- services/users/pkg/config/config.go | 2 +- 19 files changed, 98 insertions(+), 32 deletions(-) diff --git a/changelog/unreleased/set-service-transport.md b/changelog/unreleased/set-service-transport.md index 48b6346c37..9dd3eae9a0 100644 --- a/changelog/unreleased/set-service-transport.md +++ b/changelog/unreleased/set-service-transport.md @@ -1,6 +1,6 @@ Enhancement: We now set the configured protocol transport for service metadata -This allows using `dns` or `unix` as the grpc protocol for services. Requires reva changes to have an effect +This allows configuring services to listan on `tcp` or `unix` sockets and clients to use the `dns`, `kubernetes` or `unix` protocol URIs instead of service names. https://github.com/owncloud/ocis/pull/9490 https://github.com/cs3org/reva/pull/4744 \ No newline at end of file diff --git a/services/app-provider/pkg/config/config.go b/services/app-provider/pkg/config/config.go index 4421ca692a..2bce382967 100644 --- a/services/app-provider/pkg/config/config.go +++ b/services/app-provider/pkg/config/config.go @@ -48,7 +48,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"APP_PROVIDER_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"APP_PROVIDER_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;APP_PROVIDER_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` } type Drivers struct { diff --git a/services/app-registry/pkg/config/config.go b/services/app-registry/pkg/config/config.go index c46b5d7530..4a385e09dc 100644 --- a/services/app-registry/pkg/config/config.go +++ b/services/app-registry/pkg/config/config.go @@ -47,7 +47,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"APP_REGISTRY_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"APP_REGISTRY_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;APP_REGISTRY_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } type AppRegistry struct { diff --git a/services/auth-app/pkg/config/config.go b/services/auth-app/pkg/config/config.go index 175b74fd8a..0c484fa5f3 100644 --- a/services/auth-app/pkg/config/config.go +++ b/services/auth-app/pkg/config/config.go @@ -58,7 +58,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"AUTH_APP_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"%%NEXT%%"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"AUTH_APP_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"%%NEXT%%"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;AUTH_APP_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"%%NEXT%%"` } // HTTP defines the available http configuration. diff --git a/services/auth-basic/pkg/config/config.go b/services/auth-basic/pkg/config/config.go index b7458a1aab..3d2e3ea773 100644 --- a/services/auth-basic/pkg/config/config.go +++ b/services/auth-basic/pkg/config/config.go @@ -48,7 +48,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"AUTH_BASIC_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"AUTH_BASIC_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;AUTH_BASIC_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } type AuthProviders struct { diff --git a/services/auth-bearer/pkg/config/config.go b/services/auth-bearer/pkg/config/config.go index f0fcf5a2cd..57a2db3f90 100644 --- a/services/auth-bearer/pkg/config/config.go +++ b/services/auth-bearer/pkg/config/config.go @@ -48,7 +48,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"AUTH_BEARER_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"AUTH_BEARER_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;AUTH_BEARER_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } type OIDC struct { diff --git a/services/auth-machine/pkg/config/config.go b/services/auth-machine/pkg/config/config.go index 29f9a1e623..b92a860b58 100644 --- a/services/auth-machine/pkg/config/config.go +++ b/services/auth-machine/pkg/config/config.go @@ -48,5 +48,5 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"AUTH_MACHINE_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"AUTH_MACHINE_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;AUTH_MACHINE_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } diff --git a/services/auth-service/pkg/config/config.go b/services/auth-service/pkg/config/config.go index 0e7ee48c9d..4c62426242 100644 --- a/services/auth-service/pkg/config/config.go +++ b/services/auth-service/pkg/config/config.go @@ -47,7 +47,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"AUTH_SERVICE_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"AUTH_SERVICE_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;AUTH_SERVICE_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"5.0"` } // ServiceAccount is the configuration for the used service account diff --git a/services/collaboration/pkg/config/grpc.go b/services/collaboration/pkg/config/grpc.go index d52d16228d..e8fd1e25bc 100644 --- a/services/collaboration/pkg/config/grpc.go +++ b/services/collaboration/pkg/config/grpc.go @@ -3,7 +3,7 @@ package config // GRPC defines the available grpc configuration. type GRPC struct { Addr string `yaml:"addr" env:"COLLABORATION_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"6.0.0"` - Protocol string `yaml:"protocol" env:"COLLABORATION_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"%%NEXT%%"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;COLLABORATION_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"%%NEXT%%"` Namespace string `yaml:"-"` } diff --git a/services/gateway/README.md b/services/gateway/README.md index f4ead34bae..8d4598e612 100644 --- a/services/gateway/README.md +++ b/services/gateway/README.md @@ -31,6 +31,72 @@ Store specific notes: - When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus. - When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc. +## Service endpoints + +The gateway acts as a proxy for other CS3 services. As such it has to forward requests to a lot of services and needs to establish connections by looking up the IP address using the service registry. Instead of using the service registry each endpoint can also be configured to use the grpc `dns://` or `kubernetes://` URLs, which might be useful when running in kubernetes. + +For a local single node deployment you might want to use `unix:` sockets as shown below. Using unix sockets will reduce the amount of service lookups and omit the TCP stack. For now, this is experimental and the services do not delete the socket on shutdown. PRs welcome. + +```console +USERS_GRPC_PROTOCOL=unix" +USERS_GRPC_ADDR=/var/run/ocis/users.sock" +GATEWAY_USERS_ENDPOINT=unix:/var/run/ocis/users.sock" + +GROUPS_GRPC_PROTOCOL=unix" +GROUPS_GRPC_ADDR=/var/run/ocis/groups.sock" +GATEWAY_GROUPS_ENDPOINT=unix:/var/run/ocis/groups.sock" + +AUTH_APP_GRPC_PROTOCOL=unix" +AUTH_APP_GRPC_ADDR=/var/run/ocis/auth-app.sock" +GATEWAY_AUTH_APP_ENDPOINT=unix:/var/run/ocis/auth-app.sock" + +AUTH_BASIC_GRPC_PROTOCOL=unix" +AUTH_BASIC_GRPC_ADDR=/var/run/ocis/auth-basic.sock" +GATEWAY_AUTH_BASIC_ENDPOINT=unix:/var/run/ocis/auth-basic.sock" + +AUTH_MACHINE_GRPC_PROTOCOL=unix" +AUTH_MACHINE_GRPC_ADDR=/var/run/ocis/auth-machine.sock" +GATEWAY_AUTH_MACHINE_ENDPOINT=unix:/var/run/ocis/auth-machine.sock" + +AUTH_SERVICE_GRPC_PROTOCOL=unix" +AUTH_SERVICE_GRPC_ADDR=/var/run/ocis/auth-service.sock" +GATEWAY_AUTH_SERVICE_ENDPOINT=unix:/var/run/ocis/auth-service.sock" + +STORAGE_PUBLIC_LINK_GRPC_PROTOCOL=unix" +STORAGE_PUBLIC_LINK_GRPC_ADDR=/var/run/ocis/storage-public-link.sock" +GATEWAY_STORAGE_PUBLIC_LINK_ENDPOINT=unix:/var/run/ocis/storage-public-link.sock" + +STORAGE_USERS_GRPC_PROTOCOL=unix" +STORAGE_USERS_GRPC_ADDR=/var/run/ocis/storage-users.sock" +GATEWAY_STORAGE_USERS_ENDPOINT=unix:/var/run/ocis/storage-users.sock" +// graph sometimes bypasses the gateway so we need to configure the socket here as wel +GRAPH_SPACES_STORAGE_USERS_ADDRESS=unix:/var/run/ocis/storage-users.sock" + +STORAGE_SHARES_GRPC_PROTOCOL=unix" +STORAGE_SHARES_GRPC_ADDR=/var/run/ocis/storage-shares.sock" +GATEWAY_STORAGE_SHARES_ENDPOINT=unix:/var/run/ocis/storage-shares.sock" + +APP_REGISTRY_GRPC_PROTOCOL=unix" +APP_REGISTRY_GRPC_ADDR=/var/run/ocis/app-registry.sock" +GATEWAY_APP_REGISTRY_ENDPOINT=unix:/var/run/ocis/app-registry.sock" + +OCM_GRPC_PROTOCOL=unix" +OCM_GRPC_ADDR=/var/run/ocis/ocm.sock" +GATEWAY_OCM_ENDPOINT=unix:/var/run/ocis/ocm.sock" + +// storage system +STORAGE_SYSTEM_GRPC_PROTOCOL="unix" +STORAGE_SYSTEM_GRPC_ADDR="/var/run/ocis/storage-system.sock" +STORAGE_GATEWAY_GRPC_ADDR="unix:/var/run/ocis/storage-system.sock" +STORAGE_GRPC_ADDR="unix:/var/run/ocis/storage-system.sock" +SHARING_USER_CS3_PROVIDER_ADDR="unix:/var/run/ocis/storage-system.sock" +SHARING_USER_JSONCS3_PROVIDER_ADDR="unix:/var/run/ocis/storage-system.sock" +SHARING_PUBLIC_CS3_PROVIDER_ADDR="unix:/var/run/ocis/storage-system.sock" +SHARING_PUBLIC_JSONCS3_PROVIDER_ADDR="unix:/var/run/ocis/storage-system.sock" +``` + + + ## Storage registry In order to add another storage provider the CS3 storage registry that is running as part of the CS3 gateway hes to be made aware of it. The easiest cleanest way to do it is to set `GATEWAY_STORAGE_REGISTRY_CONFIG_JSON=/path/to/storages.json` and list all storage providers like this: diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 2c4e90e694..46ed4044ab 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -31,20 +31,20 @@ type Config struct { FrontendPublicURL string `yaml:"frontend_public_url" env:"OCIS_URL;GATEWAY_FRONTEND_PUBLIC_URL" desc:"The public facing URL of the oCIS frontend." introductionVersion:"pre5.0"` - UsersEndpoint string `yaml:"users_endpoint" env:"GATEWAY_USERS_ENDPOINT" desc:"The USERS API endpoint." introductionVersion:"%%NEXT%%"` - GroupsEndpoint string `yaml:"groups_endpoint" env:"GATEWAY_GROUPS_ENDPOINT" desc:"The GROUPS API endpoint." introductionVersion:"%%NEXT%%"` - PermissionsEndpoint string `yaml:"permissions_endpoint" env:"GATEWAY_PERMISSIONS_ENDPOINT" desc:"The SETTINGS API endpoint." introductionVersion:"%%NEXT%%"` - SharingEndpoint string `yaml:"sharing_endpoint" env:"GATEWAY_SHARING_ENDPOINT" desc:"The SHARE API endpoint." introductionVersion:"%%NEXT%%"` - AuthAppEndpoint string `yaml:"auth_app_endpoint" env:"GATEWAY_AUTH_APP_ENDPOINT" desc:"The AUTH APP API endpoint." introductionVersion:"%%NEXT%%"` - AuthBasicEndpoint string `yaml:"auth_basic_endpoint" env:"GATEWAY_AUTH_BASIC_ENDPOINT" desc:"The AUTH BASIC API endpoint." introductionVersion:"%%NEXT%%"` - AuthBearerEndpoint string `yaml:"auth_bearer_endpoint" env:"GATEWAY_AUTH_BEARER_ENDPOINT" desc:"The AUTH BEARER API endpoint." introductionVersion:"%%NEXT%%"` - AuthMachineEndpoint string `yaml:"auth_machine_endpoint" env:"GATEWAY_AUTH_MACHINE_ENDPOINT" desc:"The AUTH MACHINE API endpoint." introductionVersion:"%%NEXT%%"` - AuthServiceEndpoint string `yaml:"auth_service_endpoint" env:"GATEWAY_AUTH_SERVICE_ENDPOINT" desc:"The AUTH SERVICE API endpoint." introductionVersion:"%%NEXT%%"` - StoragePublicLinkEndpoint string `yaml:"storage_public_link_endpoint" env:"GATEWAY_STORAGE_PUBLIC_LINK_ENDPOINT" desc:"The STORAGE PUBLICLINK API endpoint." introductionVersion:"%%NEXT%%"` - StorageUsersEndpoint string `yaml:"storage_users_endpoint" env:"GATEWAY_STORAGE_USERS_ENDPOINT" desc:"The STORAGE USERS API endpoint." introductionVersion:"%%NEXT%%"` - StorageSharesEndpoint string `yaml:"storage_shares_endpoint" env:"GATEWAY_STORAGE_SHARES_ENDPOINT" desc:"The STORAGE SHARES API endpoint." introductionVersion:"%%NEXT%%"` - AppRegistryEndpoint string `yaml:"app_registry_endpoint" env:"GATEWAY_APP_REGISTRY_ENDPOINT" desc:"The APP REGISTRY API endpoint." introductionVersion:"%%NEXT%%"` - OCMEndpoint string `yaml:"ocm_endpoint" env:"GATEWAY_OCM_ENDPOINT" desc:"The OCM API endpoint." introductionVersion:"%%NEXT%%"` + UsersEndpoint string `yaml:"users_endpoint" env:"GATEWAY_USERS_ENDPOINT" desc:"The endpoint of the users service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + GroupsEndpoint string `yaml:"groups_endpoint" env:"GATEWAY_GROUPS_ENDPOINT" desc:"The endpoint of the groups service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + PermissionsEndpoint string `yaml:"permissions_endpoint" env:"GATEWAY_PERMISSIONS_ENDPOINT" desc:"The endpoint of the permissions service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + SharingEndpoint string `yaml:"sharing_endpoint" env:"GATEWAY_SHARING_ENDPOINT" desc:"The endpoint of the shares service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + AuthAppEndpoint string `yaml:"auth_app_endpoint" env:"GATEWAY_AUTH_APP_ENDPOINT" desc:"The endpoint of the auth-app service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + AuthBasicEndpoint string `yaml:"auth_basic_endpoint" env:"GATEWAY_AUTH_BASIC_ENDPOINT" desc:"The endpoint of the auth-basic service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + AuthBearerEndpoint string `yaml:"auth_bearer_endpoint" env:"GATEWAY_AUTH_BEARER_ENDPOINT" desc:"The endpoint of the auth-bearer service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + AuthMachineEndpoint string `yaml:"auth_machine_endpoint" env:"GATEWAY_AUTH_MACHINE_ENDPOINT" desc:"The endpoint of the auth-machine service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + AuthServiceEndpoint string `yaml:"auth_service_endpoint" env:"GATEWAY_AUTH_SERVICE_ENDPOINT" desc:"The endpoint of the auth-service service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + StoragePublicLinkEndpoint string `yaml:"storage_public_link_endpoint" env:"GATEWAY_STORAGE_PUBLIC_LINK_ENDPOINT" desc:"The endpoint of the storage-publiclink service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + StorageUsersEndpoint string `yaml:"storage_users_endpoint" env:"GATEWAY_STORAGE_USERS_ENDPOINT" desc:"The endpoint of the storage-users service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + StorageSharesEndpoint string `yaml:"storage_shares_endpoint" env:"GATEWAY_STORAGE_SHARES_ENDPOINT" desc:"The endpoint of the storag-shares service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + AppRegistryEndpoint string `yaml:"app_registry_endpoint" env:"GATEWAY_APP_REGISTRY_ENDPOINT" desc:"The endpoint of the app-registry service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` + OCMEndpoint string `yaml:"ocm_endpoint" env:"GATEWAY_OCM_ENDPOINT" desc:"The endpoint of the ocm service. Can take a service name or a gRPC URI with the dns, kubernetes or unix protocol." introductionVersion:"%%NEXT%%"` StorageRegistry StorageRegistry `yaml:"storage_registry"` // TODO: should we even support switching this? @@ -74,7 +74,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"OCIS_GATEWAY_GRPC_ADDR;GATEWAY_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"GATEWAY_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;GATEWAY_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } type StorageRegistry struct { diff --git a/services/groups/pkg/config/config.go b/services/groups/pkg/config/config.go index 7f22179ee8..b08c1703d6 100644 --- a/services/groups/pkg/config/config.go +++ b/services/groups/pkg/config/config.go @@ -49,7 +49,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"GROUPS_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"GROUPS_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;GROUPS_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } type Drivers struct { diff --git a/services/ocm/pkg/config/config.go b/services/ocm/pkg/config/config.go index ee1cad1ac8..f7c00a6c81 100644 --- a/services/ocm/pkg/config/config.go +++ b/services/ocm/pkg/config/config.go @@ -77,7 +77,7 @@ type GRPCConfig struct { Addr string `ocisConfig:"addr" env:"OCM_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"5.0"` Namespace string `ocisConfig:"-" yaml:"-"` TLS *shared.GRPCServiceTLS `yaml:"tls"` - Protocol string `yaml:"protocol" env:"OCM_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;OCM_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"5.0"` } type ScienceMesh struct { diff --git a/services/sharing/pkg/config/config.go b/services/sharing/pkg/config/config.go index 73f41240e5..2bc58d850b 100644 --- a/services/sharing/pkg/config/config.go +++ b/services/sharing/pkg/config/config.go @@ -56,7 +56,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"SHARING_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"SHARING_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;SHARING_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } type UserSharingDrivers struct { diff --git a/services/storage-publiclink/pkg/config/config.go b/services/storage-publiclink/pkg/config/config.go index 26b1c184d3..ba35b21238 100644 --- a/services/storage-publiclink/pkg/config/config.go +++ b/services/storage-publiclink/pkg/config/config.go @@ -48,7 +48,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"STORAGE_PUBLICLINK_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"STORAGE_PUBLICLINK_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;STORAGE_PUBLICLINK_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } type StorageProvider struct { diff --git a/services/storage-shares/pkg/config/config.go b/services/storage-shares/pkg/config/config.go index 06e0281258..a166646007 100644 --- a/services/storage-shares/pkg/config/config.go +++ b/services/storage-shares/pkg/config/config.go @@ -49,5 +49,5 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"STORAGE_SHARES_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"STORAGE_SHARES_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;STORAGE_SHARES_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service." introductionVersion:"pre5.0"` } diff --git a/services/storage-system/pkg/config/config.go b/services/storage-system/pkg/config/config.go index 44a1f8dac4..cc35e32107 100644 --- a/services/storage-system/pkg/config/config.go +++ b/services/storage-system/pkg/config/config.go @@ -61,7 +61,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"STORAGE_SYSTEM_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"STORAGE_SYSTEM_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;STORAGE_SYSTEM_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` } // HTTPConfig holds HTTPConfig config diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index ef852f6616..4f61d3dab5 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -75,7 +75,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"STORAGE_USERS_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"STORAGE_USERS_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;STORAGE_USERS_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` } // HTTPConfig is the configuration for the http server diff --git a/services/users/pkg/config/config.go b/services/users/pkg/config/config.go index 2f25ab044d..2dd956c8b3 100644 --- a/services/users/pkg/config/config.go +++ b/services/users/pkg/config/config.go @@ -48,7 +48,7 @@ type GRPCConfig struct { Addr string `yaml:"addr" env:"USERS_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"pre5.0"` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` - Protocol string `yaml:"protocol" env:"USERS_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` + Protocol string `yaml:"protocol" env:"OCIS_GRPC_PROTOCOL;USERS_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service." introductionVersion:"pre5.0"` } type Drivers struct { From f735031d69258bf03574119616d596dbc1c32328 Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Mon, 26 Aug 2024 11:14:33 +0200 Subject: [PATCH 56/71] Fix response code when upload a file over locked --- changelog/unreleased/fix-upload-response-code.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/fix-upload-response-code.md diff --git a/changelog/unreleased/fix-upload-response-code.md b/changelog/unreleased/fix-upload-response-code.md new file mode 100644 index 0000000000..ebb9f4ae88 --- /dev/null +++ b/changelog/unreleased/fix-upload-response-code.md @@ -0,0 +1,6 @@ +Bugfix: Fix response code when upload a file over locked + +We fixed a bug where the response code was incorrect when uploading a file over a locked file. + +https://github.com/owncloud/ocis/pull/9894 +https://github.com/owncloud/ocis/issues/7638 From f8cb2e98e74b0548be11d7dc5bf4eaab81f2929d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 26 Aug 2024 10:06:40 +0000 Subject: [PATCH 57/71] Automated changelog update [skip ci] --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea4cc2054f..f72aa2e406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ The following sections list the changes for unreleased. * Bugfix - Use key to get specific trash item: [#9879](https://github.com/owncloud/ocis/pull/9879) * Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) +* Enhancement - We now set the configured protocol transport for service metadata: [#9490](https://github.com/owncloud/ocis/pull/9490) * Enhancement - Bump reva: [#9879](https://github.com/owncloud/ocis/pull/9879) * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) * Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) @@ -111,6 +112,15 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/1357 https://github.com/owncloud/ocis/pull/9890 +* Enhancement - We now set the configured protocol transport for service metadata: [#9490](https://github.com/owncloud/ocis/pull/9490) + + This allows configuring services to listan on `tcp` or `unix` sockets and + clients to use the `dns`, `kubernetes` or `unix` protocol URIs instead of + service names. + + https://github.com/owncloud/ocis/pull/9490 + https://github.com/cs3org/reva/pull/4744 + * Enhancement - Bump reva: [#9879](https://github.com/owncloud/ocis/pull/9879) Bumps reva version From 109b23966c6bd3fc8b8f18528009e19f48f27ffb Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Wed, 21 Aug 2024 16:24:36 +0200 Subject: [PATCH 58/71] bump some jwt related go modules to current version go-jwt/jwt to v5.2.1 MicahParks/keyfunc to v2.1.0 --- go.mod | 6 +- go.sum | 9 +- ocis-pkg/oidc/client.go | 27 ++--- ocis-pkg/oidc/metadata.go | 2 +- ocis-pkg/oidc/mocks/oidc_client.go | 4 +- .../collaboration/pkg/middleware/claims.go | 2 +- .../pkg/middleware/wopicontext.go | 7 +- .../pkg/service/grpc/v0/service_test.go | 2 +- .../pkg/middleware/authentication_test.go | 2 +- services/proxy/pkg/middleware/oidc_auth.go | 2 +- .../proxy/pkg/middleware/oidc_auth_test.go | 2 +- .../thumbnails/pkg/service/grpc/v0/service.go | 2 +- .../thumbnails/pkg/service/http/v0/service.go | 7 +- services/thumbnails/pkg/service/jwt/jwt.go | 2 +- .../MicahParks/keyfunc/{ => v2}/LICENSE | 0 .../MicahParks/keyfunc/{ => v2}/README.md | 111 ++++++++++++------ .../MicahParks/keyfunc/{ => v2}/ecdsa.go | 0 .../MicahParks/keyfunc/{ => v2}/eddsa.go | 0 .../keyfunc/{ => v2}/example_jwks.json | 0 .../MicahParks/keyfunc/{ => v2}/get.go | 14 ++- .../MicahParks/keyfunc/{ => v2}/given.go | 68 ++--------- .../MicahParks/keyfunc/{ => v2}/jwks.go | 0 .../MicahParks/keyfunc/{ => v2}/keyfunc.go | 5 +- .../MicahParks/keyfunc/{ => v2}/multiple.go | 13 +- .../MicahParks/keyfunc/{ => v2}/oct.go | 0 .../MicahParks/keyfunc/{ => v2}/options.go | 14 ++- .../MicahParks/keyfunc/{ => v2}/rsa.go | 0 .../golang-jwt/jwt/v5/MIGRATION_GUIDE.md | 14 ++- vendor/github.com/golang-jwt/jwt/v5/ecdsa.go | 4 +- .../github.com/golang-jwt/jwt/v5/ed25519.go | 7 +- .../golang-jwt/jwt/v5/errors_go_other.go | 2 +- vendor/github.com/golang-jwt/jwt/v5/hmac.go | 4 +- vendor/github.com/golang-jwt/jwt/v5/none.go | 2 +- vendor/github.com/golang-jwt/jwt/v5/parser.go | 85 +++++++++----- .../golang-jwt/jwt/v5/parser_option.go | 8 ++ vendor/github.com/golang-jwt/jwt/v5/rsa.go | 4 +- .../github.com/golang-jwt/jwt/v5/rsa_pss.go | 4 +- vendor/github.com/golang-jwt/jwt/v5/token.go | 14 +++ vendor/github.com/golang-jwt/jwt/v5/types.go | 5 +- .../github.com/golang-jwt/jwt/v5/validator.go | 47 +++++--- vendor/modules.txt | 8 +- 41 files changed, 285 insertions(+), 224 deletions(-) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/LICENSE (100%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/README.md (81%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/ecdsa.go (100%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/eddsa.go (100%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/example_jwks.json (100%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/get.go (94%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/given.go (54%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/jwks.go (100%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/keyfunc.go (90%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/multiple.go (78%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/oct.go (100%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/options.go (90%) rename vendor/github.com/MicahParks/keyfunc/{ => v2}/rsa.go (100%) diff --git a/go.mod b/go.mod index 8e745fb65b..67972ac18a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/CiscoM31/godata v1.0.10 github.com/KimMachineGun/automemlimit v0.6.1 github.com/Masterminds/semver v1.5.0 - github.com/MicahParks/keyfunc v1.9.0 + github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/Nerzal/gocloak/v13 v13.9.0 github.com/bbalet/stopwords v1.0.0 github.com/beevik/etree v1.4.1 @@ -42,7 +42,7 @@ require ( github.com/go-micro/plugins/v4/wrapper/trace/opentelemetry v1.2.0 github.com/go-playground/validator/v10 v10.22.0 github.com/gofrs/uuid v4.4.0+incompatible - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.6.0 github.com/google/go-tika v0.3.1 @@ -220,7 +220,7 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/glog v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index 80e745ce62..1d25b439d5 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= -github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= +github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= +github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -482,11 +482,10 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= diff --git a/ocis-pkg/oidc/client.go b/ocis-pkg/oidc/client.go index 5c73f7134f..741f679372 100644 --- a/ocis-pkg/oidc/client.go +++ b/ocis-pkg/oidc/client.go @@ -15,10 +15,10 @@ import ( "sync" "time" - "github.com/MicahParks/keyfunc" + "github.com/MicahParks/keyfunc/v2" goidc "github.com/coreos/go-oidc/v3/oidc" "github.com/go-jose/go-jose/v3" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/proxy/pkg/config" "golang.org/x/oauth2" @@ -296,7 +296,14 @@ func (c *oidcClient) verifyAccessTokenJWT(token string) (RegClaimsWithSID, jwt.M return claims, mapClaims, errors.New("error initializing jwks keyfunc") } - _, err := jwt.ParseWithClaims(token, &claims, jwks.Keyfunc) + issuer := c.issuer + if c.provider.AccessTokenIssuer != "" { + // AD FS .well-known/openid-configuration has an optional `access_token_issuer` which takes precedence over `issuer` + // See https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-oidce/586de7dd-3385-47c7-93a2-935d9e90441c + issuer = c.provider.AccessTokenIssuer + } + + _, err := jwt.ParseWithClaims(token, &claims, jwks.Keyfunc, jwt.WithIssuer(issuer)) if err != nil { return claims, mapClaims, err } @@ -308,20 +315,6 @@ func (c *oidcClient) verifyAccessTokenJWT(token string) (RegClaimsWithSID, jwt.M return claims, mapClaims, err } - issuer := c.issuer - if c.provider.AccessTokenIssuer != "" { - // AD FS .well-known/openid-configuration has an optional `access_token_issuer` which takes precedence over `issuer` - // See https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-oidce/586de7dd-3385-47c7-93a2-935d9e90441c - issuer = c.provider.AccessTokenIssuer - } - - if !claims.VerifyIssuer(issuer, true) { - vErr := jwt.ValidationError{} - vErr.Inner = jwt.ErrTokenInvalidIssuer - vErr.Errors |= jwt.ValidationErrorIssuer - return claims, mapClaims, vErr - } - return claims, mapClaims, nil } diff --git a/ocis-pkg/oidc/metadata.go b/ocis-pkg/oidc/metadata.go index 2f952e226f..2d2708a0f3 100644 --- a/ocis-pkg/oidc/metadata.go +++ b/ocis-pkg/oidc/metadata.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/owncloud/ocis/v2/ocis-pkg/log" ) diff --git a/ocis-pkg/oidc/mocks/oidc_client.go b/ocis-pkg/oidc/mocks/oidc_client.go index 0aa0dfdfee..450b135de7 100644 --- a/ocis-pkg/oidc/mocks/oidc_client.go +++ b/ocis-pkg/oidc/mocks/oidc_client.go @@ -1,11 +1,11 @@ -// Code generated by mockery v2.40.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks import ( context "context" - jwt "github.com/golang-jwt/jwt/v4" + jwt "github.com/golang-jwt/jwt/v5" mock "github.com/stretchr/testify/mock" oauth2 "golang.org/x/oauth2" diff --git a/services/collaboration/pkg/middleware/claims.go b/services/collaboration/pkg/middleware/claims.go index 893e19e940..c357c8119b 100644 --- a/services/collaboration/pkg/middleware/claims.go +++ b/services/collaboration/pkg/middleware/claims.go @@ -1,6 +1,6 @@ package middleware -import "github.com/golang-jwt/jwt/v4" +import "github.com/golang-jwt/jwt/v5" // Claims contains the jwt registered claims plus the used WOPI context type Claims struct { diff --git a/services/collaboration/pkg/middleware/wopicontext.go b/services/collaboration/pkg/middleware/wopicontext.go index e6b9dff0af..22dcdf0089 100644 --- a/services/collaboration/pkg/middleware/wopicontext.go +++ b/services/collaboration/pkg/middleware/wopicontext.go @@ -11,7 +11,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/owncloud/ocis/v2/services/collaboration/pkg/config" "github.com/owncloud/ocis/v2/services/collaboration/pkg/helpers" "github.com/rs/zerolog" @@ -69,11 +69,6 @@ func WopiContextAuthMiddleware(cfg *config.Config, next http.Handler) http.Handl return } - if err := claims.Valid(); err != nil { - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) - return - } - ctx := r.Context() wopiContextAccessToken, err := DecryptAES([]byte(cfg.Wopi.Secret), claims.WopiContext.AccessToken) diff --git a/services/collaboration/pkg/service/grpc/v0/service_test.go b/services/collaboration/pkg/service/grpc/v0/service_test.go index 68f5aa1416..6fd77b1250 100644 --- a/services/collaboration/pkg/service/grpc/v0/service_test.go +++ b/services/collaboration/pkg/service/grpc/v0/service_test.go @@ -6,7 +6,7 @@ import ( "time" "github.com/cs3org/reva/v2/pkg/utils" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" diff --git a/services/proxy/pkg/middleware/authentication_test.go b/services/proxy/pkg/middleware/authentication_test.go index 25601dbcd9..4b15deb591 100644 --- a/services/proxy/pkg/middleware/authentication_test.go +++ b/services/proxy/pkg/middleware/authentication_test.go @@ -10,7 +10,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/owncloud/ocis/v2/ocis-pkg/log" diff --git a/services/proxy/pkg/middleware/oidc_auth.go b/services/proxy/pkg/middleware/oidc_auth.go index fea68dc5f1..47b9c424d8 100644 --- a/services/proxy/pkg/middleware/oidc_auth.go +++ b/services/proxy/pkg/middleware/oidc_auth.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/oidc" "github.com/pkg/errors" diff --git a/services/proxy/pkg/middleware/oidc_auth_test.go b/services/proxy/pkg/middleware/oidc_auth_test.go index eddb5eb72c..70a047706c 100644 --- a/services/proxy/pkg/middleware/oidc_auth_test.go +++ b/services/proxy/pkg/middleware/oidc_auth_test.go @@ -5,7 +5,7 @@ import ( "net/http/httptest" "time" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/owncloud/ocis/v2/ocis-pkg/log" diff --git a/services/thumbnails/pkg/service/grpc/v0/service.go b/services/thumbnails/pkg/service/grpc/v0/service.go index 7049064b70..e7bde4c5df 100644 --- a/services/thumbnails/pkg/service/grpc/v0/service.go +++ b/services/thumbnails/pkg/service/grpc/v0/service.go @@ -15,7 +15,7 @@ import ( "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/pkg/errors" merrors "go-micro.dev/v4/errors" "google.golang.org/grpc/metadata" diff --git a/services/thumbnails/pkg/service/http/v0/service.go b/services/thumbnails/pkg/service/http/v0/service.go index 175b24230c..eb7446d1eb 100644 --- a/services/thumbnails/pkg/service/http/v0/service.go +++ b/services/thumbnails/pkg/service/http/v0/service.go @@ -3,12 +3,13 @@ package svc import ( "context" "fmt" - "github.com/go-chi/chi/v5" - "github.com/golang-jwt/jwt/v4" - "github.com/riandyrn/otelchi" "net/http" "strconv" + "github.com/go-chi/chi/v5" + "github.com/golang-jwt/jwt/v5" + "github.com/riandyrn/otelchi" + "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/tracing" "github.com/owncloud/ocis/v2/services/thumbnails/pkg/config" diff --git a/services/thumbnails/pkg/service/jwt/jwt.go b/services/thumbnails/pkg/service/jwt/jwt.go index 86e1ff02cc..14df7560b1 100644 --- a/services/thumbnails/pkg/service/jwt/jwt.go +++ b/services/thumbnails/pkg/service/jwt/jwt.go @@ -1,6 +1,6 @@ package jwt -import "github.com/golang-jwt/jwt/v4" +import "github.com/golang-jwt/jwt/v5" // ThumbnailClaims defines the claims for thumb-nailing type ThumbnailClaims struct { diff --git a/vendor/github.com/MicahParks/keyfunc/LICENSE b/vendor/github.com/MicahParks/keyfunc/v2/LICENSE similarity index 100% rename from vendor/github.com/MicahParks/keyfunc/LICENSE rename to vendor/github.com/MicahParks/keyfunc/v2/LICENSE diff --git a/vendor/github.com/MicahParks/keyfunc/README.md b/vendor/github.com/MicahParks/keyfunc/v2/README.md similarity index 81% rename from vendor/github.com/MicahParks/keyfunc/README.md rename to vendor/github.com/MicahParks/keyfunc/v2/README.md index ed07eaa7f1..c04b3dcff8 100644 --- a/vendor/github.com/MicahParks/keyfunc/README.md +++ b/vendor/github.com/MicahParks/keyfunc/v2/README.md @@ -1,35 +1,41 @@ -[![Go Report Card](https://goreportcard.com/badge/github.com/MicahParks/keyfunc)](https://goreportcard.com/report/github.com/MicahParks/keyfunc) [![Go Reference](https://pkg.go.dev/badge/github.com/MicahParks/keyfunc.svg)](https://pkg.go.dev/github.com/MicahParks/keyfunc) +[![Go Report Card](https://goreportcard.com/badge/github.com/MicahParks/keyfunc/v2)](https://goreportcard.com/report/github.com/MicahParks/keyfunc/v2) [![Go Reference](https://pkg.go.dev/badge/github.com/MicahParks/keyfunc/v2.svg)](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2) # keyfunc The purpose of this package is to provide a -[`jwt.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#Keyfunc) for the -[github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) package using a JSON Web Key Set (JWK Set or JWKS) for +[`jwt.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#Keyfunc) for the +[github.com/golang-jwt/jwt/v5](https://github.com/golang-jwt/jwt) package using a JSON Web Key Set (JWK Set or JWKS) for parsing and verifying JSON Web Tokens (JWTs). +The last version to support `github.com/golang-jwt/jwt/v4` +is [`v1.9.0`](https://github.com/MicahParks/keyfunc/releases/tag/v1.9.0). + There is legacy support for `github.com/dgrijalva/jwt-go` and its popular forks. It's in a separate project to keep this project minimal. If your use case supports a legacy fork, please -see: [github.com/MicahParks/compatibility-keyfunc](https://github.com/MicahParks/compatibility-keyfunc). +see: [github.com/MicahParks/compatibility-keyfunc](https://github.com/MicahParks/compatibility-keyfunc). If an updated +to `keyfunc` is needed for `github.com/golang-jwt/jwt/v4` users, it will be placed into this separate project. It's common for an identity provider, such as [Keycloak](https://www.keycloak.org/) or [Amazon Cognito (AWS)](https://aws.amazon.com/cognito/) to expose a JWKS via an HTTPS endpoint. This package has the ability to consume that JWKS and produce a -[`jwt.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#Keyfunc). It is important that a JWKS endpoint is using +[`jwt.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#Keyfunc). It is important that a JWKS endpoint is using HTTPS to ensure the keys are from the correct trusted source. -This repository only depends on: [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) +This repository only depends on: [github.com/golang-jwt/jwt/v5](https://github.com/golang-jwt/jwt) `jwt.Keyfunc` signatures are imported from these, implemented, then exported as methods. ## Supported Algorithms Currently, this package supports JWTs signed with a `kty` that matches one of the following: + * `EC` [Elliptic Curve Digital Signature Algorithm (ECDSA)](https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm) * `RSA` [Rivest–Shamir–Adleman (RSA)](https://en.wikipedia.org/wiki/RSA_(cryptosystem)) * `OKP` [Edwards-curve Digital Signature Algorithm (EdDSA)](https://en.wikipedia.org/wiki/EdDSA) * `OCT` [HMAC](https://en.wikipedia.org/wiki/HMAC), [AES Key Wrap](https://en.wikipedia.org/wiki/Key_Wrap), and others Additionally, the supported `EC` elliptical curve types are below: + * `P-256` * `P-384` * `P-521` @@ -48,13 +54,14 @@ this Go package, please open an issue or pull request. For complete examples, please see the `examples` directory. ```go -import "github.com/MicahParks/keyfunc" +import "github.com/MicahParks/keyfunc/v2" ``` #### A note on read-only keys -The [`JWKS.ReadOnlyKeys`](https://pkg.go.dev/github.com/MicahParks/keyfunc#JWKS.ReadOnlyKeys) method returns a read-only + +The [`JWKS.ReadOnlyKeys`](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2#JWKS.ReadOnlyKeys) method returns a read-only copy of a `map[string]interface{}`. The key to this map is the key ID, `kid`, and the value is the cryptographic key. -This is a useful map for use of keys within a JWKS outside of `github.com/golang-jwt/jwt/v4`. +This is a useful map for use of keys within a JWKS outside of `github.com/golang-jwt/jwt/v5`. The map itself is a copy. So it can be modified safely. However, the values are of type `interface{}`. If these values are modified, it may cause undefined behavior. @@ -62,7 +69,7 @@ are modified, it may cause undefined behavior. ### Preconditions: Acquire the JWKS URL, JSON, or gather cryptographic keys (given keys) A JWKS URL is not required, one can be created directly from JSON with the -[`keyfunc.NewJSON`](https://pkg.go.dev/github.com/MicahParks/keyfunc#NewJSON) function. +[`keyfunc.NewJSON`](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2#NewJSON) function. ```go // Get the JWKS URL from an environment variable. @@ -77,6 +84,7 @@ if jwksURL == "" { ### Step 1: Create the JWKS Via HTTP: + ```go // Create the JWKS from the resource at the given URL. jwks, err := keyfunc.Get(jwksURL, keyfunc.Options{}) // See recommended options in the examples directory. @@ -84,7 +92,9 @@ if err != nil { log.Fatalf("Failed to get the JWKS from the given URL.\nError: %s", err) } ``` + Via JSON: + ```go // Get the JWKS as JSON. var jwksJSON = json.RawMessage(`{"keys":[{"kid":"zXew0UJ1h6Q4CCcd_9wxMzvcp5cEBifH0KWrCz2Kyxc","kty":"RSA","alg":"PS256","use":"sig","n":"wqS81x6fItPUdh1OWCT8p3AuLYgFlpmg61WXp6sp1pVijoyF29GOSaD9xE-vLtegX-5h0BnP7va0bwsOAPdh6SdeVslEifNGHCtID0xNFqHNWcXSt4eLfQKAPFUq0TsEO-8P1QHRq6yeG8JAFaxakkaagLFuV8Vd_21PGJFWhvJodJLhX_-Ym9L8XUpIPps_mQriMUOWDe-5DWjHnDtfV7mgaOxbBvVo3wj8V2Lmo5Li4HabT4MEzeJ6e9IdFo2kj_44Yy9osX-PMPtu8BQz_onPgf0wjrVWt349Rj6OkS8RxlNGYeuIxYZr0TOhP5F-yEPhSXDsKdVTwPf7zAAaKQ","e":"AQAB","x5c":["MIICmzCCAYMCBgF4HR7HNDANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwMzEwMTcwOTE5WhcNMzEwMzEwMTcxMDU5WjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDCpLzXHp8i09R2HU5YJPyncC4tiAWWmaDrVZenqynWlWKOjIXb0Y5JoP3ET68u16Bf7mHQGc/u9rRvCw4A92HpJ15WyUSJ80YcK0gPTE0Woc1ZxdK3h4t9AoA8VSrROwQ77w/VAdGrrJ4bwkAVrFqSRpqAsW5XxV3/bU8YkVaG8mh0kuFf/5ib0vxdSkg+mz+ZCuIxQ5YN77kNaMecO19XuaBo7FsG9WjfCPxXYuajkuLgdptPgwTN4np70h0WjaSP/jhjL2ixf48w+27wFDP+ic+B/TCOtVa3fj1GPo6RLxHGU0Zh64jFhmvRM6E/kX7IQ+FJcOwp1VPA9/vMABopAgMBAAEwDQYJKoZIhvcNAQELBQADggEBALILq1Z4oQNJZEUt24VZcvknsWtQtvPxl3JNcBQgDR5/IMgl5VndRZ9OT56KUqrR5xRsWiCvh5Lgv4fUEzAAo9ToiPLub1SKP063zWrvfgi3YZ19bty0iXFm7l2cpQ3ejFV7WpcdLJE0lapFdPLo6QaRdgNu/1p4vbYg7zSK1fQ0OY5b3ajhAx/bhWlrN685owRbO5/r4rUOa6oo9l4Qn7jUxKUx4rcoe7zUM7qrpOPqKvn0DBp3n1/+9pOZXCjIfZGvYwP5NhzBDCkRzaXcJHlOqWzMBzyovVrzVmUilBcj+EsTYJs0gVXKzduX5zO6YWhFs23lu7AijdkxTY65YM0="],"x5t":"IYIeevIT57t8ppUejM42Bqx6f3I","x5t#S256":"TuOrBy2NcTlFSWuZ8Kh8W8AjQagb4fnfP1SlKMO8-So"},{"kid":"ebJxnm9B3QDBljB5XJWEu72qx6BawDaMAhwz4aKPkQ0","kty":"EC","alg":"ES512","use":"sig","crv":"P-521","x":"YQ95Xj8MTzcHytbU1h8YkCN2kdEQA7ThuZ1ctB9Ekiw6tlM9RwL62eQvzEt4Rz8qN69uRqgU9RzxQOkSU5xVvyo","y":"SMMuP3QnAPHtx7Go2ARsG3NBaySWBLmVvS8s2Ss7Vm_ISWenNbdjKOsY1XvtiQz5scGzWDCEUoZzgV8Ve1mLOV0"},{"kid":"TVAAet63O3xy_KK6_bxVIu7Ra3_z1wlB543Fbwi5VaU","kty":"EC","alg":"ES384","use":"sig","crv":"P-384","x":"Pik2o5as-evijFABH5p6YLXHnWw8iQ_N1ummPY1c_UgG6NO0za-gNOhTz2-tsd_w","y":"e98VSff71k19SY_mHgp3707lgQVrhfVpiGa-sGaKxOWVpxd2jWMhB0Q4RpSRuCp5"},{"kid":"arlUxX4hh56rNO-XdIPhDT7bqBMqcBwNQuP_TnZJNGs","kty":"RSA","alg":"RS512","use":"sig","n":"hhtifu8LL3ICE3BAX5l1KZv6Lni0lhlhBusSfepnpxcb4C_z2U71cQTnLY27kt8WB4bNG6e5_KMx9K3xUdd3euj9MCq8vytwEPieeHE1KXQuhJfLv017lhpK_dRMOHyc-9-50YNdgs_8KWRkrzjjuYrCiO9Iu76n5319e-SC8OPvNUglqxp2N0Sp2ltne2ZrpN8T3OEEXT62TSGmLAVopRGw5gllNVrJfmEyZJCRrBM6s5CQcz8un0FjkAAC4DI6QD-eBL0qG3_NR0hQvR1he2o4BLwjOKH45Pk_jj-eArp-DD6Xq6ABQVb5SNOSdaxl5lnmuotRoY3G5d9YSl-K3w","e":"AQAB","x5c":["MIICmzCCAYMCBgF4HSCcDzANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwMzEwMTcxMTE5WhcNMzEwMzEwMTcxMjU5WjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCGG2J+7wsvcgITcEBfmXUpm/oueLSWGWEG6xJ96menFxvgL/PZTvVxBOctjbuS3xYHhs0bp7n8ozH0rfFR13d66P0wKry/K3AQ+J54cTUpdC6El8u/TXuWGkr91Ew4fJz737nRg12Cz/wpZGSvOOO5isKI70i7vqfnfX175ILw4+81SCWrGnY3RKnaW2d7Zmuk3xPc4QRdPrZNIaYsBWilEbDmCWU1Wsl+YTJkkJGsEzqzkJBzPy6fQWOQAALgMjpAP54EvSobf81HSFC9HWF7ajgEvCM4ofjk+T+OP54Cun4MPperoAFBVvlI05J1rGXmWea6i1Ghjcbl31hKX4rfAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAB7bpwPoL02WGCCVhCsbDkq9GeFUwF01opVyFTijZlTUoTf5RcaR2qAH9/irkLjZeFeyozzC5mGvIVruBwnx/6l4PcAMxKK4YiheFVoO/dytpGMCj6ToNmKpjlXzOLAHelieWIUDtAFSYzENjIO01PyXTGYpxebpQCocJBvppj5HqARS9iNPcqBltMhxWrWmMu81tOG3Y7yd2xsIYXk6KjaoefLeN8Was4BPJ0zR6tTSEm6ZOvSRvlppqh84kz7LmWem7gGHAsY2G3tWBUmOdO/SMNMThqV62yLf7sKsuoE1w06lfmrf6D2zGwoEyz+TT6fdSkc34Yeh7+c01X6nFWU="],"x5t":"geiCPGtT_10T8xGLUK1LA0_YQEE","x5t#S256":"dLp3_QNGwMbYll5VecnR8Q9NSeFVfqJPBTa2_8qf48I"},{"kid":"tW6ae7TomE6_2jooM-sf9N_6lWg7HNtaQXrDsElBzM4","kty":"RSA","alg":"PS512","use":"sig","n":"p32N7jqKfMUB6_dKY1uZ3wizzPlBAXg9XrntfUcwNLRPfTBnshpt4uQBf3T8fexkbzhtR18oHvim-YvcWfC5eLGQmWHYiVwACa_C7oGqx51ijK2LRbUg4TKhnZX2X3Ld9xvr3HsosKh2UXn_Ay8nuvdfH-U6S7btT6a-AIFlt3BpqZP0EOl7rY-ie8nXoA13xX6BoyzYiNcugdYCU6czQcmTIJ1JLS0zohi4aTNehRt-1VMRpIMx7q7Ouq3Zhbi7RcDo-_D8FPRhWc2eEKd-h8ebFTIxEOrkguBIomjEFTf3SfYbOB_h-14v9Q2yz-NzyId3-ujRCQGC0hn-cixe2w","e":"AQAB","x5c":["MIICmzCCAYMCBgF4BKAxqzANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwMzA1MjMwMDEwWhcNMzEwMzA1MjMwMTUwWjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCnfY3uOop8xQHr90pjW5nfCLPM+UEBeD1eue19RzA0tE99MGeyGm3i5AF/dPx97GRvOG1HXyge+Kb5i9xZ8Ll4sZCZYdiJXAAJr8LugarHnWKMrYtFtSDhMqGdlfZfct33G+vceyiwqHZRef8DLye6918f5TpLtu1Ppr4AgWW3cGmpk/QQ6Xutj6J7ydegDXfFfoGjLNiI1y6B1gJTpzNByZMgnUktLTOiGLhpM16FG37VUxGkgzHurs66rdmFuLtFwOj78PwU9GFZzZ4Qp36Hx5sVMjEQ6uSC4EiiaMQVN/dJ9hs4H+H7Xi/1DbLP43PIh3f66NEJAYLSGf5yLF7bAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAHVWNBTExqlg4LTcyhUXI5U0iNPcMIVdKDoGPDc3EPjXyYNyjURX0oZ6b1Wv5t+XGmpZRqJNYb92xraQatIzLEsRn4IrmzViP+dIyFU8BEDubixTxeqx7LSw2j6LIFnZ05XdmWknlksNTlqi4CT6KL+1c24+QU3CcmU3mkQEIPA2yC4SdAB1oXI0jh49uP6a+JrE7JREZGAdwbIpZ1cqV6acPiJW3tOYfLrHwo7KYn3KwJvIBHXgFBNwx7fl2gYNQ0VEGKub3qVwW5RO5R/6Tcla9uZEfEiamms/Pn4hFA1qbsNHtA9IRGVRSmVeBKDxRvo0fxOUXp+NuZxEnhsoP3I="],"x5t":"f1l1fxICz1fe9mI-sSrtc19EDhU","x5t#S256":"NUJWRA4ADpLEg_SMkSoE4FKQN0H1Tlz85L-i7puVcqQ"},{"kid":"Lx1FmayP2YBtxaqS1SKJRJGiXRKnw2ov5WmYIMG-BLE","kty":"RSA","alg":"PS384","use":"sig","n":"q7WM4SnrdzlFSo_A1DRhc-8Ho-pBsfs49kGRbw3O_OKFIUyZrzHaRuovW_QaEAyiO3HX8CNcGPcpHdmpl4DhTGEBLcd6xXtCaa65ct00Mq7ZHCRRCrKLh6lJ0rY9fP8vCV0RBigpkNoRfrqLQQN4VeVFTbGSrDaS0LzPbap0-q5FKXUR-OQmQEtOupXhKFQtbB73tL83YnG6Swl7nXsx54ulEoDzcCCYt7pjCVVp7L9fzI2_ucTdtQclAJVQZGKpsx7vabOJuiMUwuAIz56lOJyXRMePsW8UogwC4FA2A52STsYlhOPsDEW4iIExFVNqs-CGoDGhYLIavaCkZhXM0w","e":"AQAB","x5c":["MIICmzCCAYMCBgF4HR+9XjANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwMzEwMTcxMDIyWhcNMzEwMzEwMTcxMjAyWjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrtYzhKet3OUVKj8DUNGFz7wej6kGx+zj2QZFvDc784oUhTJmvMdpG6i9b9BoQDKI7cdfwI1wY9ykd2amXgOFMYQEtx3rFe0Jprrly3TQyrtkcJFEKsouHqUnStj18/y8JXREGKCmQ2hF+uotBA3hV5UVNsZKsNpLQvM9tqnT6rkUpdRH45CZAS066leEoVC1sHve0vzdicbpLCXudezHni6USgPNwIJi3umMJVWnsv1/Mjb+5xN21ByUAlVBkYqmzHu9ps4m6IxTC4AjPnqU4nJdEx4+xbxSiDALgUDYDnZJOxiWE4+wMRbiIgTEVU2qz4IagMaFgshq9oKRmFczTAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADTgP3SrcG3p9XUB7sM4a2IeY0J4bSEtqlZBuHgdgekYJ5DXETJ3hV/82GjitU50NBup0IJyI9KZ0KCwqHIKC2Jn/6biOpM9Ipk4BtNVzx3qKNsDac9qZmyMpm4V9QuWakajknerhwyynG3siGUntbPmLvf5UKvKtbiKlWS4dBPwfedIUnC85mYEnNKSzSI1NiM6TWHB9zQYkARXlb89sh0HBYs08BfRMyBVM+l3OczIyGeQAfhcL+pxPP/0jqPr1ctHUBj2zXkjZxDw1oJFgeD9GDtPcjc3spB20vsRtQUBlzbJElbGflqWGHJK5l5n7gNd3ZXZT0HJ+wUpPE8EUaM="],"x5t":"fjRYR1986VCLzbaZaw5r25UKahw","x5t#S256":"ZHNHpizlsjD3qSZh7gJQQBu8W9jBL2HR0y7-3u2Wb-g"},{"kid":"gnmAfvmlsi3kKH3VlM1AJ85P2hekQ8ON_XvJqs3xPD8","kty":"RSA","alg":"RS384","use":"sig","n":"qUNQewKl3APQcbpACMNJ2XphPpupt395z6OZvj5CW9tiRXY3J7dqi8U0bWoIhtmmc7Js6hjp-A5W_FVStuXlT1hLyjJsHeu9ZVPnfIl2MnYN83zQBKw8E4mFsVv0UXNvkVPBF_k0yXrz-ABleWLOgFGnkNU9csc3Z5aihHcwRmC_oS7PZ9Vc-l0xBCyF3YRHI-al8ppSHwFreOweF3-JP3poNAXd906_tjX2KlHSJmNqcUNiSfEluyCp02ALlRFKXUQ1HlfSupHcHySDlanfUyIzZgM9ysCvC1vfNdAuwZ44oUBMul_XPxxhzlewL2Y8PtSDLUDWGTIou8M8049D8Q","e":"AQAB","x5c":["MIICmzCCAYMCBgF4BJVfaDANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwMzA1MjI0ODIxWhcNMzEwMzA1MjI1MDAxWjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCpQ1B7AqXcA9BxukAIw0nZemE+m6m3f3nPo5m+PkJb22JFdjcnt2qLxTRtagiG2aZzsmzqGOn4Dlb8VVK25eVPWEvKMmwd671lU+d8iXYydg3zfNAErDwTiYWxW/RRc2+RU8EX+TTJevP4AGV5Ys6AUaeQ1T1yxzdnlqKEdzBGYL+hLs9n1Vz6XTEELIXdhEcj5qXymlIfAWt47B4Xf4k/emg0Bd33Tr+2NfYqUdImY2pxQ2JJ8SW7IKnTYAuVEUpdRDUeV9K6kdwfJIOVqd9TIjNmAz3KwK8LW9810C7BnjihQEy6X9c/HGHOV7AvZjw+1IMtQNYZMii7wzzTj0PxAgMBAAEwDQYJKoZIhvcNAQELBQADggEBABoThxhMd7Xiq4x0GJeoJFv2yDKXCL3dJEAEWtOr2+PqdeJl/ZfOxBXynIvrdtYnQdICztN5ydEgDsZ02piDsxZ+s/0SA0iqjw/MEoBYobmr8V+xwUv+WtRLpTBXqWGMuG7NEtrbjKid0iKLLAOAU4dcHQ49iOF9VLnbTkf1EXp4iphJreaubOXMwT6/JDzQPT1dRR34hlhYeKKzMSA0Cz5aYL1tI+eH12rar0MDczXykLChNS/8MlyTzreEf0siUiS9S1kj/lOZKQDg9E/z8fm5vmHEHzAVwf4ON5iO29tDsqLw7BeJqC4AESjliXIqMrdpFynfPnIsGgf3dnph5BM="],"x5t":"CmRnQVduZWtEsdOC4mauUUsSWxA","x5t#S256":"BvC0LmuM8ZIApN3TQQZWWbGO-d082Ah5d3D6vPvahGw"},{"kid":"CGt0ZWS4Lc5faiKSdi0tU0fjCAdvGROQRGU9iR7tV0A","kty":"EC","alg":"ES256","use":"sig","crv":"P-256","x":"DPW7n9yjfE6Rt-VvVmEdeu4QdW44qifocAPPDxACDDY","y":"-ejsVw8222-hg2dJWx3QV0hE4-I0Ujp7ZsWebE68JE0"},{"kid":"C65q0EKQyhpd1m4fr7SKO2He_nAxgCtAdws64d2BLt8","kty":"RSA","alg":"RS256","use":"sig","n":"ja99ybDrLvw11Z4CvNlDI-kkqJEBpSnvDf0pZF2DvBlvYmeVYL_ChqIe8E9GyHUmLMdtO_jifSgOqE5b8vILwi1kZnJR7N857uEnbWM9YTeevi_RZ-E_hr4frW2NKJ78YGvCzwLKG2GgtSjj0zuTLnSaK8fCGzqXgy6paXNhgHUSZgGwvO0YItpMlyJeqEj1wGTWz1IyA1sguF1cC7K0fojPbPoBwrhvaAeoGRPLraE0rrBsQv8iiLwnRBIez9B1j0NiUG8Iad953Y7UzaKOAw8crIEK45NIK_yxHUpxqcHLjPIcRyIyJGioRyGK7cp-_7iPLOCutQc-u46mom1_ZQ","e":"AQAB","x5c":["MIICmzCCAYMCBgF4BJRpbzANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwMzA1MjI0NzE4WhcNMzEwMzA1MjI0ODU4WjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCNr33JsOsu/DXVngK82UMj6SSokQGlKe8N/SlkXYO8GW9iZ5Vgv8KGoh7wT0bIdSYsx207+OJ9KA6oTlvy8gvCLWRmclHs3znu4SdtYz1hN56+L9Fn4T+Gvh+tbY0onvxga8LPAsobYaC1KOPTO5MudJorx8IbOpeDLqlpc2GAdRJmAbC87Rgi2kyXIl6oSPXAZNbPUjIDWyC4XVwLsrR+iM9s+gHCuG9oB6gZE8utoTSusGxC/yKIvCdEEh7P0HWPQ2JQbwhp33ndjtTNoo4DDxysgQrjk0gr/LEdSnGpwcuM8hxHIjIkaKhHIYrtyn7/uI8s4K61Bz67jqaibX9lAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAHrGJFhVNiQupIwkn2jiW/jBobm9CHUxOwQL5E7WdRz5uaOJ0v62PrynOQE9xim9Qk8bT3q7DThZs66U9bpIk3msKVRgXRfn5FZy1H5RKOlEEFZhGakPqSlC1yPbhUNhHXMs3GTzdGMLtYaGvSy6XM/8/zqVqVwgh6BpbAR9RfiSdyaiNTSBriu+n/tHW934G9J8UIzdfpVcb0Yt9y4o0UgIXt64NtGFq7zmNJijH88AxBZFB6eUUmQQCczebzoAjyYbVOes5gGFzboVWcyLe3iyD0vvsAVHJViXeiGoxhpKnc8ryISpRUBzsKngf5uZo3bnrD9PHLYBoGOHgzII1xw="],"x5t":"5GNr3LeRXHWI4YR8-QTSsF98oTI","x5t#S256":"Dgd0_wZZqvRuf4GEISPNHREX-1ixTMIsrPeGzk0bCxs"}]}`) @@ -95,7 +105,9 @@ if err != nil { log.Fatalf("Failed to create JWKS from JSON.\nError: %s", err) } ``` + Via a given key: + ```go // Get an HMAC key. key := []byte("example secret") @@ -107,11 +119,11 @@ jwks := keyfunc.NewGiven(map[string]keyfunc.GivenKey{ }) ``` -Additional options can be passed to the [`keyfunc.Get`](https://pkg.go.dev/github.com/golang-jwt/jwt/v4/keyfunc#Get) -function. See [`keyfunc.Options`](https://pkg.go.dev/github.com/golang-jwt/jwt/v4/keyfunc#Options) and the additional +Additional options can be passed to the [`keyfunc.Get`](https://pkg.go.dev/github.com/golang-jwt/jwt/v5/keyfunc#Get) +function. See [`keyfunc.Options`](https://pkg.go.dev/github.com/golang-jwt/jwt/v5/keyfunc#Options) and the additional features mentioned at the bottom of this `README.md`. -### Step 2: Use the [`JWKS.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v4/keyfunc#JWKS.Keyfunc) method as the [`jwt.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#Keyfunc) when parsing tokens +### Step 2: Use the [`JWKS.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v5/keyfunc#JWKS.Keyfunc) method as the [`jwt.Keyfunc`](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#Keyfunc) when parsing tokens ```go // Parse the JWT. @@ -121,7 +133,7 @@ if err != nil { } ``` -The [`JWKS.Keyfunc`](https://pkg.go.dev/github.com/MicahParks/keyfunc#JWKS.Keyfunc) method will automatically select the +The [`JWKS.Keyfunc`](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2#JWKS.Keyfunc) method will automatically select the key with the matching `kid` (if present) and return its public key as the correct Go type to its caller. ## Test coverage @@ -133,50 +145,54 @@ would accomplish the same purpose. There are some hard-coded JWTs which are expi coded JWTs cannot check for parsing and validation errors, just errors within the `jwt.Keyfunc` itself. ## Additional features + These features can be configured by populating fields in the -[`keyfunc.Options`](https://pkg.go.dev/github.com/MicahParks/keyfunc#Options) argument to the -[`keyfunc.Get`](https://pkg.go.dev/github.com/MicahParks/keyfunc#Get) function. +[`keyfunc.Options`](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2#Options) argument to the +[`keyfunc.Get`](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2#Get) function. + * A background refresh of the JWKS keys can be performed. - * A custom background refresh interval can be specified. For an example, please see the `examples/interval` - directory. - * A custom background refresh request context timeout can be specified. Defaults to one minute. For an example, - please see the `examples/ctx` directory. - * A custom background refresh error handling function can be specified. If none is specified, errors go unhandled - silently. For an example, please see the `examples/recommended_options` directory. - * A custom rate limit can be specified to prevent too many requests for a JWKS refresh. For an example, please see - the `examples/recommended_options` directory. - * JWTs with a previously unseen `kid` can prompt an automatic refresh of the remote JWKS resource. This should be - paired with `RefreshRateLimit` to prevent abuse. For an example, please see the `examples/recommended_options` - directory. + * A custom background refresh interval can be specified. For an example, please see the `examples/interval` + directory. + * A custom background refresh request context timeout can be specified. Defaults to one minute. For an example, + please see the `examples/ctx` directory. + * A custom background refresh error handling function can be specified. If none is specified, errors go unhandled + silently. For an example, please see the `examples/recommended_options` directory. + * A custom rate limit can be specified to prevent too many requests for a JWKS refresh. For an example, please see + the `examples/recommended_options` directory. + * JWTs with a previously unseen `kid` can prompt an automatic refresh of the remote JWKS resource. This should be + paired with `RefreshRateLimit` to prevent abuse. For an example, please see the `examples/recommended_options` + directory. * A custom HTTP client can be used. * A custom HTTP request factory can be provided to create HTTP requests for the remote JWKS resource. For example, an HTTP header can be added to indicate a User-Agent. * A custom HTTP response extractor can be provided to get the raw JWKS JSON from the `*http.Response`. For example, the HTTP response code could be checked. Implementations are responsible for closing the response body. - * By default, - the [`keyfunc.ResponseExtractorStatusOK`](https://pkg.go.dev/github.com/MicahParks/keyfunc#ResponseExtractorStatusOK) - function is used. The default behavior changed in `v1.4.0`. + * By default, + the [`keyfunc.ResponseExtractorStatusOK`](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2#ResponseExtractorStatusOK) + function is used. The default behavior changed in `v1.4.0`. * A custom whitelist of acceptable JSON Web Key `"use"` parameter values can be specified. Values not whitelisted will - cause an error from the [`.Keyfunc`](https://pkg.go.dev/github.com/MicahParks/keyfunc#JWKS.Keyfunc) method. This + cause an error from the [`.Keyfunc`](https://pkg.go.dev/github.com/MicahParks/keyfunc/v2#JWKS.Keyfunc) method. This whitelist can be disabled with the `JWKUseNoWhitelist` option. - * By default, only JSON Web Keys with a `"use"` parameter value of `"sig"`, an empty string `""`, or a completely - omitted `"use"` parameter will be returned. The default behavior changed in `v1.5.0`. - * This `"use"` whitelisting behavior is only available with `keyfunc.Get`. It is not available with - `keyfunc.NewJSON` or `keyfunc.NewGiven`. Please open a GitHub issue if you would like this feature added to the - other creation methods. + * By default, only JSON Web Keys with a `"use"` parameter value of `"sig"`, an empty string `""`, or a completely + omitted `"use"` parameter will be returned. The default behavior changed in `v1.5.0`. + * This `"use"` whitelisting behavior is only available with `keyfunc.Get`. It is not available with + `keyfunc.NewJSON` or `keyfunc.NewGiven`. Please open a GitHub issue if you would like this feature added to the + other creation methods. * A map of JWT key IDs (`kid`) to keys can be given and used for the `jwt.Keyfunc`. For an example, see the `examples/given` directory. * A copy of the latest raw JWKS `[]byte` can be returned. * Custom cryptographic algorithms can be used. Make sure to - use [`jwt.RegisterSigningMethod`](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#RegisterSigningMethod) before + use [`jwt.RegisterSigningMethod`](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#RegisterSigningMethod) before parsing JWTs. For an example, see the `examples/custom` directory. * The remote JWKS resource can be refreshed manually using the `.Refresh` method. This can bypass the rate limit, if the option is set. * There is support for creating one `jwt.Keyfunc` from multiple JWK Sets through the use of the `keyfunc.GetMultiple`. ## Notes + Trailing padding is required to be removed from base64url encoded keys inside a JWKS. This is because RFC 7517 defines base64url the same as RFC 7515 Section 2: + * https://datatracker.ietf.org/doc/html/rfc7517#section-1.1 * https://datatracker.ietf.org/doc/html/rfc7515#section-2 @@ -188,8 +204,29 @@ before returning the key for signature verification. If the `alg`s do not match, prevent the key being used for signature verification. If the `alg` is not present in the JWK, this check will not occur. +## Related projects + +### [`github.com/MicahParks/jwkset`](https://github.com/MicahParks/jwkset): + +A JWK Set implementation. Currently, it is only server-side assets. `keyfunc` is my JWK Set client-side implementation. +This project has not had a stable release yet. + +### [`github.com/MicahParks/jcp`](https://github.com/MicahParks/jcp): + +A JWK Set client proxy. JCP for short. This project is a standalone service that uses `keyfunc` under the hood. It +primarily exists for these use cases: + +1. The language or shell a program is written in does not have an adequate JWK Set client. Validate JWTs with `curl`? + Why not? +2. Restrictive networking policies prevent a program from accessing the remote JWK Set directly. +3. Many co-located services need to validate JWTs that were signed by a key that lives in a remote JWK Set. + +If you can integrate `keyfunc` directly into your program, you likely don't need JCP. + ## References + This project was built and tested using various RFCs and services. The services are listed below: + * [Keycloak](https://www.keycloak.org/) * [Sample JWKS Service](https://jwks-service.appspot.com/) * connect2id's [Server JWKSet Gen](https://connect2id.com/products/server/docs/config/jwk-set) diff --git a/vendor/github.com/MicahParks/keyfunc/ecdsa.go b/vendor/github.com/MicahParks/keyfunc/v2/ecdsa.go similarity index 100% rename from vendor/github.com/MicahParks/keyfunc/ecdsa.go rename to vendor/github.com/MicahParks/keyfunc/v2/ecdsa.go diff --git a/vendor/github.com/MicahParks/keyfunc/eddsa.go b/vendor/github.com/MicahParks/keyfunc/v2/eddsa.go similarity index 100% rename from vendor/github.com/MicahParks/keyfunc/eddsa.go rename to vendor/github.com/MicahParks/keyfunc/v2/eddsa.go diff --git a/vendor/github.com/MicahParks/keyfunc/example_jwks.json b/vendor/github.com/MicahParks/keyfunc/v2/example_jwks.json similarity index 100% rename from vendor/github.com/MicahParks/keyfunc/example_jwks.json rename to vendor/github.com/MicahParks/keyfunc/v2/example_jwks.json diff --git a/vendor/github.com/MicahParks/keyfunc/get.go b/vendor/github.com/MicahParks/keyfunc/v2/get.go similarity index 94% rename from vendor/github.com/MicahParks/keyfunc/get.go rename to vendor/github.com/MicahParks/keyfunc/v2/get.go index 5dd754b740..7ba613f0b7 100644 --- a/vendor/github.com/MicahParks/keyfunc/get.go +++ b/vendor/github.com/MicahParks/keyfunc/v2/get.go @@ -49,11 +49,21 @@ func Get(jwksURL string, options Options) (jwks *JWKS, err error) { err = jwks.refresh() if err != nil { - return nil, err + if options.TolerateInitialJWKHTTPError { + if jwks.refreshErrorHandler != nil { + jwks.refreshErrorHandler(err) + } + jwks.keys = make(map[string]parsedJWK) + } else { + return nil, err + } } if jwks.refreshInterval != 0 || jwks.refreshUnknownKID { - jwks.ctx, jwks.cancel = context.WithCancel(context.Background()) + if jwks.ctx == nil { + jwks.ctx = context.Background() + } + jwks.ctx, jwks.cancel = context.WithCancel(jwks.ctx) jwks.refreshRequests = make(chan refreshRequest, 1) go jwks.backgroundRefresh() } diff --git a/vendor/github.com/MicahParks/keyfunc/given.go b/vendor/github.com/MicahParks/keyfunc/v2/given.go similarity index 54% rename from vendor/github.com/MicahParks/keyfunc/given.go rename to vendor/github.com/MicahParks/keyfunc/v2/given.go index 68c8abd7f0..f66df814d4 100644 --- a/vendor/github.com/MicahParks/keyfunc/given.go +++ b/vendor/github.com/MicahParks/keyfunc/v2/given.go @@ -42,28 +42,14 @@ func NewGiven(givenKeys map[string]GivenKey) (jwks *JWKS) { } } -// NewGivenCustom creates a new GivenKey given an untyped variable. The key argument is expected to be a supported +// NewGivenCustom creates a new GivenKey given an untyped variable. The key argument is expected to be a type supported // by the jwt package used. // -// See the https://pkg.go.dev/github.com/golang-jwt/jwt/v4#RegisterSigningMethod function for registering an unsupported -// signing method. -// -// Deprecated: This function does not allow the user to specify the JWT's signing algorithm. Use -// NewGivenCustomWithOptions instead. -func NewGivenCustom(key interface{}) (givenKey GivenKey) { - return GivenKey{ - inter: key, - } -} - -// NewGivenCustomWithOptions creates a new GivenKey given an untyped variable. The key argument is expected to be a type -// supported by the jwt package used. -// // Consider the options carefully as each field may have a security implication. // -// See the https://pkg.go.dev/github.com/golang-jwt/jwt/v4#RegisterSigningMethod function for registering an unsupported +// See the https://pkg.go.dev/github.com/golang-jwt/jwt/v5#RegisterSigningMethod function for registering an unsupported // signing method. -func NewGivenCustomWithOptions(key interface{}, options GivenKeyOptions) (givenKey GivenKey) { +func NewGivenCustom(key interface{}, options GivenKeyOptions) (givenKey GivenKey) { return GivenKey{ algorithm: options.Algorithm, inter: key, @@ -72,18 +58,8 @@ func NewGivenCustomWithOptions(key interface{}, options GivenKeyOptions) (givenK // NewGivenECDSA creates a new GivenKey given an ECDSA public key. // -// Deprecated: This function does not allow the user to specify the JWT's signing algorithm. Use -// NewGivenECDSACustomWithOptions instead. -func NewGivenECDSA(key *ecdsa.PublicKey) (givenKey GivenKey) { - return GivenKey{ - inter: key, - } -} - -// NewGivenECDSACustomWithOptions creates a new GivenKey given an ECDSA public key. -// // Consider the options carefully as each field may have a security implication. -func NewGivenECDSACustomWithOptions(key *ecdsa.PublicKey, options GivenKeyOptions) (givenKey GivenKey) { +func NewGivenECDSA(key *ecdsa.PublicKey, options GivenKeyOptions) (givenKey GivenKey) { return GivenKey{ algorithm: options.Algorithm, inter: key, @@ -92,18 +68,8 @@ func NewGivenECDSACustomWithOptions(key *ecdsa.PublicKey, options GivenKeyOption // NewGivenEdDSA creates a new GivenKey given an EdDSA public key. // -// Deprecated: This function does not allow the user to specify the JWT's signing algorithm. Use -// NewGivenEdDSACustomWithOptions instead. -func NewGivenEdDSA(key ed25519.PublicKey) (givenKey GivenKey) { - return GivenKey{ - inter: key, - } -} - -// NewGivenEdDSACustomWithOptions creates a new GivenKey given an EdDSA public key. -// // Consider the options carefully as each field may have a security implication. -func NewGivenEdDSACustomWithOptions(key ed25519.PublicKey, options GivenKeyOptions) (givenKey GivenKey) { +func NewGivenEdDSA(key ed25519.PublicKey, options GivenKeyOptions) (givenKey GivenKey) { return GivenKey{ algorithm: options.Algorithm, inter: key, @@ -112,18 +78,8 @@ func NewGivenEdDSACustomWithOptions(key ed25519.PublicKey, options GivenKeyOptio // NewGivenHMAC creates a new GivenKey given an HMAC key in a byte slice. // -// Deprecated: This function does not allow the user to specify the JWT's signing algorithm. Use -// NewGivenHMACCustomWithOptions instead. -func NewGivenHMAC(key []byte) (givenKey GivenKey) { - return GivenKey{ - inter: key, - } -} - -// NewGivenHMACCustomWithOptions creates a new GivenKey given an HMAC key in a byte slice. -// // Consider the options carefully as each field may have a security implication. -func NewGivenHMACCustomWithOptions(key []byte, options GivenKeyOptions) (givenKey GivenKey) { +func NewGivenHMAC(key []byte, options GivenKeyOptions) (givenKey GivenKey) { return GivenKey{ algorithm: options.Algorithm, inter: key, @@ -132,18 +88,8 @@ func NewGivenHMACCustomWithOptions(key []byte, options GivenKeyOptions) (givenKe // NewGivenRSA creates a new GivenKey given an RSA public key. // -// Deprecated: This function does not allow the user to specify the JWT's signing algorithm. Use -// NewGivenRSACustomWithOptions instead. -func NewGivenRSA(key *rsa.PublicKey) (givenKey GivenKey) { - return GivenKey{ - inter: key, - } -} - -// NewGivenRSACustomWithOptions creates a new GivenKey given an RSA public key. -// // Consider the options carefully as each field may have a security implication. -func NewGivenRSACustomWithOptions(key *rsa.PublicKey, options GivenKeyOptions) (givenKey GivenKey) { +func NewGivenRSA(key *rsa.PublicKey, options GivenKeyOptions) (givenKey GivenKey) { return GivenKey{ algorithm: options.Algorithm, inter: key, diff --git a/vendor/github.com/MicahParks/keyfunc/jwks.go b/vendor/github.com/MicahParks/keyfunc/v2/jwks.go similarity index 100% rename from vendor/github.com/MicahParks/keyfunc/jwks.go rename to vendor/github.com/MicahParks/keyfunc/v2/jwks.go diff --git a/vendor/github.com/MicahParks/keyfunc/keyfunc.go b/vendor/github.com/MicahParks/keyfunc/v2/keyfunc.go similarity index 90% rename from vendor/github.com/MicahParks/keyfunc/keyfunc.go rename to vendor/github.com/MicahParks/keyfunc/v2/keyfunc.go index 1f082bda00..ae62503f72 100644 --- a/vendor/github.com/MicahParks/keyfunc/keyfunc.go +++ b/vendor/github.com/MicahParks/keyfunc/v2/keyfunc.go @@ -6,7 +6,7 @@ import ( "fmt" "strings" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" ) var ( @@ -14,7 +14,7 @@ var ( ErrKID = errors.New("the JWT has an invalid kid") ) -// Keyfunc matches the signature of github.com/golang-jwt/jwt/v4's jwt.Keyfunc function. +// Keyfunc matches the signature of github.com/golang-jwt/jwt/v5's jwt.Keyfunc function. func (j *JWKS) Keyfunc(token *jwt.Token) (interface{}, error) { kid, alg, err := kidAlg(token) if err != nil { @@ -23,6 +23,7 @@ func (j *JWKS) Keyfunc(token *jwt.Token) (interface{}, error) { return j.getKey(alg, kid) } +// Keyfunc matches the signature of github.com/golang-jwt/jwt/v5's jwt.Keyfunc function. func (m *MultipleJWKS) Keyfunc(token *jwt.Token) (interface{}, error) { return m.keySelector(m, token) } diff --git a/vendor/github.com/MicahParks/keyfunc/multiple.go b/vendor/github.com/MicahParks/keyfunc/v2/multiple.go similarity index 78% rename from vendor/github.com/MicahParks/keyfunc/multiple.go rename to vendor/github.com/MicahParks/keyfunc/v2/multiple.go index 61ea30b80a..a7ff6fab8f 100644 --- a/vendor/github.com/MicahParks/keyfunc/multiple.go +++ b/vendor/github.com/MicahParks/keyfunc/v2/multiple.go @@ -4,11 +4,11 @@ import ( "errors" "fmt" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" ) // ErrMultipleJWKSSize is returned when the number of JWKS given are not enough to make a MultipleJWKS. -var ErrMultipleJWKSSize = errors.New("multiple JWKS must have two or more remote JWK Set resources") +var ErrMultipleJWKSSize = errors.New("multiple JWKS must have one or more remote JWK Set resources") // MultipleJWKS manages multiple JWKS and has a field for jwt.Keyfunc. type MultipleJWKS struct { @@ -16,14 +16,14 @@ type MultipleJWKS struct { sets map[string]*JWKS // No lock is required because this map is read-only after initialization. } -// GetMultiple creates a new MultipleJWKS. A map of length two or more JWKS URLs to Options is required. +// GetMultiple creates a new MultipleJWKS. A map of length one or more JWKS URLs to Options is required. // // Be careful when choosing Options for each JWKS in the map. If RefreshUnknownKID is set to true for all JWKS in the // map then many refresh requests would take place each time a JWT is processed, this should be rate limited by // RefreshRateLimit. func GetMultiple(multiple map[string]Options, options MultipleOptions) (multiJWKS *MultipleJWKS, err error) { - if multiple == nil || len(multiple) < 2 { - return nil, fmt.Errorf("multiple JWKS must have two or more remote JWK Set resources: %w", ErrMultipleJWKSSize) + if len(multiple) < 1 { + return nil, fmt.Errorf("multiple JWKS must have one or more remote JWK Set resources: %w", ErrMultipleJWKSSize) } if options.KeySelector == nil { @@ -46,6 +46,8 @@ func GetMultiple(multiple map[string]Options, options MultipleOptions) (multiJWK return multiJWKS, nil } +// JWKSets returns a copy of the map of JWK Sets. The map itself is a copy, but the JWKS are not and should be treated +// as read-only. func (m *MultipleJWKS) JWKSets() map[string]*JWKS { sets := make(map[string]*JWKS, len(m.sets)) for u, jwks := range m.sets { @@ -54,6 +56,7 @@ func (m *MultipleJWKS) JWKSets() map[string]*JWKS { return sets } +// KeySelectorFirst returns the first key found in the multiple JWK Sets. func KeySelectorFirst(multiJWKS *MultipleJWKS, token *jwt.Token) (key interface{}, err error) { kid, alg, err := kidAlg(token) if err != nil { diff --git a/vendor/github.com/MicahParks/keyfunc/oct.go b/vendor/github.com/MicahParks/keyfunc/v2/oct.go similarity index 100% rename from vendor/github.com/MicahParks/keyfunc/oct.go rename to vendor/github.com/MicahParks/keyfunc/v2/oct.go diff --git a/vendor/github.com/MicahParks/keyfunc/options.go b/vendor/github.com/MicahParks/keyfunc/v2/options.go similarity index 90% rename from vendor/github.com/MicahParks/keyfunc/options.go rename to vendor/github.com/MicahParks/keyfunc/v2/options.go index cc4cf5efe6..6291ffb8fc 100644 --- a/vendor/github.com/MicahParks/keyfunc/options.go +++ b/vendor/github.com/MicahParks/keyfunc/v2/options.go @@ -9,7 +9,7 @@ import ( "net/http" "time" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" ) // ErrInvalidHTTPStatusCode indicates that the HTTP status code is invalid. @@ -17,7 +17,7 @@ var ErrInvalidHTTPStatusCode = errors.New("invalid HTTP status code") // Options represents the configuration options for a JWKS. // -// If RefreshInterval and or RefreshUnknownKID is not nil, then a background goroutine will be launched to refresh the +// If either RefreshInterval is non-zero or RefreshUnknownKID is true, then a background goroutine will be launched to refresh the // remote JWKS under the specified circumstances. // // When using a background refresh goroutine, make sure to use RefreshRateLimit if paired with RefreshUnknownKID. Also @@ -54,7 +54,7 @@ type Options struct { // if a background refresh goroutine is active. RefreshErrorHandler ErrorHandler - // RefreshInterval is the duration to refresh the JWKS in the background via a new HTTP request. If this is not nil, + // RefreshInterval is the duration to refresh the JWKS in the background via a new HTTP request. If this is not zero, // then a background goroutine will be used to refresh the JWKS once per the given interval. Make sure to call the // JWKS.EndBackground method to end this goroutine when it's no longer needed. RefreshInterval time.Duration @@ -84,6 +84,14 @@ type Options struct { // ResponseExtractor consumes a *http.Response and produces the raw JSON for the JWKS. By default, the // ResponseExtractorStatusOK function is used. The default behavior changed in v1.4.0. ResponseExtractor func(ctx context.Context, resp *http.Response) (json.RawMessage, error) + + // TolerateInitialJWKHTTPError will tolerate any error from the initial HTTP JWKS request. If an error occurs, + // the RefreshErrorHandler will be given the error. The program will continue to run as if the error did not occur + // and a valid JWK Set with no keys was received in the response. This allows for the background goroutine to + // request the JWKS at a later time. + // + // It does not make sense to mark this field as true unless the background refresh goroutine is active. + TolerateInitialJWKHTTPError bool } // MultipleOptions is used to configure the behavior when multiple JWKS are used by MultipleJWKS. diff --git a/vendor/github.com/MicahParks/keyfunc/rsa.go b/vendor/github.com/MicahParks/keyfunc/v2/rsa.go similarity index 100% rename from vendor/github.com/MicahParks/keyfunc/rsa.go rename to vendor/github.com/MicahParks/keyfunc/v2/rsa.go diff --git a/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md b/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md index 6ad1c22bbe..ff9c57e1d8 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md +++ b/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md @@ -17,7 +17,7 @@ and corresponding updates for existing programs. ## Parsing and Validation Options -Under the hood, a new `validator` struct takes care of validating the claims. A +Under the hood, a new `Validator` struct takes care of validating the claims. A long awaited feature has been the option to fine-tune the validation of tokens. This is now possible with several `ParserOption` functions that can be appended to most `Parse` functions, such as `ParseWithClaims`. The most important options @@ -68,6 +68,16 @@ type Claims interface { } ``` +Users that previously directly called the `Valid` function on their claims, +e.g., to perform validation independently of parsing/verifying a token, can now +use the `jwt.NewValidator` function to create a `Validator` independently of the +`Parser`. + +```go +var v = jwt.NewValidator(jwt.WithLeeway(5*time.Second)) +v.Validate(myClaims) +``` + ### Supported Claim Types and Removal of `StandardClaims` The two standard claim types supported by this library, `MapClaims` and @@ -169,7 +179,7 @@ be a drop-in replacement, if you're having troubles migrating, please open an issue. You can replace all occurrences of `github.com/dgrijalva/jwt-go` or -`github.com/golang-jwt/jwt` with `github.com/golang-jwt/jwt/v5`, either manually +`github.com/golang-jwt/jwt` with `github.com/golang-jwt/jwt/v4`, either manually or by using tools such as `sed` or `gofmt`. And then you'd typically run: diff --git a/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go b/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go index 4ccae2a857..c929e4a02f 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go +++ b/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go @@ -62,7 +62,7 @@ func (m *SigningMethodECDSA) Verify(signingString string, sig []byte, key interf case *ecdsa.PublicKey: ecdsaKey = k default: - return ErrInvalidKeyType + return newError("ECDSA verify expects *ecdsa.PublicKey", ErrInvalidKeyType) } if len(sig) != 2*m.KeySize { @@ -96,7 +96,7 @@ func (m *SigningMethodECDSA) Sign(signingString string, key interface{}) ([]byte case *ecdsa.PrivateKey: ecdsaKey = k default: - return nil, ErrInvalidKeyType + return nil, newError("ECDSA sign expects *ecdsa.PrivateKey", ErrInvalidKeyType) } // Create the hasher diff --git a/vendor/github.com/golang-jwt/jwt/v5/ed25519.go b/vendor/github.com/golang-jwt/jwt/v5/ed25519.go index 3db00e4a23..c2138119e5 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/ed25519.go +++ b/vendor/github.com/golang-jwt/jwt/v5/ed25519.go @@ -1,11 +1,10 @@ package jwt import ( - "errors" - "crypto" "crypto/ed25519" "crypto/rand" + "errors" ) var ( @@ -39,7 +38,7 @@ func (m *SigningMethodEd25519) Verify(signingString string, sig []byte, key inte var ok bool if ed25519Key, ok = key.(ed25519.PublicKey); !ok { - return ErrInvalidKeyType + return newError("Ed25519 verify expects ed25519.PublicKey", ErrInvalidKeyType) } if len(ed25519Key) != ed25519.PublicKeySize { @@ -61,7 +60,7 @@ func (m *SigningMethodEd25519) Sign(signingString string, key interface{}) ([]by var ok bool if ed25519Key, ok = key.(crypto.Signer); !ok { - return nil, ErrInvalidKeyType + return nil, newError("Ed25519 sign expects crypto.Signer", ErrInvalidKeyType) } if _, ok := ed25519Key.Public().(ed25519.PublicKey); !ok { diff --git a/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go b/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go index 3afb04e648..2ad542f00c 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go +++ b/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go @@ -22,7 +22,7 @@ func (je joinedError) Is(err error) bool { // wrappedErrors is a workaround for wrapping multiple errors in environments // where Go 1.20 is not available. It basically uses the already implemented -// functionatlity of joinedError to handle multiple errors with supplies a +// functionality of joinedError to handle multiple errors with supplies a // custom error message that is identical to the one we produce in Go 1.20 using // multiple %w directives. type wrappedErrors struct { diff --git a/vendor/github.com/golang-jwt/jwt/v5/hmac.go b/vendor/github.com/golang-jwt/jwt/v5/hmac.go index 91b688ba9f..aca600ce1b 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/hmac.go +++ b/vendor/github.com/golang-jwt/jwt/v5/hmac.go @@ -59,7 +59,7 @@ func (m *SigningMethodHMAC) Verify(signingString string, sig []byte, key interfa // Verify the key is the right type keyBytes, ok := key.([]byte) if !ok { - return ErrInvalidKeyType + return newError("HMAC verify expects []byte", ErrInvalidKeyType) } // Can we use the specified hashing method? @@ -100,5 +100,5 @@ func (m *SigningMethodHMAC) Sign(signingString string, key interface{}) ([]byte, return hasher.Sum(nil), nil } - return nil, ErrInvalidKeyType + return nil, newError("HMAC sign expects []byte", ErrInvalidKeyType) } diff --git a/vendor/github.com/golang-jwt/jwt/v5/none.go b/vendor/github.com/golang-jwt/jwt/v5/none.go index c93daa5849..685c2ea306 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/none.go +++ b/vendor/github.com/golang-jwt/jwt/v5/none.go @@ -32,7 +32,7 @@ func (m *signingMethodNone) Verify(signingString string, sig []byte, key interfa return NoneSignatureTypeDisallowedError } // If signing method is none, signature must be an empty string - if string(sig) != "" { + if len(sig) != 0 { return newError("'none' signing method with non-empty signature", ErrTokenUnverifiable) } diff --git a/vendor/github.com/golang-jwt/jwt/v5/parser.go b/vendor/github.com/golang-jwt/jwt/v5/parser.go index f4386fbaac..ecf99af78f 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/parser.go +++ b/vendor/github.com/golang-jwt/jwt/v5/parser.go @@ -18,7 +18,7 @@ type Parser struct { // Skip claims validation during token parsing. skipClaimsValidation bool - validator *validator + validator *Validator decodeStrict bool @@ -28,7 +28,7 @@ type Parser struct { // NewParser creates a new Parser with the specified options func NewParser(options ...ParserOption) *Parser { p := &Parser{ - validator: &validator{}, + validator: &Validator{}, } // Loop through our parsing options and apply them @@ -74,24 +74,40 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf } } - // Lookup key - var key interface{} - if keyFunc == nil { - // keyFunc was not provided. short circuiting validation - return token, newError("no keyfunc was provided", ErrTokenUnverifiable) - } - if key, err = keyFunc(token); err != nil { - return token, newError("error while executing keyfunc", ErrTokenUnverifiable, err) - } - // Decode signature token.Signature, err = p.DecodeSegment(parts[2]) if err != nil { return token, newError("could not base64 decode signature", ErrTokenMalformed, err) } + text := strings.Join(parts[0:2], ".") - // Perform signature validation - if err = token.Method.Verify(strings.Join(parts[0:2], "."), token.Signature, key); err != nil { + // Lookup key(s) + if keyFunc == nil { + // keyFunc was not provided. short circuiting validation + return token, newError("no keyfunc was provided", ErrTokenUnverifiable) + } + + got, err := keyFunc(token) + if err != nil { + return token, newError("error while executing keyfunc", ErrTokenUnverifiable, err) + } + + switch have := got.(type) { + case VerificationKeySet: + if len(have.Keys) == 0 { + return token, newError("keyfunc returned empty verification key set", ErrTokenUnverifiable) + } + // Iterate through keys and verify signature, skipping the rest when a match is found. + // Return the last error if no match is found. + for _, key := range have.Keys { + if err = token.Method.Verify(text, token.Signature, key); err == nil { + break + } + } + default: + err = token.Method.Verify(text, token.Signature, have) + } + if err != nil { return token, newError("", ErrTokenSignatureInvalid, err) } @@ -99,7 +115,7 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf if !p.skipClaimsValidation { // Make sure we have at least a default validator if p.validator == nil { - p.validator = newValidator() + p.validator = NewValidator() } if err := p.validator.Validate(claims); err != nil { @@ -117,8 +133,8 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // // WARNING: Don't use this method unless you know what you're doing. // -// It's only ever useful in cases where you know the signature is valid (because it has -// been checked previously in the stack) and you want to extract values from it. +// It's only ever useful in cases where you know the signature is valid (since it has already +// been or will be checked elsewhere in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { parts = strings.Split(tokenString, ".") if len(parts) != 3 { @@ -130,9 +146,6 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke // parse Header var headerBytes []byte if headerBytes, err = p.DecodeSegment(parts[0]); err != nil { - if strings.HasPrefix(strings.ToLower(tokenString), "bearer ") { - return token, parts, newError("tokenstring should not contain 'bearer '", ErrTokenMalformed) - } return token, parts, newError("could not base64 decode header", ErrTokenMalformed, err) } if err = json.Unmarshal(headerBytes, &token.Header); err != nil { @@ -140,23 +153,33 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke } // parse Claims - var claimBytes []byte token.Claims = claims - if claimBytes, err = p.DecodeSegment(parts[1]); err != nil { + claimBytes, err := p.DecodeSegment(parts[1]) + if err != nil { return token, parts, newError("could not base64 decode claim", ErrTokenMalformed, err) } - dec := json.NewDecoder(bytes.NewBuffer(claimBytes)) - if p.useJSONNumber { - dec.UseNumber() - } - // JSON Decode. Special case for map type to avoid weird pointer behavior - if c, ok := token.Claims.(MapClaims); ok { - err = dec.Decode(&c) + + // If `useJSONNumber` is enabled then we must use *json.Decoder to decode + // the claims. However, this comes with a performance penalty so only use + // it if we must and, otherwise, simple use json.Unmarshal. + if !p.useJSONNumber { + // JSON Unmarshal. Special case for map type to avoid weird pointer behavior. + if c, ok := token.Claims.(MapClaims); ok { + err = json.Unmarshal(claimBytes, &c) + } else { + err = json.Unmarshal(claimBytes, &claims) + } } else { - err = dec.Decode(&claims) + dec := json.NewDecoder(bytes.NewBuffer(claimBytes)) + dec.UseNumber() + // JSON Decode. Special case for map type to avoid weird pointer behavior. + if c, ok := token.Claims.(MapClaims); ok { + err = dec.Decode(&c) + } else { + err = dec.Decode(&claims) + } } - // Handle decode error if err != nil { return token, parts, newError("could not JSON decode claim", ErrTokenMalformed, err) } diff --git a/vendor/github.com/golang-jwt/jwt/v5/parser_option.go b/vendor/github.com/golang-jwt/jwt/v5/parser_option.go index 1b5af970f6..88a780fbd4 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/parser_option.go +++ b/vendor/github.com/golang-jwt/jwt/v5/parser_option.go @@ -58,6 +58,14 @@ func WithIssuedAt() ParserOption { } } +// WithExpirationRequired returns the ParserOption to make exp claim required. +// By default exp claim is optional. +func WithExpirationRequired() ParserOption { + return func(p *Parser) { + p.validator.requireExp = true + } +} + // WithAudience configures the validator to require the specified audience in // the `aud` claim. Validation will fail if the audience is not listed in the // token or the `aud` claim is missing. diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa.go b/vendor/github.com/golang-jwt/jwt/v5/rsa.go index daff094313..83cbee6ae2 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/rsa.go +++ b/vendor/github.com/golang-jwt/jwt/v5/rsa.go @@ -51,7 +51,7 @@ func (m *SigningMethodRSA) Verify(signingString string, sig []byte, key interfac var ok bool if rsaKey, ok = key.(*rsa.PublicKey); !ok { - return ErrInvalidKeyType + return newError("RSA verify expects *rsa.PublicKey", ErrInvalidKeyType) } // Create hasher @@ -73,7 +73,7 @@ func (m *SigningMethodRSA) Sign(signingString string, key interface{}) ([]byte, // Validate type of key if rsaKey, ok = key.(*rsa.PrivateKey); !ok { - return nil, ErrInvalidKey + return nil, newError("RSA sign expects *rsa.PrivateKey", ErrInvalidKeyType) } // Create the hasher diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go b/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go index 9599f0a46c..28c386ec43 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go +++ b/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go @@ -88,7 +88,7 @@ func (m *SigningMethodRSAPSS) Verify(signingString string, sig []byte, key inter case *rsa.PublicKey: rsaKey = k default: - return ErrInvalidKey + return newError("RSA-PSS verify expects *rsa.PublicKey", ErrInvalidKeyType) } // Create hasher @@ -115,7 +115,7 @@ func (m *SigningMethodRSAPSS) Sign(signingString string, key interface{}) ([]byt case *rsa.PrivateKey: rsaKey = k default: - return nil, ErrInvalidKeyType + return nil, newError("RSA-PSS sign expects *rsa.PrivateKey", ErrInvalidKeyType) } // Create the hasher diff --git a/vendor/github.com/golang-jwt/jwt/v5/token.go b/vendor/github.com/golang-jwt/jwt/v5/token.go index c8ad7c7834..352873a2d9 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/token.go +++ b/vendor/github.com/golang-jwt/jwt/v5/token.go @@ -1,6 +1,7 @@ package jwt import ( + "crypto" "encoding/base64" "encoding/json" ) @@ -9,8 +10,21 @@ import ( // the key for verification. The function receives the parsed, but unverified // Token. This allows you to use properties in the Header of the token (such as // `kid`) to identify which key to use. +// +// The returned interface{} may be a single key or a VerificationKeySet containing +// multiple keys. type Keyfunc func(*Token) (interface{}, error) +// VerificationKey represents a public or secret key for verifying a token's signature. +type VerificationKey interface { + crypto.PublicKey | []uint8 +} + +// VerificationKeySet is a set of public or secret keys. It is used by the parser to verify a token. +type VerificationKeySet struct { + Keys []VerificationKey +} + // Token represents a JWT Token. Different fields will be used depending on // whether you're creating or parsing/verifying a token. type Token struct { diff --git a/vendor/github.com/golang-jwt/jwt/v5/types.go b/vendor/github.com/golang-jwt/jwt/v5/types.go index b82b38867d..b2655a9e6d 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/types.go +++ b/vendor/github.com/golang-jwt/jwt/v5/types.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "math" - "reflect" "strconv" "time" ) @@ -121,14 +120,14 @@ func (s *ClaimStrings) UnmarshalJSON(data []byte) (err error) { for _, vv := range v { vs, ok := vv.(string) if !ok { - return &json.UnsupportedTypeError{Type: reflect.TypeOf(vv)} + return ErrInvalidType } aud = append(aud, vs) } case nil: return nil default: - return &json.UnsupportedTypeError{Type: reflect.TypeOf(v)} + return ErrInvalidType } *s = aud diff --git a/vendor/github.com/golang-jwt/jwt/v5/validator.go b/vendor/github.com/golang-jwt/jwt/v5/validator.go index 3850438939..008ecd8712 100644 --- a/vendor/github.com/golang-jwt/jwt/v5/validator.go +++ b/vendor/github.com/golang-jwt/jwt/v5/validator.go @@ -28,13 +28,12 @@ type ClaimsValidator interface { Validate() error } -// validator is the core of the new Validation API. It is automatically used by +// Validator is the core of the new Validation API. It is automatically used by // a [Parser] during parsing and can be modified with various parser options. // -// Note: This struct is intentionally not exported (yet) as we want to -// internally finalize its API. In the future, we might make it publicly -// available. -type validator struct { +// The [NewValidator] function should be used to create an instance of this +// struct. +type Validator struct { // leeway is an optional leeway that can be provided to account for clock skew. leeway time.Duration @@ -42,6 +41,9 @@ type validator struct { // validation. If unspecified, this defaults to time.Now. timeFunc func() time.Time + // requireExp specifies whether the exp claim is required + requireExp bool + // verifyIat specifies whether the iat (Issued At) claim will be verified. // According to https://www.rfc-editor.org/rfc/rfc7519#section-4.1.6 this // only specifies the age of the token, but no validation check is @@ -62,16 +64,28 @@ type validator struct { expectedSub string } -// newValidator can be used to create a stand-alone validator with the supplied +// NewValidator can be used to create a stand-alone validator with the supplied // options. This validator can then be used to validate already parsed claims. -func newValidator(opts ...ParserOption) *validator { +// +// Note: Under normal circumstances, explicitly creating a validator is not +// needed and can potentially be dangerous; instead functions of the [Parser] +// class should be used. +// +// The [Validator] is only checking the *validity* of the claims, such as its +// expiration time, but it does NOT perform *signature verification* of the +// token. +func NewValidator(opts ...ParserOption) *Validator { p := NewParser(opts...) return p.validator } // Validate validates the given claims. It will also perform any custom // validation if claims implements the [ClaimsValidator] interface. -func (v *validator) Validate(claims Claims) error { +// +// Note: It will NOT perform any *signature verification* on the token that +// contains the claims and expects that the [Claim] was already successfully +// verified. +func (v *Validator) Validate(claims Claims) error { var ( now time.Time errs []error = make([]error, 0, 6) @@ -86,8 +100,9 @@ func (v *validator) Validate(claims Claims) error { } // We always need to check the expiration time, but usage of the claim - // itself is OPTIONAL. - if err = v.verifyExpiresAt(claims, now, false); err != nil { + // itself is OPTIONAL by default. requireExp overrides this behavior + // and makes the exp claim mandatory. + if err = v.verifyExpiresAt(claims, now, v.requireExp); err != nil { errs = append(errs, err) } @@ -149,7 +164,7 @@ func (v *validator) Validate(claims Claims) error { // // Additionally, if any error occurs while retrieving the claim, e.g., when its // the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *validator) verifyExpiresAt(claims Claims, cmp time.Time, required bool) error { +func (v *Validator) verifyExpiresAt(claims Claims, cmp time.Time, required bool) error { exp, err := claims.GetExpirationTime() if err != nil { return err @@ -170,7 +185,7 @@ func (v *validator) verifyExpiresAt(claims Claims, cmp time.Time, required bool) // // Additionally, if any error occurs while retrieving the claim, e.g., when its // the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *validator) verifyIssuedAt(claims Claims, cmp time.Time, required bool) error { +func (v *Validator) verifyIssuedAt(claims Claims, cmp time.Time, required bool) error { iat, err := claims.GetIssuedAt() if err != nil { return err @@ -191,7 +206,7 @@ func (v *validator) verifyIssuedAt(claims Claims, cmp time.Time, required bool) // // Additionally, if any error occurs while retrieving the claim, e.g., when its // the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *validator) verifyNotBefore(claims Claims, cmp time.Time, required bool) error { +func (v *Validator) verifyNotBefore(claims Claims, cmp time.Time, required bool) error { nbf, err := claims.GetNotBefore() if err != nil { return err @@ -211,7 +226,7 @@ func (v *validator) verifyNotBefore(claims Claims, cmp time.Time, required bool) // // Additionally, if any error occurs while retrieving the claim, e.g., when its // the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *validator) verifyAudience(claims Claims, cmp string, required bool) error { +func (v *Validator) verifyAudience(claims Claims, cmp string, required bool) error { aud, err := claims.GetAudience() if err != nil { return err @@ -247,7 +262,7 @@ func (v *validator) verifyAudience(claims Claims, cmp string, required bool) err // // Additionally, if any error occurs while retrieving the claim, e.g., when its // the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *validator) verifyIssuer(claims Claims, cmp string, required bool) error { +func (v *Validator) verifyIssuer(claims Claims, cmp string, required bool) error { iss, err := claims.GetIssuer() if err != nil { return err @@ -267,7 +282,7 @@ func (v *validator) verifyIssuer(claims Claims, cmp string, required bool) error // // Additionally, if any error occurs while retrieving the claim, e.g., when its // the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *validator) verifySubject(claims Claims, cmp string, required bool) error { +func (v *Validator) verifySubject(claims Claims, cmp string, required bool) error { sub, err := claims.GetSubject() if err != nil { return err diff --git a/vendor/modules.txt b/vendor/modules.txt index f991003070..576d75cd22 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -27,9 +27,9 @@ github.com/Masterminds/semver # github.com/Masterminds/sprig v2.22.0+incompatible ## explicit github.com/Masterminds/sprig -# github.com/MicahParks/keyfunc v1.9.0 -## explicit; go 1.16 -github.com/MicahParks/keyfunc +# github.com/MicahParks/keyfunc/v2 v2.1.0 +## explicit; go 1.18 +github.com/MicahParks/keyfunc/v2 # github.com/Microsoft/go-winio v0.6.2 ## explicit; go 1.21 github.com/Microsoft/go-winio @@ -1069,7 +1069,7 @@ github.com/golang-jwt/jwt # github.com/golang-jwt/jwt/v4 v4.5.0 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 -# github.com/golang-jwt/jwt/v5 v5.0.0 +# github.com/golang-jwt/jwt/v5 v5.2.1 ## explicit; go 1.18 github.com/golang-jwt/jwt/v5 # github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 From 78c6046cf7592febd1e03aa5a2f9ea15cb91b326 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 26 Aug 2024 15:50:15 +0200 Subject: [PATCH 59/71] Update README.md References: #9490 (set the configured protocol transport for service metadata) As discussed with @butonic, adding that _configurable service endpoints_ are currently experimental. --- services/gateway/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/gateway/README.md b/services/gateway/README.md index 8d4598e612..c167cb33dd 100644 --- a/services/gateway/README.md +++ b/services/gateway/README.md @@ -31,7 +31,10 @@ Store specific notes: - When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus. - When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc. -## Service endpoints +## Service Endpoints + +**IMPORTANT**\ +This functionality is currently experimental. The gateway acts as a proxy for other CS3 services. As such it has to forward requests to a lot of services and needs to establish connections by looking up the IP address using the service registry. Instead of using the service registry each endpoint can also be configured to use the grpc `dns://` or `kubernetes://` URLs, which might be useful when running in kubernetes. @@ -95,9 +98,7 @@ SHARING_PUBLIC_CS3_PROVIDER_ADDR="unix:/var/run/ocis/storage-system.sock" SHARING_PUBLIC_JSONCS3_PROVIDER_ADDR="unix:/var/run/ocis/storage-system.sock" ``` - - -## Storage registry +## Storage Registry In order to add another storage provider the CS3 storage registry that is running as part of the CS3 gateway hes to be made aware of it. The easiest cleanest way to do it is to set `GATEWAY_STORAGE_REGISTRY_CONFIG_JSON=/path/to/storages.json` and list all storage providers like this: @@ -167,4 +168,4 @@ In order to add another storage provider the CS3 storage registry that is runnin } ``` -In the above replace `{storage-users-mount-uuid}` with the mount UUID that was generated for the storage-users service. You can find it in the `config.yaml` generated on by `ocis init`. The last entry `com.owncloud.api.storage-hello` and its `providerid` `"hello-storage-id"` are an example for in additional storage provider, in this case running `hellofs`, an example minimal storage driver. \ No newline at end of file +In the above replace `{storage-users-mount-uuid}` with the mount UUID that was generated for the storage-users service. You can find it in the `config.yaml` generated on by `ocis init`. The last entry `com.owncloud.api.storage-hello` and its `providerid` `"hello-storage-id"` are an example for in additional storage provider, in this case running `hellofs`, an example minimal storage driver. From 80ce622caa00f975465cd1563509f6c65d880fa1 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Mon, 26 Aug 2024 14:58:38 +0200 Subject: [PATCH 60/71] cleanup(oidc): Verify logout tokens useing golang-jwt golang-jwt provides all the necessary functionality to parse and verify LogoutTokens. This gets us rid of the direct go-jose dependency and quite a bit of custom crafted jwt verification code. --- go.mod | 2 +- ocis-pkg/oidc/client.go | 108 +++++------------ ocis-pkg/oidc/client_test.go | 227 +++++++++++++++-------------------- ocis-pkg/oidc/metadata.go | 33 +---- ocis-pkg/oidc/options.go | 17 ++- 5 files changed, 146 insertions(+), 241 deletions(-) diff --git a/go.mod b/go.mod index 67972ac18a..a8068106e0 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/ggwhite/go-masker v1.1.0 github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/render v1.0.3 - github.com/go-jose/go-jose/v3 v3.0.3 github.com/go-ldap/ldap/v3 v3.4.8 github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3 github.com/go-micro/plugins/v4/client/grpc v1.2.1 @@ -195,6 +194,7 @@ require ( github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-jose/go-jose/v4 v4.0.2 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect diff --git a/ocis-pkg/oidc/client.go b/ocis-pkg/oidc/client.go index 741f679372..4558a24140 100644 --- a/ocis-pkg/oidc/client.go +++ b/ocis-pkg/oidc/client.go @@ -1,9 +1,7 @@ package oidc import ( - "bytes" "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -17,7 +15,6 @@ import ( "github.com/MicahParks/keyfunc/v2" goidc "github.com/coreos/go-oidc/v3/oidc" - "github.com/go-jose/go-jose/v3" "github.com/golang-jwt/jwt/v5" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/proxy/pkg/config" @@ -95,6 +92,7 @@ func NewOIDCClient(opts ...Option) OIDCClient { httpClient: options.HTTPClient, accessTokenVerifyMethod: options.AccessTokenVerifyMethod, JWKSOptions: options.JWKSOptions, // TODO I don't like that we pass down config options ... + JWKS: options.JWKS, providerLock: &sync.Mutex{}, jwksLock: &sync.Mutex{}, remoteKeySet: options.KeySet, @@ -319,75 +317,46 @@ func (c *oidcClient) verifyAccessTokenJWT(token string) (RegClaimsWithSID, jwt.M } func (c *oidcClient) VerifyLogoutToken(ctx context.Context, rawToken string) (*LogoutToken, error) { + var claims LogoutToken if err := c.lookupWellKnownOpenidConfiguration(ctx); err != nil { return nil, err } - jws, err := jose.ParseSigned(rawToken) - if err != nil { - return nil, err - } - // Throw out tokens with invalid claims before trying to verify the token. This lets - // us do cheap checks before possibly re-syncing keys. - payload, err := parseJWT(rawToken) - if err != nil { - return nil, fmt.Errorf("oidc: malformed jwt: %v", err) - } - var token LogoutToken - if err := json.Unmarshal(payload, &token); err != nil { - return nil, fmt.Errorf("oidc: failed to unmarshal claims: %v", err) + jwks := c.getKeyfunc() + if jwks == nil { + return nil, errors.New("error initializing jwks keyfunc") } - //4. Verify that the Logout Token contains a sub Claim, a sid Claim, or both. - if token.Subject == "" && token.SessionId == "" { - return nil, fmt.Errorf("oidc: logout token must contain either sub or sid and MAY contain both") - } - //5. Verify that the Logout Token contains an events Claim whose value is JSON object containing the member name http://schemas.openid.net/event/backchannel-logout. - if token.Events.Event == nil { - return nil, fmt.Errorf("oidc: logout token must contain logout event") - } - //6. Verify that the Logout Token does not contain a nonce Claim. - var n struct { - Nonce *string `json:"nonce"` - } - json.Unmarshal(payload, &n) - if n.Nonce != nil { - return nil, fmt.Errorf("oidc: nonce on logout token MUST NOT be present") - } - // Check issuer. - if !c.skipIssuerValidation && token.Issuer != c.issuer { - return nil, fmt.Errorf("oidc: logout token issued by a different provider, expected %q got %q", c.issuer, token.Issuer) - } - - switch len(jws.Signatures) { - case 0: - return nil, fmt.Errorf("oidc: logout token not signed") - case 1: - // do nothing - default: - return nil, fmt.Errorf("oidc: multiple signatures on logout token not supported") - } - - sig := jws.Signatures[0] + // From the backchannel-logout spec: Like ID Tokens, selection of the + // algorithm used is governed by the id_token_signing_alg_values_supported + // Discovery parameter and the id_token_signed_response_alg Registration + // parameter when they are used; otherwise, the value SHOULD be the default + // of RS256 supportedSigAlgs := c.algorithms if len(supportedSigAlgs) == 0 { supportedSigAlgs = []string{RS256} } - if !contains(supportedSigAlgs, sig.Header.Algorithm) { - return nil, fmt.Errorf("oidc: logout token signed with unsupported algorithm, expected %q got %q", supportedSigAlgs, sig.Header.Algorithm) - } - - gotPayload, err := c.remoteKeySet.VerifySignature(goidc.ClientContext(ctx, c.httpClient), rawToken) + _, err := jwt.ParseWithClaims(rawToken, &claims, jwks.Keyfunc, jwt.WithValidMethods(supportedSigAlgs), jwt.WithIssuer(c.issuer)) if err != nil { - return nil, fmt.Errorf("failed to verify signature: %v", err) + c.Logger.Debug().Err(err).Msg("Failed to parse logout token") + return nil, err + } + // Basic token validation has happened in ParseWithClaims (signature, + // issuer, audience, ...). Now for some logout token specific checks. + // 1. Verify that the Logout Token contains a sub Claim, a sid Claim, or both. + if claims.Subject == "" && claims.SessionId == "" { + return nil, fmt.Errorf("oidc: logout token must contain either sub or sid and MAY contain both") + } + // 2. Verify that the Logout Token contains an events Claim whose value is JSON object containing the member name http://schemas.openid.net/event/backchannel-logout. + if claims.Events.Event == nil { + return nil, fmt.Errorf("oidc: logout token must contain logout event") + } + // 3. Verify that the Logout Token does not contain a nonce Claim. + if claims.Nonce != nil { + return nil, fmt.Errorf("oidc: nonce on logout token MUST NOT be present") } - // Ensure that the payload returned by the square actually matches the payload parsed earlier. - if !bytes.Equal(gotPayload, payload) { - return nil, errors.New("oidc: internal error, payload parsed did not match previous payload") - } - - return &token, nil + return &claims, nil } func unmarshalResp(r *http.Response, body []byte, v interface{}) error { @@ -402,24 +371,3 @@ func unmarshalResp(r *http.Response, body []byte, v interface{}) error { } return fmt.Errorf("expected Content-Type = application/json, got %q: %v", ct, err) } - -func contains(sli []string, ele string) bool { - for _, s := range sli { - if s == ele { - return true - } - } - return false -} - -func parseJWT(p string) ([]byte, error) { - parts := strings.Split(p, ".") - if len(parts) < 2 { - return nil, fmt.Errorf("oidc: malformed jwt, expected 3 parts got %d", len(parts)) - } - payload, err := base64.RawURLEncoding.DecodeString(parts[1]) - if err != nil { - return nil, fmt.Errorf("oidc: malformed jwt payload: %v", err) - } - return payload, nil -} diff --git a/ocis-pkg/oidc/client_test.go b/ocis-pkg/oidc/client_test.go index d876079917..5fa498c2de 100644 --- a/ocis-pkg/oidc/client_test.go +++ b/ocis-pkg/oidc/client_test.go @@ -2,152 +2,124 @@ package oidc_test import ( "context" - "crypto/ecdsa" - "crypto/elliptic" "crypto/rand" "crypto/rsa" - "fmt" "testing" - goidc "github.com/coreos/go-oidc/v3/oidc" - "github.com/go-jose/go-jose/v3" + "github.com/MicahParks/keyfunc/v2" + "github.com/golang-jwt/jwt/v5" "github.com/owncloud/ocis/v2/ocis-pkg/oidc" ) type signingKey struct { - keyID string // optional - priv interface{} - pub interface{} - alg jose.SignatureAlgorithm -} - -// sign creates a JWS using the private key from the provided payload. -func (s *signingKey) sign(t testing.TB, payload []byte) string { - privKey := &jose.JSONWebKey{Key: s.priv, Algorithm: string(s.alg), KeyID: s.keyID} - - signer, err := jose.NewSigner(jose.SigningKey{Algorithm: s.alg, Key: privKey}, nil) - if err != nil { - t.Fatal(err) - } - jws, err := signer.Sign(payload) - if err != nil { - t.Fatal(err) - } - - data, err := jws.CompactSerialize() - if err != nil { - t.Fatal(err) - } - return data -} - -func (s *signingKey) jwk() jose.JSONWebKey { - return jose.JSONWebKey{Key: s.pub, Use: "sig", Algorithm: string(s.alg), KeyID: s.keyID} + priv interface{} + jwks *keyfunc.JWKS } func TestLogoutVerify(t *testing.T) { tests := []logoutVerificationTest{ { name: "good token", - logoutToken: ` { - "iss": "https://foo", - "sub": "248289761001", - "aud": "s6BhdRkqt3", - "iat": 1471566154, - "jti": "bWJq", - "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", - "events": { - "http://schemas.openid.net/event/backchannel-logout": {} - } - }`, + logoutToken: jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": "https://foo", + "sub": "248289761001", + "aud": "s6BhdRkqt3", + "iat": 1471566154, + "jti": "bWJq", + "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", + "events": map[string]interface{}{ + "http://schemas.openid.net/event/backchannel-logout": struct{}{}, + }, + }), signKey: newRSAKey(t), }, { - name: "invalid issuer", - issuer: "https://bar", - logoutToken: `{"iss":"https://foo"}`, - config: goidc.Config{ - SkipExpiryCheck: true, - }, + name: "invalid issuer", + issuer: "https://bar", + logoutToken: jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": "https://foo1", + "sub": "248289761001", + "events": map[string]interface{}{ + "http://schemas.openid.net/event/backchannel-logout": struct{}{}, + }, + }), signKey: newRSAKey(t), wantErr: true, }, { name: "invalid sig", - logoutToken: `{ - "iss": "https://foo", - "sub": "248289761001", - "aud": "s6BhdRkqt3", - "iat": 1471566154, - "jti": "bWJq", - "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", - "events": { - "http://schemas.openid.net/event/backchannel-logout": {} - } - }`, - config: goidc.Config{ - SkipExpiryCheck: true, - }, + logoutToken: jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": "https://foo", + "sub": "248289761001", + "aud": "s6BhdRkqt3", + "iat": 1471566154, + "jti": "bWJq", + "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", + "events": map[string]interface{}{ + "http://schemas.openid.net/event/backchannel-logout": struct{}{}, + }, + }), signKey: newRSAKey(t), verificationKey: newRSAKey(t), wantErr: true, }, { name: "no sid and no sub", - logoutToken: ` { - "iss": "https://foo", - "aud": "s6BhdRkqt3", - "iat": 1471566154, - "jti": "bWJq", - "events": { - "http://schemas.openid.net/event/backchannel-logout": {} - } - }`, + logoutToken: jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": "https://foo", + "aud": "s6BhdRkqt3", + "iat": 1471566154, + "jti": "bWJq", + "events": map[string]interface{}{ + "http://schemas.openid.net/event/backchannel-logout": struct{}{}, + }, + }), signKey: newRSAKey(t), wantErr: true, }, { name: "Prohibited nonce present", - logoutToken: ` { - "iss": "https://foo", - "sub": "248289761001", - "aud": "s6BhdRkqt3", - "iat": 1471566154, - "jti": "bWJq", - "nonce" : "prohibited", - "events": { - "http://schemas.openid.net/event/backchannel-logout": {} - } - }`, + logoutToken: jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": "https://foo", + "sub": "248289761001", + "aud": "s6BhdRkqt3", + "iat": 1471566154, + "jti": "bWJq", + "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", + "nonce": "123", + "events": map[string]interface{}{ + "http://schemas.openid.net/event/backchannel-logout": struct{}{}, + }, + }), signKey: newRSAKey(t), wantErr: true, }, { name: "Wrong Event string", - logoutToken: ` { - "iss": "https://foo", - "sub": "248289761001", - "aud": "s6BhdRkqt3", - "iat": 1471566154, - "jti": "bWJq", - "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", - "events": { - "not a logout event": {} - } - }`, + logoutToken: jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": "https://foo", + "sub": "248289761001", + "aud": "s6BhdRkqt3", + "iat": 1471566154, + "jti": "bWJq", + "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", + "events": map[string]interface{}{ + "http://blah.blah.blash/event/backchannel-logout": struct{}{}, + }, + }), signKey: newRSAKey(t), wantErr: true, }, { name: "No Event string", - logoutToken: ` { - "iss": "https://foo", - "sub": "248289761001", - "aud": "s6BhdRkqt3", - "iat": 1471566154, - "jti": "bWJq", - "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", - }`, + logoutToken: jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": "https://foo", + "sub": "248289761001", + "aud": "s6BhdRkqt3", + "iat": 1471566154, + "jti": "bWJq", + "sid": "08a5019c-17e1-4977-8f42-65a12843ea02", + }), signKey: newRSAKey(t), wantErr: true, }, @@ -165,7 +137,7 @@ type logoutVerificationTest struct { issuer string // JWT payload (just the claims). - logoutToken string + logoutToken *jwt.Token // Key to sign the ID Token with. signKey *signingKey @@ -173,40 +145,31 @@ type logoutVerificationTest struct { // testing invalid signatures. verificationKey *signingKey - config goidc.Config wantErr bool } -type testVerifier struct { - jwk jose.JSONWebKey -} - -func (t *testVerifier) VerifySignature(ctx context.Context, jwt string) ([]byte, error) { - jws, err := jose.ParseSigned(jwt) - if err != nil { - return nil, fmt.Errorf("oidc: malformed jwt: %v", err) - } - return jws.Verify(&t.jwk) -} - func (v logoutVerificationTest) runGetToken(t *testing.T) (*oidc.LogoutToken, error) { - token := v.signKey.sign(t, []byte(v.logoutToken)) + // token := v.signKey.sign(t, []byte(v.logoutToken)) + v.logoutToken.Header["kid"] = "1" + token, err := v.logoutToken.SignedString(v.signKey.priv) + if err != nil { + t.Fatal(err) + } ctx, cancel := context.WithCancel(context.Background()) defer cancel() issuer := "https://foo" - var ks goidc.KeySet + var jwks *keyfunc.JWKS if v.verificationKey == nil { - ks = &testVerifier{v.signKey.jwk()} + jwks = v.signKey.jwks } else { - ks = &testVerifier{v.verificationKey.jwk()} + jwks = v.verificationKey.jwks } pm := oidc.ProviderMetadata{} verifier := oidc.NewOIDCClient( oidc.WithOidcIssuer(issuer), - oidc.WithKeySet(ks), - oidc.WithConfig(&v.config), + oidc.WithJWKS(jwks), oidc.WithProviderMetadata(&pm), ) @@ -228,13 +191,15 @@ func newRSAKey(t testing.TB) *signingKey { if err != nil { t.Fatal(err) } - return &signingKey{"", priv, priv.Public(), jose.RS256} -} + givenKey := keyfunc.NewGivenRSA( + &priv.PublicKey, + keyfunc.GivenKeyOptions{Algorithm: jwt.SigningMethodRS256.Alg()}, + ) + jwks := keyfunc.NewGiven( + map[string]keyfunc.GivenKey{ + "1": givenKey, + }, + ) -func newECDSAKey(t *testing.T) *signingKey { - priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } - return &signingKey{"", priv, priv.Public(), jose.ES256} + return &signingKey{priv, jwks} } diff --git a/ocis-pkg/oidc/metadata.go b/ocis-pkg/oidc/metadata.go index 2d2708a0f3..662d9ec522 100644 --- a/ocis-pkg/oidc/metadata.go +++ b/ocis-pkg/oidc/metadata.go @@ -59,35 +59,12 @@ type ProviderMetadata struct { // Logout Token defines an logout Token type LogoutToken struct { - // The URL of the server which issued this token. OpenID Connect - // requires this value always be identical to the URL used for - // initial discovery. - // - // Note: Because of a known issue with Google Accounts' implementation - // this value may differ when using Google. - // - // See: https://developers.google.com/identity/protocols/OpenIDConnect#obtainuserinfo - Issuer string `json:"iss"` // example "https://server.example.com" - - // A unique string which identifies the end user. - Subject string `json:"sub"` //"248289761001" - - // The client ID, or set of client IDs, that this token is issued for. For - // common uses, this is the client that initialized the auth flow. - // - // This package ensures the audience contains an expected value. - Audience jwt.ClaimStrings `json:"aud"` // "s6BhdRkqt3" - - // When the token was issued by the provider. - IssuedAt *jwt.NumericDate `json:"iat"` - + jwt.RegisteredClaims // The Session Id - SessionId string `json:"sid"` - - Events LogoutEvent `json:"events"` - - // Jwt Id - JwtID string `json:"jti"` + SessionId string `json:"sid"` + Events LogoutEvent `json:"events"` + // Note: This is just here to be able to check for nonce being absent + Nonce *string `json:"nonce"` } // LogoutEvent defines a logout Event diff --git a/ocis-pkg/oidc/options.go b/ocis-pkg/oidc/options.go index 149433ebaf..fbfc5fe7a9 100644 --- a/ocis-pkg/oidc/options.go +++ b/ocis-pkg/oidc/options.go @@ -3,6 +3,7 @@ package oidc import ( "net/http" + "github.com/MicahParks/keyfunc/v2" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/proxy/pkg/config" @@ -22,7 +23,14 @@ type Options struct { OIDCIssuer string // JWKSOptions to use when retrieving keys JWKSOptions config.JWKS - // KeySet to use when verifiing signatures + // the JWKS keyset to use for verifying signatures of Access- and + // Logout-Tokens + // this option is mostly needed for unit test. To avoid fetching the keys + // from the issuer + JWKS *keyfunc.JWKS + // KeySet to use when verifiing signatures of jwt encoded + // user info responses + // TODO move userinfo verification to use jwt/keyfunc as well KeySet KeySet // AccessTokenVerifyMethod to use when verifying access tokens // TODO pass a function or interface to verify? an AccessTokenVerifier? @@ -80,6 +88,13 @@ func WithJWKSOptions(val config.JWKS) Option { } } +// WithJWKS provides a function to set the JWKS option (mainly useful for testing). +func WithJWKS(val *keyfunc.JWKS) Option { + return func(o *Options) { + o.JWKS = val + } +} + // WithKeySet provides a function to set the KeySet option. func WithKeySet(val KeySet) Option { return func(o *Options) { From 18037f530e053e3727d1afe6ef1374432b84ec84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 26 Aug 2024 16:42:59 +0200 Subject: [PATCH 61/71] fix listing ocm properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../service/v0/api_driveitem_permissions.go | 2 +- services/graph/pkg/service/v0/base.go | 4 +- services/graph/pkg/service/v0/utils.go | 60 ++++++++++++------- services/graph/pkg/unifiedrole/unifiedrole.go | 32 ++++++---- .../graph/pkg/unifiedrole/unifiedrole_test.go | 2 +- 5 files changed, 64 insertions(+), 36 deletions(-) diff --git a/services/graph/pkg/service/v0/api_driveitem_permissions.go b/services/graph/pkg/service/v0/api_driveitem_permissions.go index 292a098b51..591d72948c 100644 --- a/services/graph/pkg/service/v0/api_driveitem_permissions.go +++ b/services/graph/pkg/service/v0/api_driveitem_permissions.go @@ -124,7 +124,7 @@ func (s DriveItemPermissionsService) Invite(ctx context.Context, resourceId *sto cs3ResourcePermissions := unifiedrole.PermissionsToCS3ResourcePermissions(unifiedRolePermissions) permission := &libregraph.Permission{} - if role := unifiedrole.CS3ResourcePermissionsToUnifiedRole(cs3ResourcePermissions, condition); role != nil { + if role := unifiedrole.CS3ResourcePermissionsToUnifiedRole(cs3ResourcePermissions, condition, false); role != nil { permission.Roles = []string{role.GetId()} } diff --git a/services/graph/pkg/service/v0/base.go b/services/graph/pkg/service/v0/base.go index 6c8a58ecd6..534cf0321d 100644 --- a/services/graph/pkg/service/v0/base.go +++ b/services/graph/pkg/service/v0/base.go @@ -191,7 +191,7 @@ func (g BaseGraphService) cs3SpacePermissionsToLibreGraph(ctx context.Context, s p.SetExpirationDateTime(time.Unix(int64(exp.GetSeconds()), int64(exp.GetNanos()))) } - if role := unifiedrole.CS3ResourcePermissionsToUnifiedRole(perm, unifiedrole.UnifiedRoleConditionDrive); role != nil { + if role := unifiedrole.CS3ResourcePermissionsToUnifiedRole(perm, unifiedrole.UnifiedRoleConditionDrive, false); role != nil { switch apiVersion { case APIVersion_1: if r := unifiedrole.GetLegacyName(*role); r != "" { @@ -459,6 +459,7 @@ func (g BaseGraphService) cs3UserShareToPermission(ctx context.Context, share *c role := unifiedrole.CS3ResourcePermissionsToUnifiedRole( share.GetPermissions().GetPermissions(), roleCondition, + false, ) if role != nil { perm.SetRoles([]string{role.GetId()}) @@ -542,6 +543,7 @@ func (g BaseGraphService) cs3OCMShareToPermission(ctx context.Context, share *oc role := unifiedrole.CS3ResourcePermissionsToUnifiedRole( permissions, roleCondition, + true, ) if role != nil { perm.SetRoles([]string{role.GetId()}) diff --git a/services/graph/pkg/service/v0/utils.go b/services/graph/pkg/service/v0/utils.go index d08a05567c..c8a45a85e2 100644 --- a/services/graph/pkg/service/v0/utils.go +++ b/services/graph/pkg/service/v0/utils.go @@ -445,7 +445,7 @@ func cs3ReceivedShareToLibreGraphPermissions(ctx context.Context, logger *log.Lo if err != nil { return nil, err } - role := unifiedrole.CS3ResourcePermissionsToUnifiedRole(permissionSet, condition) + role := unifiedrole.CS3ResourcePermissionsToUnifiedRole(permissionSet, condition, false) if role != nil { permission.SetRoles([]string{role.GetId()}) @@ -491,6 +491,17 @@ func roleConditionForResourceType(ri *storageprovider.ResourceInfo) (string, err } } +func federatedRoleConditionForResourceType(ri *storageprovider.ResourceInfo) (string, error) { + switch { + case ri.Type == storageprovider.ResourceType_RESOURCE_TYPE_CONTAINER: + return unifiedrole.UnifiedRoleConditionFolderFederatedUser, nil + case ri.Type == storageprovider.ResourceType_RESOURCE_TYPE_FILE: + return unifiedrole.UnifiedRoleConditionFileFederatedUser, nil + default: + return "", errorcode.New(errorcode.InvalidRequest, "unsupported resource type for federated role") + } +} + // ExtractShareIdFromResourceId is a bit of a hack. // We should not rely on a specific format of the item id. // But currently there is no other way to get the ShareID. @@ -764,36 +775,43 @@ func fillDriveItemPropertiesFromReceivedOCMShare(ctx context.Context, logger *lo func cs3ReceivedOCMShareToLibreGraphPermissions(ctx context.Context, logger *log.Logger, identityCache identity.IdentityCache, receivedShare *ocm.ReceivedShare, - _ *storageprovider.ResourceInfo) (*libregraph.Permission, error) { + resourceInfo *storageprovider.ResourceInfo) (*libregraph.Permission, error) { permission := libregraph.NewPermission() if id := receivedShare.GetId().GetOpaqueId(); id != "" { permission.SetId(id) } + if cTime := receivedShare.GetCtime(); cTime != nil { + permission.SetCreatedDateTime(cs3TimestampToTime(cTime)) + } + if expiration := receivedShare.GetExpiration(); expiration != nil { permission.SetExpirationDateTime(cs3TimestampToTime(expiration)) } - /* - if permissionSet := receivedShare.GetShare().GetPermissions().GetPermissions(); permissionSet != nil { - condition, err := roleConditionForResourceType(resourceInfo) - if err != nil { - return nil, err - } - role := unifiedrole.CS3ResourcePermissionsToUnifiedRole(*permissionSet, condition) - - if role != nil { - permission.SetRoles([]string{role.GetId()}) - } - - actions := unifiedrole.CS3ResourcePermissionsToLibregraphActions(*permissionSet) - - // actions only make sense if no role is set - if role == nil && len(actions) > 0 { - permission.SetLibreGraphPermissionsActions(actions) - } + var permissions *storageprovider.ResourcePermissions + for _, protocol := range receivedShare.GetProtocols() { + if protocol.GetWebdavOptions().GetPermissions() != nil { + permissions = protocol.GetWebdavOptions().GetPermissions().GetPermissions() } - */ + } + condition, err := federatedRoleConditionForResourceType(resourceInfo) + if err != nil { + return nil, err + } + role := unifiedrole.CS3ResourcePermissionsToUnifiedRole( + permissions, + condition, + true, + ) + if role != nil { + permission.SetRoles([]string{role.GetId()}) + } else { + actions := unifiedrole.CS3ResourcePermissionsToLibregraphActions(permissions) + permission.SetLibreGraphPermissionsActions(actions) + permission.SetRoles(nil) + } + switch grantee := receivedShare.GetGrantee(); { case grantee.GetType() == storageprovider.GranteeType_GRANTEE_TYPE_USER: user, err := cs3UserIdToIdentity(ctx, identityCache, grantee.GetUserId()) diff --git a/services/graph/pkg/unifiedrole/unifiedrole.go b/services/graph/pkg/unifiedrole/unifiedrole.go index cd40dcd18d..d20d1bd19e 100644 --- a/services/graph/pkg/unifiedrole/unifiedrole.go +++ b/services/graph/pkg/unifiedrole/unifiedrole.go @@ -527,28 +527,36 @@ func GetLegacyName(role libregraph.UnifiedRoleDefinition) string { // CS3ResourcePermissionsToUnifiedRole tries to find the UnifiedRoleDefinition that matches the supplied // CS3 ResourcePermissions and constraints. -func CS3ResourcePermissionsToUnifiedRole(p *provider.ResourcePermissions, constraints string) *libregraph.UnifiedRoleDefinition { - actionSet := map[string]struct{}{} - for _, action := range CS3ResourcePermissionsToLibregraphActions(p) { - actionSet[action] = struct{}{} - } +func CS3ResourcePermissionsToUnifiedRole(p *provider.ResourcePermissions, constraints string, listFederatedRoles bool) *libregraph.UnifiedRoleDefinition { + actions := CS3ResourcePermissionsToLibregraphActions(p) var res *libregraph.UnifiedRoleDefinition for _, uRole := range GetBuiltinRoleDefinitionList() { - matchFound := false + definitionMatch := false for _, uPerm := range uRole.GetRolePermissions() { - if uPerm.GetCondition() != constraints { - // the requested constraints don't match, this isn't our role + + // this is a dirty comparison because we are not really parsing the SDDL, but as long as we && the conditions we are good + isFederatedRole := strings.Contains(uPerm.GetCondition(), UnifiedRoleConditionFederatedUser) + switch { + case !strings.Contains(uPerm.GetCondition(), constraints): + continue + case listFederatedRoles && !isFederatedRole: + continue + case !listFederatedRoles && isFederatedRole: continue } // if the actions converted from the ResourcePermissions equal the action the defined for the role, we have match - if resourceActionsEqual(actionSet, uPerm.GetAllowedResourceActions()) { - matchFound = true - break + for i, action := range uPerm.GetAllowedResourceActions() { + if !slices.Contains(actions, action) { + break + } + if i == len(uPerm.GetAllowedResourceActions())-1 { + definitionMatch = true + } } } - if matchFound { + if definitionMatch { res = uRole break } diff --git a/services/graph/pkg/unifiedrole/unifiedrole_test.go b/services/graph/pkg/unifiedrole/unifiedrole_test.go index 145dd66c62..d43898f732 100644 --- a/services/graph/pkg/unifiedrole/unifiedrole_test.go +++ b/services/graph/pkg/unifiedrole/unifiedrole_test.go @@ -19,7 +19,7 @@ var _ = Describe("unifiedroles", func() { func(legacyRole *rConversions.Role, unifiedRole *libregraph.UnifiedRoleDefinition, constraints string) { cs3perm := legacyRole.CS3ResourcePermissions() - r := unifiedrole.CS3ResourcePermissionsToUnifiedRole(cs3perm, constraints) + r := unifiedrole.CS3ResourcePermissionsToUnifiedRole(cs3perm, constraints, false) Expect(r.GetId()).To(Equal(unifiedRole.GetId())) }, From 93c2e33b1e958a7e64db027338f6f3e247887774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 26 Aug 2024 16:43:35 +0200 Subject: [PATCH 62/71] bump reva MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- go.mod | 2 +- go.sum | 4 ++-- .../github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go | 1 + vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8e745fb65b..cb404a5d21 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.11.0 github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb - github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7 + github.com/cs3org/reva/v2 v2.23.1-0.20240826144102-af5123b523cf github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index 80e745ce62..19e0ac46f3 100644 --- a/go.sum +++ b/go.sum @@ -255,8 +255,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb h1:KmYZDReplv/yfwc1LNYpDcVhVujC3Pasv6WjXx1haSU= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb/go.mod h1:yyP8PRo0EZou3nSH7H4qjlzQwaydPeIRNgX50npQHpE= -github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7 h1:q5U8sebSA3VqeLuf8Xhg1bVRxc8oJuRjQCjkl8xQPaI= -github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7/go.mod h1:p7CHBXcg6sSqB+0JMNDfC1S7TSh9FghXkw1kTV3KcJI= +github.com/cs3org/reva/v2 v2.23.1-0.20240826144102-af5123b523cf h1:VMg9uATNCBjJhU0dJJ5wgchLCCkZr/6IxjJxY+8hAAs= +github.com/cs3org/reva/v2 v2.23.1-0.20240826144102-af5123b523cf/go.mod h1:p7CHBXcg6sSqB+0JMNDfC1S7TSh9FghXkw1kTV3KcJI= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go b/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go index ae7147a69e..0b1fea976d 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go @@ -285,6 +285,7 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc Mtime: &typepb.Timestamp{ Seconds: uint64(f.ModTime().Unix()), }, + Etag: webdavFile.ETag(), Owner: share.Creator, PermissionSet: webdavProtocol.Permissions.Permissions, Checksum: &provider.ResourceChecksum{ diff --git a/vendor/modules.txt b/vendor/modules.txt index f991003070..b2bd3589e4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -367,7 +367,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.23.1-0.20240823142954-51e6e33750e7 +# github.com/cs3org/reva/v2 v2.23.1-0.20240826144102-af5123b523cf ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime From 6c13e5fc285893b59eba060a54d575252c4892ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 26 Aug 2024 16:45:23 +0200 Subject: [PATCH 63/71] bump reva & changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/bump-reva.md | 1 + changelog/unreleased/ocm-listing-fixes.md | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog/unreleased/ocm-listing-fixes.md diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md index eb8a031538..70745801d0 100644 --- a/changelog/unreleased/bump-reva.md +++ b/changelog/unreleased/bump-reva.md @@ -2,5 +2,6 @@ Enhancement: Bump reva Bumps reva version +https://github.com/owncloud/ocis/pull/9920 https://github.com/owncloud/ocis/pull/9879 https://github.com/owncloud/ocis/pull/9860 diff --git a/changelog/unreleased/ocm-listing-fixes.md b/changelog/unreleased/ocm-listing-fixes.md new file mode 100644 index 0000000000..ffff55eab6 --- /dev/null +++ b/changelog/unreleased/ocm-listing-fixes.md @@ -0,0 +1,5 @@ +Bugfix: add missing properties to when listing ocm shares + +The libre graph API now returns an etag, the role and the creation time for ocm shares. + +https://github.com/owncloud/ocis/pull/9920 From e933299c8d21a6ff80989597a0c77baac03fdc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 26 Aug 2024 17:17:51 +0200 Subject: [PATCH 64/71] all actions must be part of the role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- services/graph/pkg/unifiedrole/unifiedrole.go | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/services/graph/pkg/unifiedrole/unifiedrole.go b/services/graph/pkg/unifiedrole/unifiedrole.go index d20d1bd19e..5f33950409 100644 --- a/services/graph/pkg/unifiedrole/unifiedrole.go +++ b/services/graph/pkg/unifiedrole/unifiedrole.go @@ -528,17 +528,20 @@ func GetLegacyName(role libregraph.UnifiedRoleDefinition) string { // CS3ResourcePermissionsToUnifiedRole tries to find the UnifiedRoleDefinition that matches the supplied // CS3 ResourcePermissions and constraints. func CS3ResourcePermissionsToUnifiedRole(p *provider.ResourcePermissions, constraints string, listFederatedRoles bool) *libregraph.UnifiedRoleDefinition { - actions := CS3ResourcePermissionsToLibregraphActions(p) + actionSet := map[string]struct{}{} + for _, action := range CS3ResourcePermissionsToLibregraphActions(p) { + actionSet[action] = struct{}{} + } var res *libregraph.UnifiedRoleDefinition for _, uRole := range GetBuiltinRoleDefinitionList() { definitionMatch := false - for _, uPerm := range uRole.GetRolePermissions() { + for _, permission := range uRole.GetRolePermissions() { // this is a dirty comparison because we are not really parsing the SDDL, but as long as we && the conditions we are good - isFederatedRole := strings.Contains(uPerm.GetCondition(), UnifiedRoleConditionFederatedUser) + isFederatedRole := strings.Contains(permission.GetCondition(), UnifiedRoleConditionFederatedUser) switch { - case !strings.Contains(uPerm.GetCondition(), constraints): + case !strings.Contains(permission.GetCondition(), constraints): continue case listFederatedRoles && !isFederatedRole: continue @@ -547,13 +550,9 @@ func CS3ResourcePermissionsToUnifiedRole(p *provider.ResourcePermissions, constr } // if the actions converted from the ResourcePermissions equal the action the defined for the role, we have match - for i, action := range uPerm.GetAllowedResourceActions() { - if !slices.Contains(actions, action) { - break - } - if i == len(uPerm.GetAllowedResourceActions())-1 { - definitionMatch = true - } + if resourceActionsEqual(actionSet, permission.GetAllowedResourceActions()) { + definitionMatch = true + break } } if definitionMatch { From 716c32730aaed56c19a826e6438d3a104c6e13fe Mon Sep 17 00:00:00 2001 From: ownClouders Date: Tue, 27 Aug 2024 00:06:17 +0000 Subject: [PATCH 65/71] [tx] updated from transifex --- .../pkg/email/l10n/locale/ja_JP/LC_MESSAGES/notifications.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/pkg/email/l10n/locale/ja_JP/LC_MESSAGES/notifications.po b/services/notifications/pkg/email/l10n/locale/ja_JP/LC_MESSAGES/notifications.po index af337990a6..7eb469fc7b 100644 --- a/services/notifications/pkg/email/l10n/locale/ja_JP/LC_MESSAGES/notifications.po +++ b/services/notifications/pkg/email/l10n/locale/ja_JP/LC_MESSAGES/notifications.po @@ -11,7 +11,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: EMAIL\n" -"POT-Creation-Date: 2024-08-06 04:39+0000\n" +"POT-Creation-Date: 2024-08-27 00:04+0000\n" "PO-Revision-Date: 2023-04-19 11:11+0000\n" "Last-Translator: hoshi1800 , 2024\n" "Language-Team: Japanese (Japan) (https://app.transifex.com/owncloud-org/teams/6149/ja_JP/)\n" From 3b6eaf52714cbcf05d1337724440cf98f6f4f0f7 Mon Sep 17 00:00:00 2001 From: Prajwol Amatya Date: Mon, 26 Aug 2024 16:37:50 +0545 Subject: [PATCH 66/71] change suite name to apiCollaboration --- .drone.star | 82 ++++++------------- tests/acceptance/config/behat.yml | 14 ++-- .../wopi.feature} | 6 +- tests/config/drone/app-registry.yaml | 4 +- 4 files changed, 35 insertions(+), 71 deletions(-) rename tests/acceptance/features/{apiAppProvider/appProvider.feature => apiCollaboration/wopi.feature} (86%) diff --git a/.drone.star b/.drone.star index bb4287ee79..657952abef 100644 --- a/.drone.star +++ b/.drone.star @@ -154,14 +154,9 @@ config = { "OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE": "%s" % dirs["ocmProviders"], }, }, - "cli": { + "apiWopi": { "suites": [ - "cliCommands", - ], - }, - "apiAppProvider": { - "suites": [ - "apiAppProvider", + "apiCollaboration", ], "skip": False, "collaborationServiceNeeded": True, @@ -173,6 +168,12 @@ config = { "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", }, }, + "cli": { + "suites": [ + "cliCommands", + ], + "skip": False, + }, }, "apiTests": { "numberOfParts": 10, @@ -895,7 +896,7 @@ def localApiTestPipeline(ctx): (waitForClamavService() if params["antivirusNeeded"] else []) + (waitForEmailService() if params["emailNeeded"] else []) + (ocisServer(storage, params["accounts_hash_difficulty"], deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"]) if params["federationServer"] else []) + - (collaborationService(extra_environment = params["extraCollaborationEnvironment"]) if params["collaborationServiceNeeded"] else []) + + (collaborationService(params["extraCollaborationEnvironment"]) if params["collaborationServiceNeeded"] else []) + localApiTests(suite, storage, params["extraEnvironment"]) + logRequests(), "services": emailService() if params["emailNeeded"] else [] + clamavService() if params["antivirusNeeded"] else [] + fakeOffice() if params["collaborationServiceNeeded"] else [], @@ -1003,38 +1004,25 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = "/app/wopiserver.py", ], }, + { + "name": "wait-for-wopi-server", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it wopiserver:9300 -t 300", + ], + }, ] else: extra_server_environment = { "OCIS_EXCLUDE_RUN_SERVICES": "app-provider", } - wopiServer = [ - { - "name": "wopiserver", - "image": OC_CI_GOLANG, - "detach": True, - "environment": { - "MICRO_REGISTRY": "nats-js-kv", - "MICRO_REGISTRY_ADDRESS": "ocis-server:9233", - "COLLABORATION_LOG_LEVEL": "debug", - "COLLABORATION_HTTP_ADDR": "0.0.0.0:9300", - "COLLABORATION_GRPC_ADDR": "0.0.0.0:9301", - # no proof keys available in the FakeOffice - "COLLABORATION_APP_PROOF_DISABLE": "true", - "COLLABORATION_APP_NAME": "FakeOffice", - "COLLABORATION_APP_ADDR": "http://fakeoffice:8080", - "COLLABORATION_APP_INSECURE": "true", - "COLLABORATION_WOPI_SRC": "http://wopiserver:9300", - "COLLABORATION_WOPI_SECRET": "some-wopi-secret", - "COLLABORATION_CS3API_DATAGATEWAY_INSECURE": "true", - "OCIS_JWT_SECRET": "some-ocis-jwt-secret", - }, - "commands": [ - "%s collaboration server" % ocis_bin, - ], - }, - ] + extra_environment = { + "COLLABORATION_APP_NAME": "FakeOffice", + "COLLABORATION_APP_ADDR": "http://fakeoffice:8080", + } + + wopiServer = collaborationService(extra_environment) wopiTestCases = dirs["base"] + "/tests/config/drone/wopiValidatorCustomTestCases.xml" for testgroup in testgroups: @@ -1079,34 +1067,10 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = }, "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - [ - { - "name": "fakeoffice", - "image": OC_CI_ALPINE, - "detach": True, - "environment": {}, - "commands": [ - "sh %s/tests/config/drone/serve-hosting-discovery.sh" % (dirs["base"]), - ], - }, - { - "name": "wait-for-fakeoffice", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it fakeoffice:8080 -t 300", - ], - }, - ] + + fakeOffice() + ocisServer(storage, accounts_hash_difficulty, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) + wopiServer + [ - { - "name": "wait-for-wopi-server", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it wopiserver:9300 -t 300", - ], - }, { "name": "prepare-test-file", "image": OC_CI_ALPINE, diff --git a/tests/acceptance/config/behat.yml b/tests/acceptance/config/behat.yml index 7bcecc6b35..930e5861a5 100644 --- a/tests/acceptance/config/behat.yml +++ b/tests/acceptance/config/behat.yml @@ -400,6 +400,13 @@ default: - SharingNgContext: - GraphContext: + apiCollaboration: + paths: + - "%paths.base%/../features/apiCollaboration" + context: *common_ldap_suite_context + contexts: + - FeatureContext: *common_feature_context_params + cliCommands: paths: - "%paths.base%/../features/cliCommands" @@ -414,13 +421,6 @@ default: - TagContext: - TrashbinContext: - apiAppProvider: - paths: - - "%paths.base%/../features/apiAppProvider" - context: *common_ldap_suite_context - contexts: - - FeatureContext: *common_feature_context_params - extensions: rdx\behatvars\BehatVariablesExtension: ~ diff --git a/tests/acceptance/features/apiAppProvider/appProvider.feature b/tests/acceptance/features/apiCollaboration/wopi.feature similarity index 86% rename from tests/acceptance/features/apiAppProvider/appProvider.feature rename to tests/acceptance/features/apiCollaboration/wopi.feature index 859d9fb74d..4584b52594 100644 --- a/tests/acceptance/features/apiAppProvider/appProvider.feature +++ b/tests/acceptance/features/apiCollaboration/wopi.feature @@ -1,10 +1,10 @@ -Feature: App Provider +Feature: collaboration (wopi) As a user I want to access files with collaboration service apps So that I can collaborate with other users - Scenario Outline: open file with .odt extension with collaboration app + Scenario Outline: open file with .odt extension Given user "Alice" has been created with default attributes and without skeleton files And user "Alice" has uploaded file "filesForUpload/simple.odt" to "simple.odt" And we save it into "FILEID" @@ -22,7 +22,7 @@ Feature: App Provider "properties": { "app_url": { "type": "string", - "pattern": "^https:\\/\\/(.*?)\\?WOPISrc=http%3A%2F%2F(.*?)%3A9300%2Fwopi%2Ffiles%2F[a-fA-F0-9]{64}$" + "pattern": "^.*\\?WOPISrc=.*wopi%2Ffiles%2F[a-fA-F0-9]{64}$" }, "method": { "const": "POST" diff --git a/tests/config/drone/app-registry.yaml b/tests/config/drone/app-registry.yaml index 674829a89c..a1042ec45f 100644 --- a/tests/config/drone/app-registry.yaml +++ b/tests/config/drone/app-registry.yaml @@ -2,8 +2,8 @@ app_registry: mimetypes: - mime_type: application/vnd.oasis.opendocument.text extension: odt - name: FakeOffice - description: FakeOffice text document + name: OpenDocument + description: OpenDocument text document icon: "" default_app: FakeOffice allow_creation: true From ca7604e2894900d410ad5719d1c98c86c96c2ead Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 06:07:23 +0000 Subject: [PATCH 67/71] Bump github.com/pkg/xattr from 0.4.9 to 0.4.10 Bumps [github.com/pkg/xattr](https://github.com/pkg/xattr) from 0.4.9 to 0.4.10. - [Release notes](https://github.com/pkg/xattr/releases) - [Commits](https://github.com/pkg/xattr/compare/v0.4.9...v0.4.10) --- updated-dependencies: - dependency-name: github.com/pkg/xattr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/pkg/xattr/xattr.go | 9 +++++---- vendor/github.com/pkg/xattr/xattr_solaris.go | 20 +++++++++++++++----- vendor/modules.txt | 2 +- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 8e745fb65b..526c209443 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/orcaman/concurrent-map v1.0.0 github.com/owncloud/libre-graph-api-go v1.0.5-0.20240820135012-5fac8096ce9c github.com/pkg/errors v0.9.1 - github.com/pkg/xattr v0.4.9 + github.com/pkg/xattr v0.4.10 github.com/prometheus/client_golang v1.20.1 github.com/r3labs/sse/v2 v2.10.0 github.com/riandyrn/otelchi v0.9.0 diff --git a/go.sum b/go.sum index 80e745ce62..62aedc64cf 100644 --- a/go.sum +++ b/go.sum @@ -969,8 +969,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= -github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= -github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= +github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/vendor/github.com/pkg/xattr/xattr.go b/vendor/github.com/pkg/xattr/xattr.go index 8b2b5fe80e..e34e274d51 100644 --- a/vendor/github.com/pkg/xattr/xattr.go +++ b/vendor/github.com/pkg/xattr/xattr.go @@ -87,8 +87,8 @@ func get(path string, name string, getxattrFunc getxattrFunc) ([]byte, error) { initialBufSize = 1024 // The theoretical maximum xattr value size on MacOS is 64 MB. On Linux it's - // much smaller at 64 KB. Unless the kernel is evil or buggy, we should never - // hit the limit. + // much smaller: documented at 64 KB. However, at least on TrueNAS SCALE, a + // Debian-based Linux distro, it can be larger. maxBufSize = 64 * 1024 * 1024 // Function name as reported in error messages @@ -102,14 +102,15 @@ func get(path string, name string, getxattrFunc getxattrFunc) ([]byte, error) { // If the buffer was too small to fit the value, Linux and MacOS react // differently: - // Linux: returns an ERANGE error and "-1" bytes. + // Linux: returns an ERANGE error and "-1" bytes. However, the TrueNAS + // SCALE distro sometimes returns E2BIG. // MacOS: truncates the value and returns "size" bytes. If the value // happens to be exactly as big as the buffer, we cannot know if it was // truncated, and we retry with a bigger buffer. Contrary to documentation, // MacOS never seems to return ERANGE! // To keep the code simple, we always check both conditions, and sometimes // double the buffer size without it being strictly necessary. - if err == syscall.ERANGE || read == size { + if err == syscall.ERANGE || err == syscall.E2BIG || read == size { // The buffer was too small. Try again. size <<= 1 if size >= maxBufSize { diff --git a/vendor/github.com/pkg/xattr/xattr_solaris.go b/vendor/github.com/pkg/xattr/xattr_solaris.go index 8d65b8d8d6..7c98b4afba 100644 --- a/vendor/github.com/pkg/xattr/xattr_solaris.go +++ b/vendor/github.com/pkg/xattr/xattr_solaris.go @@ -24,7 +24,7 @@ const ( ) func getxattr(path string, name string, data []byte) (int, error) { - f, err := os.OpenFile(path, os.O_RDONLY, 0) + f, err := openNonblock(path) if err != nil { return 0, err } @@ -50,7 +50,7 @@ func fgetxattr(f *os.File, name string, data []byte) (int, error) { } func setxattr(path string, name string, data []byte, flags int) error { - f, err := os.OpenFile(path, os.O_RDONLY, 0) + f, err := openNonblock(path) if err != nil { return err } @@ -87,7 +87,8 @@ func fsetxattr(f *os.File, name string, data []byte, flags int) error { } func removexattr(path string, name string) error { - fd, err := unix.Open(path, unix.O_RDONLY|unix.O_XATTR, 0) + mode := unix.O_RDONLY | unix.O_XATTR | unix.O_NONBLOCK | unix.O_CLOEXEC + fd, err := unix.Open(path, mode, 0) if err != nil { return err } @@ -114,7 +115,7 @@ func fremovexattr(f *os.File, name string) error { } func listxattr(path string, data []byte) (int, error) { - f, err := os.OpenFile(path, os.O_RDONLY, 0) + f, err := openNonblock(path) if err != nil { return 0, err } @@ -151,8 +152,17 @@ func flistxattr(f *os.File, data []byte) (int, error) { return copy(data, buf), nil } +// Like os.Open, but passes O_NONBLOCK to the open(2) syscall. +func openNonblock(path string) (*os.File, error) { + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), path), err +} + // stringsFromByteSlice converts a sequence of attributes to a []string. -// On Darwin and Linux, each entry is a NULL-terminated string. +// We simulate Linux/Darwin, where each entry is a NULL-terminated string. func stringsFromByteSlice(buf []byte) (result []string) { offset := 0 for index, b := range buf { diff --git a/vendor/modules.txt b/vendor/modules.txt index f991003070..0c419f5b8b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1658,7 +1658,7 @@ github.com/pjbgf/sha1cd/ubc # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors -# github.com/pkg/xattr v0.4.9 +# github.com/pkg/xattr v0.4.10 ## explicit; go 1.14 github.com/pkg/xattr # github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 From 21a74e74cd7f2a5fd04b603aad8646edb4dc7291 Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Tue, 27 Aug 2024 07:34:33 +0000 Subject: [PATCH 68/71] Automated changelog update [skip ci] --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f72aa2e406..97742fe0d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ The following sections list the changes for unreleased. * Bugfix - Activity Translations: [#9856](https://github.com/owncloud/ocis/pull/9856) * Bugfix - The user attributes `userType` and `memberOf` are readonly: [#9867](https://github.com/owncloud/ocis/pull/9867) * Bugfix - Use key to get specific trash item: [#9879](https://github.com/owncloud/ocis/pull/9879) +* Bugfix - Fix response code when upload a file over locked: [#9894](https://github.com/owncloud/ocis/pull/9894) * Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) * Enhancement - We now set the configured protocol transport for service metadata: [#9490](https://github.com/owncloud/ocis/pull/9490) @@ -98,6 +99,14 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/pull/9879 +* Bugfix - Fix response code when upload a file over locked: [#9894](https://github.com/owncloud/ocis/pull/9894) + + We fixed a bug where the response code was incorrect when uploading a file over + a locked file. + + https://github.com/owncloud/ocis/issues/7638 + https://github.com/owncloud/ocis/pull/9894 + * Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) The libre graph API now returns OCM shares when listing driveItem permissions. From 0cccc5f9822309b1ee03c15cf4abe28efb50a9b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 27 Aug 2024 09:53:08 +0000 Subject: [PATCH 69/71] Automated changelog update [skip ci] --- CHANGELOG.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97742fe0d8..6734132e56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,12 +59,13 @@ The following sections list the changes for unreleased. * Bugfix - Use key to get specific trash item: [#9879](https://github.com/owncloud/ocis/pull/9879) * Bugfix - Fix response code when upload a file over locked: [#9894](https://github.com/owncloud/ocis/pull/9894) * Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) +* Bugfix - Add missing properties to when listing ocm shares: [#9920](https://github.com/owncloud/ocis/pull/9920) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) * Enhancement - We now set the configured protocol transport for service metadata: [#9490](https://github.com/owncloud/ocis/pull/9490) -* Enhancement - Bump reva: [#9879](https://github.com/owncloud/ocis/pull/9879) * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) * Enhancement - Allow setting default locale of activitylog: [#9892](https://github.com/owncloud/ocis/pull/9892) * Enhancement - Graph translation path: [#9902](https://github.com/owncloud/ocis/pull/9902) +* Enhancement - Bump reva: [#9920](https://github.com/owncloud/ocis/pull/9920) ## Details @@ -114,6 +115,13 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/9898 https://github.com/owncloud/ocis/pull/9905 +* Bugfix - Add missing properties to when listing ocm shares: [#9920](https://github.com/owncloud/ocis/pull/9920) + + The libre graph API now returns an etag, the role and the creation time for ocm + shares. + + https://github.com/owncloud/ocis/pull/9920 + * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) We have removed the unused store service. @@ -130,13 +138,6 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/pull/9490 https://github.com/cs3org/reva/pull/4744 -* Enhancement - Bump reva: [#9879](https://github.com/owncloud/ocis/pull/9879) - - Bumps reva version - - https://github.com/owncloud/ocis/pull/9879 - https://github.com/owncloud/ocis/pull/9860 - * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) The `revisions purge` command would time out on big spaces. We have improved @@ -156,6 +157,14 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/pull/9902 +* Enhancement - Bump reva: [#9920](https://github.com/owncloud/ocis/pull/9920) + + Bumps reva version + + https://github.com/owncloud/ocis/pull/9920 + https://github.com/owncloud/ocis/pull/9879 + https://github.com/owncloud/ocis/pull/9860 + # Changelog for [6.3.0] (2024-08-20) The following sections list the changes for 6.3.0. From 6dc6fdacf4ed359d6bcd6f9912a188b47f0217f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 27 Aug 2024 12:39:37 +0200 Subject: [PATCH 70/71] list ocm shares in sharedByMe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/ocm-listing-fixes.md | 4 +++- services/graph/pkg/service/v0/sharedbyme.go | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/changelog/unreleased/ocm-listing-fixes.md b/changelog/unreleased/ocm-listing-fixes.md index ffff55eab6..6e308789ab 100644 --- a/changelog/unreleased/ocm-listing-fixes.md +++ b/changelog/unreleased/ocm-listing-fixes.md @@ -1,5 +1,7 @@ -Bugfix: add missing properties to when listing ocm shares +Bugfix: fix listing ocm shares The libre graph API now returns an etag, the role and the creation time for ocm shares. +It also includes ocm shares in the sharedByMe endpoint. +https://github.com/owncloud/ocis/pull/9925 https://github.com/owncloud/ocis/pull/9920 diff --git a/services/graph/pkg/service/v0/sharedbyme.go b/services/graph/pkg/service/v0/sharedbyme.go index 286476dc1e..22a9be3026 100644 --- a/services/graph/pkg/service/v0/sharedbyme.go +++ b/services/graph/pkg/service/v0/sharedbyme.go @@ -25,6 +25,14 @@ func (g Graph) GetSharedByMe(w http.ResponseWriter, r *http.Request) { return } + if g.config.IncludeOCMSharees { + driveItems, err = g.listOCMShares(ctx, nil, driveItems) + if err != nil { + errorcode.RenderError(w, r, err) + return + } + } + driveItems, err = g.listPublicShares(ctx, nil, driveItems) if err != nil { errorcode.RenderError(w, r, err) From e31d05f4f546d3a22376adaac17a856f22589987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 27 Aug 2024 11:09:18 +0000 Subject: [PATCH 71/71] Automated changelog update [skip ci] --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6734132e56..4a220b1312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,7 +59,7 @@ The following sections list the changes for unreleased. * Bugfix - Use key to get specific trash item: [#9879](https://github.com/owncloud/ocis/pull/9879) * Bugfix - Fix response code when upload a file over locked: [#9894](https://github.com/owncloud/ocis/pull/9894) * Bugfix - List OCM permissions as graph drive item permissions: [#9905](https://github.com/owncloud/ocis/pull/9905) -* Bugfix - Add missing properties to when listing ocm shares: [#9920](https://github.com/owncloud/ocis/pull/9920) +* Bugfix - Fix listing ocm shares: [#9925](https://github.com/owncloud/ocis/pull/9925) * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890) * Enhancement - We now set the configured protocol transport for service metadata: [#9490](https://github.com/owncloud/ocis/pull/9490) * Enhancement - Improve revisions purge: [#9891](https://github.com/owncloud/ocis/pull/9891) @@ -115,11 +115,12 @@ The following sections list the changes for unreleased. https://github.com/owncloud/ocis/issues/9898 https://github.com/owncloud/ocis/pull/9905 -* Bugfix - Add missing properties to when listing ocm shares: [#9920](https://github.com/owncloud/ocis/pull/9920) +* Bugfix - Fix listing ocm shares: [#9925](https://github.com/owncloud/ocis/pull/9925) The libre graph API now returns an etag, the role and the creation time for ocm - shares. + shares. It also includes ocm shares in the sharedByMe endpoint. + https://github.com/owncloud/ocis/pull/9925 https://github.com/owncloud/ocis/pull/9920 * Change - Remove store service: [#9890](https://github.com/owncloud/ocis/pull/9890)