From ee03db134db594087858eb47d88622d4ffcdfd16 Mon Sep 17 00:00:00 2001 From: Pascal Bleser Date: Mon, 6 Jul 2026 19:07:25 +0200 Subject: [PATCH] groupware: add Groupware integration testing * split off the Stalwart integration test boilerplate into a new jmaptest package, in order to be able to use it from the groupware package as well * remove the JMAP Session patching in tests and, instead, pick random free local ports for HTTP and IMAPS manually, and set those explicitly in the Stalwart testcontainer, so that we can set the PUBLIC_URL environment variable that Stalwart then uses to define the URLs that are served as part of the JMAP Sessions (API URL, etc...) --- pkg/jmap/client.go | 8 +- pkg/jmap/http.go | 2 +- pkg/jmap/tools.go | 9 +- pkg/jmaptest/export_integration_test.go | 1 - pkg/jmaptest/integration_addressbook_test.go | 30 +-- pkg/jmaptest/integration_calendar_test.go | 26 +-- pkg/jmaptest/integration_email_test.go | 98 ++++----- pkg/jmaptest/integration_ws_test.go | 18 +- pkg/jmaptest/jmap_stalwart.go | 197 ++++++++++-------- pkg/jmaptest/tools.go | 26 ++- .../groupware/pkg/groupware/api_index_test.go | 33 +++ services/groupware/pkg/groupware/framework.go | 171 ++++++++------- .../groupware/groupware_integration_test.go | 172 +++++++++++++++ services/groupware/pkg/middleware/auth.go | 8 +- 14 files changed, 529 insertions(+), 270 deletions(-) delete mode 100644 pkg/jmaptest/export_integration_test.go create mode 100644 services/groupware/pkg/groupware/api_index_test.go create mode 100644 services/groupware/pkg/groupware/groupware_integration_test.go diff --git a/pkg/jmap/client.go b/pkg/jmap/client.go index 5a28075b10..ecccacb7b5 100644 --- a/pkg/jmap/client.go +++ b/pkg/jmap/client.go @@ -44,8 +44,8 @@ func (j *Client) Api() ApiClient { return j.api } -func NewClient(session SessionClient, api ApiClient, blob BlobClient, ws WsClientFactory) Client { - return Client{ +func NewClient(session SessionClient, api ApiClient, blob BlobClient, ws WsClientFactory) *Client { + return &Client{ session: session, api: api, blob: blob, @@ -73,11 +73,11 @@ func (j *Client) OnNotification(username string, stateChange StateChange) { // Retrieve JMAP well-known data from the Stalwart server and create a Session from that. func (j *Client) FetchSession(ctx context.Context, sessionUrl *url.URL, username string, logger *log.Logger) (Session, Error) { - wk, err := j.session.GetSession(ctx, sessionUrl, username, logger) + sessionResponse, err := j.session.GetSession(ctx, sessionUrl, username, logger) if err != nil { return Session{}, err } - return newSession(wk) + return newSession(sessionResponse) } func (j *Client) logger(operation string, ctx Context) *log.Logger { diff --git a/pkg/jmap/http.go b/pkg/jmap/http.go index 1f3e3e31a5..29705e5da6 100644 --- a/pkg/jmap/http.go +++ b/pkg/jmap/http.go @@ -377,7 +377,7 @@ func (h *HttpJmapClient) Command(operation Operation, request Request, ctx Conte if res.StatusCode < 200 || res.StatusCode > 299 { h.listener.OnFailedRequestWithStatus(endpoint, operation, res.StatusCode) logger.Error().Str(logEndpoint, endpoint).Str(logHttpStatus, log.SafeString(res.Status)).Msg("HTTP response status code is not 2xx") //NOSONAR - return nil, language, jmapError(err, JmapErrorServerResponse) + return nil, language, jmapError(fmt.Errorf("JMAP server responsed with '%s'", res.Status), JmapErrorServerResponse) } h.listener.OnSuccessfulRequest(endpoint, operation, res.StatusCode) diff --git a/pkg/jmap/tools.go b/pkg/jmap/tools.go index 789b2ca5d7..852ec5af87 100644 --- a/pkg/jmap/tools.go +++ b/pkg/jmap/tools.go @@ -26,7 +26,7 @@ var UintPtrZero *uint = uintPtr(0) type eventListeners[T any] struct { listeners []T - m sync.Mutex + m sync.Mutex // TODO get rid of the mutex to avoid lock contention, use a lock-free alternative, or get rid of it altogether } func (e *eventListeners[T]) add(listener T) { @@ -81,6 +81,9 @@ func command[T any](client Cmdr, //NOSONAR if jmapErr != nil { return ZeroResult[T](single(duration)), jmapErr } + if responseBody == nil { + return ZeroResult[T](single(duration)), jmapError(fmt.Errorf("empty response body"), JmapErrorInvalidJmapResponsePayload) + } var response Response if err := json.NewDecoder(responseBody).Decode(&response); err != nil { @@ -450,6 +453,8 @@ func ptr[T any | int | uint | bool | string](t T) *T { func dumpHttpRequest(req *http.Request, maxBodySize int64, closure func(method string, uri string, content string, truncated bool)) error { var logBuilder bytes.Buffer + uri := req.URL.String() + fmt.Fprintf(&logBuilder, "%s %s\n", req.Method, uri) for key, values := range req.Header { if key == "Authorization" { continue @@ -474,7 +479,7 @@ func dumpHttpRequest(req *http.Request, maxBodySize int64, closure func(method s req.Body = io.NopCloser(fullBodyReader) } } - closure(req.Method, req.URL.String(), logBuilder.String(), truncated) + closure(req.Method, uri, logBuilder.String(), truncated) return nil } diff --git a/pkg/jmaptest/export_integration_test.go b/pkg/jmaptest/export_integration_test.go deleted file mode 100644 index d5638a54c2..0000000000 --- a/pkg/jmaptest/export_integration_test.go +++ /dev/null @@ -1 +0,0 @@ -package jmaptest diff --git a/pkg/jmaptest/integration_addressbook_test.go b/pkg/jmaptest/integration_addressbook_test.go index 874e03b85c..afb69ef091 100644 --- a/pkg/jmaptest/integration_addressbook_test.go +++ b/pkg/jmaptest/integration_addressbook_test.go @@ -47,13 +47,13 @@ func TestAddressBooks(t *testing.T) { list, getid, func(s *StalwartTest, accountId AccountId, ids []string, ctx Context) (Result[AddressBookGetResponse], error) { - return s.client.GetAddressbooks(accountId, ids, ctx) + return s.Client.GetAddressbooks(accountId, ids, ctx) }, func(s *StalwartTest, accountId AccountId, id string, change AddressBookChange, ctx Context) (Result[AddressBook], error) { //NOSONAR - return s.client.UpdateAddressBook(accountId, id, change, ctx) + return s.Client.UpdateAddressBook(accountId, id, change, ctx) }, func(s *StalwartTest, accountId AccountId, ids []string, ctx Context) (Result[map[string]SetError], error) { //NOSONAR - return s.client.DeleteAddressBook(accountId, ids, ctx) + return s.Client.DeleteAddressBook(accountId, ids, ctx) }, func(s *StalwartTest, t *testing.T, accountId AccountId, count uint, ctx Context, user User, principalIds []PrincipalId) (AddressBookBoxes, []AddressBook, SessionState, State, error) { return s.fillAddressBook(t, accountId, count, ctx, user, principalIds) @@ -86,7 +86,7 @@ func TestContacts(t *testing.T) { defer s.Close() user := pickUser() - session := s.Session(user.email) + session := s.Session(user.Email) ctx := s.Context(session) accountId, addressbookId, expectedContactCardsById, boxes, err := s.fillContacts(t, count, session, ctx, user) @@ -105,7 +105,7 @@ func TestContacts(t *testing.T) { ss := EmptySessionState os := EmptyState { - result, err := s.client.QueryContactCards(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) + result, err := s.Client.QueryContactCards(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) require.NoError(err) require.Len(result.Payload, 1) @@ -134,7 +134,7 @@ func TestContacts(t *testing.T) { // retrieve all objects at once { ids := structs.Map(results.Results, func(c ContactCard) string { return c.Id }) - result, err := s.client.GetContactCards(accountId, ids, ctx) + result, err := s.Client.GetContactCards(accountId, ids, ctx) require.NoError(err) require.Empty(result.Payload.NotFound) require.Len(result.Payload.List, len(ids)) @@ -148,7 +148,7 @@ func TestContacts(t *testing.T) { // retrieve each object one by one for _, actual := range results.Results { - result, err := s.client.GetContactCards(accountId, []string{actual.Id}, ctx) + result, err := s.Client.GetContactCards(accountId, []string{actual.Id}, ctx) require.NoError(err) require.Len(result.Payload.List, 1) matchContact(t, result.Payload.List[0], actual) @@ -162,7 +162,7 @@ func TestContacts(t *testing.T) { for i := range slices { position := int(i * limit) page := min(remainder, limit) - result, err := s.client.QueryContactCards(map[AccountId]QueryParams{accountId: {Position: position}}, &limit, filter, sortBy, true, ctx) + result, err := s.Client.QueryContactCards(map[AccountId]QueryParams{accountId: {Position: position}}, &limit, filter, sortBy, true, ctx) require.NoError(err) require.Len(result.Payload, 1) require.Contains(result.Payload, accountId) @@ -187,7 +187,7 @@ func TestContacts(t *testing.T) { offset := 0 i := 0 for chunk := range slices.Chunk(results.Results, chunkSize) { - result, err := s.client.QueryContactCards(map[AccountId]QueryParams{accountId: {Anchor: anchor, AnchorOffset: &offset}}, uintPtr(chunkSize), filter, sortBy, true, ctx) + result, err := s.Client.QueryContactCards(map[AccountId]QueryParams{accountId: {Anchor: anchor, AnchorOffset: &offset}}, uintPtr(chunkSize), filter, sortBy, true, ctx) require.Equal(ss, result.GetSessionState()) require.NoError(err) require.Len(result.Payload, 1) @@ -217,7 +217,7 @@ func TestContacts(t *testing.T) { Language: ptr("xyz"), Updated: ptr(now), } - result, err := s.client.UpdateContactCard(accountId, event.Id, change, ctx) + result, err := s.Client.UpdateContactCard(accountId, event.Id, change, ctx) require.NoError(err) require.Equal("xyz", result.Payload.Language) require.Equal(now, result.Payload.Updated) @@ -228,7 +228,7 @@ func TestContacts(t *testing.T) { } { ids := structs.Map(slices.Collect(maps.Values(expectedContactCardsById)), func(e ContactCard) string { return e.Id }) - result, err := s.client.DeleteContactCard(accountId, ids, ctx) + result, err := s.Client.DeleteContactCard(accountId, ids, ctx) require.NoError(err) require.Empty(result.Payload) @@ -237,7 +237,7 @@ func TestContacts(t *testing.T) { os = result.GetState() } { - result, err := s.client.QueryContactCards(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) + result, err := s.Client.QueryContactCards(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) require.NoError(err) require.Contains(result.Payload, accountId) resp := result.Payload[accountId] @@ -330,7 +330,7 @@ func (s *StalwartTest) fillAddressBook( //NOSONAR abook.ShareWith = m } - result, err := s.client.CreateAddressBook(accountId, abook, ctx) + result, err := s.Client.CreateAddressBook(accountId, abook, ctx) if err != nil { return boxes, created, ss, as, err } @@ -360,7 +360,7 @@ func (s *StalwartTest) fillContacts( //NOSONAR user User, ) (AccountId, string, map[string]ContactCard, ContactsBoxes, error) { require := require.New(t) - c, err := NewTestJmapClient(session, user.email, user.password, true, true) + c, err := NewTestJmapClient(session, user.Email, user.Password, true, true) require.NoError(err) defer c.Close() @@ -655,7 +655,7 @@ func (s *StalwartTest) fillContacts( //NOSONAR return "", "", nil, boxes, err } - result, err := s.client.CreateContactCard(accountId, card, ctx) + result, err := s.Client.CreateContactCard(accountId, card, ctx) if err != nil { return accountId, addressbookId, filled, boxes, err } diff --git a/pkg/jmaptest/integration_calendar_test.go b/pkg/jmaptest/integration_calendar_test.go index ec26233c6a..9ccbf24ac2 100644 --- a/pkg/jmaptest/integration_calendar_test.go +++ b/pkg/jmaptest/integration_calendar_test.go @@ -37,13 +37,13 @@ func TestCalendars(t *testing.T) { //NOSONAR func(resp CalendarGetResponse) []Calendar { return resp.List }, func(obj Calendar) string { return obj.Id }, func(s *StalwartTest, accountId AccountId, ids []string, ctx Context) (Result[CalendarGetResponse], error) { - return s.client.GetCalendars(accountId, ids, ctx) + return s.Client.GetCalendars(accountId, ids, ctx) }, func(s *StalwartTest, accountId AccountId, id string, change CalendarChange, ctx Context) (Result[Calendar], error) { //NOSONAR - return s.client.UpdateCalendar(accountId, id, change, ctx) + return s.Client.UpdateCalendar(accountId, id, change, ctx) }, func(s *StalwartTest, accountId AccountId, ids []string, ctx Context) (Result[map[string]SetError], error) { //NOSONAR - return s.client.DeleteCalendar(accountId, ids, ctx) + return s.Client.DeleteCalendar(accountId, ids, ctx) }, func(s *StalwartTest, t *testing.T, accountId AccountId, count uint, ctx Context, user User, principalIds []PrincipalId) (CalendarBoxes, []Calendar, SessionState, State, error) { return s.fillCalendar(t, accountId, count, ctx, user, principalIds) @@ -76,7 +76,7 @@ func TestEvents(t *testing.T) { defer s.Close() user := pickUser() - session := s.Session(user.email) + session := s.Session(user.Email) ctx := s.Context(session) accountId, calendarId, expectedEventsById, boxes, err := s.fillEvents(t, count, ctx, user) @@ -95,7 +95,7 @@ func TestEvents(t *testing.T) { os := EmptyState var results *CalendarEventSearchResults { - result, err := s.client.QueryCalendarEvents(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) + result, err := s.Client.QueryCalendarEvents(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) require.NoError(err) require.Len(result.Payload, 1) @@ -128,7 +128,7 @@ func TestEvents(t *testing.T) { for i := range slices { position := int(i * limit) page := min(remainder, limit) - result, err := s.client.QueryCalendarEvents(map[AccountId]QueryParams{accountId: {Position: position}}, &limit, filter, sortBy, true, ctx) + result, err := s.Client.QueryCalendarEvents(map[AccountId]QueryParams{accountId: {Position: position}}, &limit, filter, sortBy, true, ctx) require.NoError(err) require.Len(result.Payload, 1) require.Contains(result.Payload, accountId) @@ -153,7 +153,7 @@ func TestEvents(t *testing.T) { offset := 0 i := 0 for chunk := range slices.Chunk(results.Results, chunkSize) { - result, err := s.client.QueryCalendarEvents(map[AccountId]QueryParams{accountId: {Anchor: anchor, AnchorOffset: &offset}}, uintPtr(chunkSize), filter, sortBy, true, ctx) + result, err := s.Client.QueryCalendarEvents(map[AccountId]QueryParams{accountId: {Anchor: anchor, AnchorOffset: &offset}}, uintPtr(chunkSize), filter, sortBy, true, ctx) require.Equal(ss, result.GetSessionState()) require.NoError(err) require.Len(result.Payload, 1) @@ -186,7 +186,7 @@ func TestEvents(t *testing.T) { }, }, } - result, err := s.client.UpdateCalendarEvent(accountId, event.Id, change, ctx) + result, err := s.Client.UpdateCalendarEvent(accountId, event.Id, change, ctx) require.NoError(err) require.Equal(jscalendar.StatusCancelled, result.Payload.Status) require.Equal(uint(99), result.Payload.Sequence) @@ -198,7 +198,7 @@ func TestEvents(t *testing.T) { { ids := structs.Map(slices.Collect(maps.Values(expectedEventsById)), func(e CalendarEvent) string { return e.Id }) - result, err := s.client.DeleteCalendarEvent(accountId, ids, ctx) + result, err := s.Client.DeleteCalendarEvent(accountId, ids, ctx) require.NoError(err) require.Empty(result.Payload) @@ -208,7 +208,7 @@ func TestEvents(t *testing.T) { } { - result, err := s.client.QueryCalendarEvents(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) + result, err := s.Client.QueryCalendarEvents(ToNullQueryParams([]AccountId{accountId}), nil, filter, sortBy, true, ctx) require.NoError(err) require.Contains(result.Payload, accountId) resp := result.Payload[accountId] @@ -348,7 +348,7 @@ func (s *StalwartTest) fillCalendar( //NOSONAR cal.ShareWith = m } - result, err := s.client.CreateCalendar(accountId, cal, ctx) + result, err := s.Client.CreateCalendar(accountId, cal, ctx) if err != nil { return boxes, created, ss, as, err } @@ -383,7 +383,7 @@ func (s *StalwartTest) fillEvents( //NOSONAR user User, ) (AccountId, string, map[string]CalendarEvent, EventsBoxes, error) { require := require.New(t) - c, err := NewTestJmapClient(ctx.Session, user.email, user.password, true, true) + c, err := NewTestJmapClient(ctx.Session, user.Email, user.Password, true, true) require.NoError(err) defer c.Close() @@ -572,7 +572,7 @@ func (s *StalwartTest) fillEvents( //NOSONAR obj.RecurrenceRule = &rr } - result, err := s.client.CreateCalendarEvent(accountId, obj, ctx) + result, err := s.Client.CreateCalendarEvent(accountId, obj, ctx) if err != nil { return accountId, calendarId, nil, boxes, err } diff --git a/pkg/jmaptest/integration_email_test.go b/pkg/jmaptest/integration_email_test.go index 630b5e287d..cea49bcd64 100644 --- a/pkg/jmaptest/integration_email_test.go +++ b/pkg/jmaptest/integration_email_test.go @@ -40,7 +40,7 @@ func TestEmails(t *testing.T) { defer s.Close() user := pickUser() - session := s.Session(user.email) + session := s.Session(user.Email) ctx := s.Context(session) accountId := session.PrimaryAccounts.Mail @@ -57,18 +57,18 @@ func TestEmails(t *testing.T) { { { - result, err := s.client.GetIdentities(accountId, []string{}, ctx) + result, err := s.Client.GetIdentities(accountId, []string{}, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) require.Len(result.Payload.List, 2) - emailMatches := structs.Filter(result.Payload.List, func(i Identity) bool { return i.Email == user.email }) + emailMatches := structs.Filter(result.Payload.List, func(i Identity) bool { return i.Email == user.Email }) require.Len(emailMatches, 1) - aliasMatches := structs.Filter(result.Payload.List, func(i Identity) bool { return i.Email == user.alias }) + aliasMatches := structs.Filter(result.Payload.List, func(i Identity) bool { return i.Email == user.Alias }) require.Len(aliasMatches, 1) } { - result, err := s.client.GetAllMailboxes([]AccountId{accountId}, ctx) + result, err := s.Client.GetAllMailboxes([]AccountId{accountId}, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) require.Len(result.Payload, 1) @@ -84,7 +84,7 @@ func TestEmails(t *testing.T) { } { - result, err := s.client.GetAllEmailsInMailbox(accountId, inboxId, NullQueryParams, nil, true, false, 0, true, ctx) + result, err := s.Client.GetAllEmailsInMailbox(accountId, inboxId, NullQueryParams, nil, true, false, 0, true, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) @@ -98,7 +98,7 @@ func TestEmails(t *testing.T) { } { - result, err := s.client.GetAllEmailsInMailbox(accountId, inboxId, NullQueryParams, nil, false, false, 0, true, ctx) + result, err := s.Client.GetAllEmailsInMailbox(accountId, inboxId, NullQueryParams, nil, false, false, 0, true, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) @@ -125,29 +125,29 @@ func TestSendingEmails(t *testing.T) { defer s.Close() from := pickUser() - session := s.Session(from.email) + session := s.Session(from.Email) ctx := s.Context(session) accountId := session.PrimaryAccounts.Mail var to User { - others := structs.Filter(users[:], func(u User) bool { return u.name != from.name }) + others := structs.Filter(users[:], func(u User) bool { return u.Name != from.Name }) to = others[rand.Intn(len(others))] } - toSession := s.Session(to.email) + toSession := s.Session(to.Email) toAccountId := toSession.PrimaryAccounts.Mail var cc User { - others := structs.Filter(users[:], func(u User) bool { return u.name != from.name && u.name != to.name }) + others := structs.Filter(users[:], func(u User) bool { return u.Name != from.Name && u.Name != to.Name }) cc = others[rand.Intn(len(others))] } - ccSession := s.Session(cc.email) + ccSession := s.Session(cc.Email) ccAccountId := ccSession.PrimaryAccounts.Mail var mailboxPerRole map[string]Mailbox { - result, err := s.client.GetAllMailboxes([]AccountId{accountId}, ctx) + result, err := s.Client.GetAllMailboxes([]AccountId{accountId}, ctx) require.NoError(err) mailboxPerRole = structs.Index(result.Payload[accountId], func(m Mailbox) string { return m.Role }) require.Contains(mailboxPerRole, JmapMailboxRoleInbox) @@ -157,7 +157,7 @@ func TestSendingEmails(t *testing.T) { } { roles := []string{JmapMailboxRoleDrafts, JmapMailboxRoleSent, JmapMailboxRoleInbox} - result, err := s.client.SearchMailboxIdsPerRole([]AccountId{accountId}, roles, ctx) + result, err := s.Client.SearchMailboxIdsPerRole([]AccountId{accountId}, roles, ctx) require.NoError(err) require.Contains(result.Payload, accountId) a := result.Payload[accountId] @@ -177,7 +177,7 @@ func TestSendingEmails(t *testing.T) { Logger: ctx.Logger, AcceptLanguage: ctx.AcceptLanguage, } - result, err := s.client.GetAllMailboxes([]AccountId{u.accountId}, uctx) + result, err := s.Client.GetAllMailboxes([]AccountId{u.accountId}, uctx) require.NoError(err) for _, mailbox := range result.Payload[u.accountId] { require.Equal(0, mailbox.TotalEmails) @@ -185,16 +185,16 @@ func TestSendingEmails(t *testing.T) { } subject := fmt.Sprintf("Test Subject %d", 10000+rand.Intn(90000)) - fromName := fmt.Sprintf("%s (test %d)", from.name, 1000+rand.Intn(9000)) - sender := EmailAddress{Email: from.email, Name: from.description} + fromName := fmt.Sprintf("%s (test %d)", from.Name, 1000+rand.Intn(9000)) + sender := EmailAddress{Email: from.Email, Name: from.Description} { var identity Identity { - result, err := s.client.GetIdentities(accountId, []string{}, ctx) + result, err := s.Client.GetIdentities(accountId, []string{}, ctx) require.NoError(err) require.Len(result.Payload.List, 2) - matchesAlias := structs.Filter(result.Payload.List, func(i Identity) bool { return i.Email == from.alias }) + matchesAlias := structs.Filter(result.Payload.List, func(i Identity) bool { return i.Email == from.Alias }) require.Len(matchesAlias, 1) identity = matchesAlias[0] } @@ -206,14 +206,14 @@ func TestSendingEmails(t *testing.T) { } var created *Email { - result, err := s.client.CreateEmail(accountId, create, "", ctx) + result, err := s.Client.CreateEmail(accountId, create, "", ctx) require.NoError(err) created = result.Payload require.NotEmpty(created.Id) } { - result, err := s.client.GetEmails(accountId, []string{created.Id}, true, 0, false, false, ctx) + result, err := s.Client.GetEmails(accountId, []string{created.Id}, true, 0, false, false, ctx) require.NoError(err) require.Len(result.Payload.List, 1) require.Empty(result.Payload.NotFound) @@ -224,9 +224,9 @@ func TestSendingEmails(t *testing.T) { } update := EmailChange{ - From: []EmailAddress{{Name: fromName, Email: from.email}}, - To: []EmailAddress{{Name: to.description, Email: to.email}}, - Cc: []EmailAddress{{Name: cc.description, Email: cc.email}}, + From: []EmailAddress{{Name: fromName, Email: from.Email}}, + To: []EmailAddress{{Name: to.Description, Email: to.Email}}, + Cc: []EmailAddress{{Name: cc.Description, Email: cc.Email}}, Sender: []EmailAddress{sender}, Keywords: toBoolMapS("test"), Subject: subject, @@ -234,7 +234,7 @@ func TestSendingEmails(t *testing.T) { } var updated *Email { - result, err := s.client.CreateEmail(accountId, update, created.Id, ctx) + result, err := s.Client.CreateEmail(accountId, update, created.Id, ctx) require.NoError(err) updated = result.Payload require.NotNil(updated) @@ -244,7 +244,7 @@ func TestSendingEmails(t *testing.T) { var updatedMailboxId string { - result, err := s.client.GetEmails(accountId, []string{created.Id, updated.Id}, true, 0, false, false, ctx) + result, err := s.Client.GetEmails(accountId, []string{created.Id, updated.Id}, true, 0, false, false, ctx) require.NoError(err) require.Len(result.Payload.List, 1) require.Len(result.Payload.NotFound, 1) @@ -265,7 +265,7 @@ func TestSendingEmails(t *testing.T) { var sub EmailSubmission { - result, err := s.client.SubmitEmail(accountId, identity.Id, updated.Id, &move, ctx) + result, err := s.Client.SubmitEmail(accountId, identity.Id, updated.Id, &move, ctx) require.NoError(err) sub = result.Payload require.NotEmpty(sub.Id) @@ -275,20 +275,20 @@ func TestSendingEmails(t *testing.T) { require.Equal(sub.UndoStatus, UndoStatusPending) // this *might* be fragile: if the server is fast enough, would we get "final" here? require.Empty(sub.DsnBlobIds) require.Empty(sub.MdnBlobIds) - require.Equal(from.alias, sub.Envelope.MailFrom.Email) + require.Equal(from.Alias, sub.Envelope.MailFrom.Email) require.Nil(sub.Envelope.MailFrom.Parameters) require.Len(sub.Envelope.RcptTo, 2) - require.Contains(sub.Envelope.RcptTo, Address{Email: to.email}) - require.Contains(sub.Envelope.RcptTo, Address{Email: cc.email}) + require.Contains(sub.Envelope.RcptTo, Address{Email: to.Email}) + require.Contains(sub.Envelope.RcptTo, Address{Email: cc.Email}) require.NotZero(sub.SendAt) require.Len(sub.DeliveryStatus, 2) - require.Contains(sub.DeliveryStatus, to.email) - require.Contains(sub.DeliveryStatus, cc.email) + require.Contains(sub.DeliveryStatus, to.Email) + require.Contains(sub.DeliveryStatus, cc.Email) } a := 0 maxAttempts := 3 - delivery := sub.DeliveryStatus[to.email].Delivered + delivery := sub.DeliveryStatus[to.Email].Delivered for delivery != DeliveredYes { require.NotEqual(DeliveredNo, delivery) @@ -298,13 +298,13 @@ func TestSendingEmails(t *testing.T) { } time.Sleep(1 * time.Second) - result, err := s.client.GetEmailSubmissionStatus(accountId, []string{sub.Id}, ctx) + result, err := s.Client.GetEmailSubmissionStatus(accountId, []string{sub.Id}, ctx) require.NoError(err) require.Empty(result.Payload.NotFound) submittedIds := structs.Map(result.Payload.List, func(s EmailSubmission) string { return s.Id }) require.Contains(submittedIds, sub.Id) subs := structs.Index(result.Payload.List, func(s EmailSubmission) string { return s.Id }) - delivery = subs[sub.Id].DeliveryStatus[to.email].Delivered + delivery = subs[sub.Id].DeliveryStatus[to.Email].Delivered } require.Contains([]DeliveryStatusDelivered{DeliveredYes, DeliveredUnknown}, delivery) @@ -322,7 +322,7 @@ func TestSendingEmails(t *testing.T) { } inboxId := "" { - result, err := s.client.GetAllMailboxes([]AccountId{r.accountId}, rctx) + result, err := s.Client.GetAllMailboxes([]AccountId{r.accountId}, rctx) require.NoError(err) for _, mailbox := range result.Payload[r.accountId] { if mailbox.Role == JmapMailboxRoleInbox { @@ -330,26 +330,26 @@ func TestSendingEmails(t *testing.T) { require.Equal(1, mailbox.TotalEmails) } } - require.NotEmpty(inboxId, "failed to find the Mailbox with the 'inbox' role for %v", r.user.email) + require.NotEmpty(inboxId, "failed to find the Mailbox with the 'inbox' role for %v", r.user.Email) } - result, err := s.client.QueryEmails([]AccountId{r.accountId}, EmailFilterCondition{InMailbox: inboxId}, 0, 0, true, 0, rctx) + result, err := s.Client.QueryEmails([]AccountId{r.accountId}, EmailFilterCondition{InMailbox: inboxId}, 0, 0, true, 0, rctx) require.NoError(err) require.Contains(result.Payload, r.accountId) require.Len(result.Payload[r.accountId].Results, 1) received := result.Payload[r.accountId].Results[0] require.Len(received.From, 1) - require.Equal(from.email, received.From[0].Email) + require.Equal(from.Email, received.From[0].Email) require.Equal(fromName, received.From[0].Name) require.Len(received.Sender, 1) - require.Equal(from.email, received.Sender[0].Email) - require.Equal(from.description, received.Sender[0].Name) + require.Equal(from.Email, received.Sender[0].Email) + require.Equal(from.Description, received.Sender[0].Name) require.Len(received.To, 1) - require.Equal(to.email, received.To[0].Email) - require.Equal(to.description, received.To[0].Name) + require.Equal(to.Email, received.To[0].Email) + require.Equal(to.Description, received.To[0].Name) require.Len(received.Cc, 1) - require.Equal(cc.email, received.Cc[0].Email) - require.Equal(cc.description, received.Cc[0].Name) + require.Equal(cc.Email, received.Cc[0].Email) + require.Equal(cc.Description, received.Cc[0].Name) require.Equal(subject, received.Subject) } } @@ -392,7 +392,7 @@ func matchEmail(t *testing.T, actual Email, expected filledMail, hasBodies bool) func (s *StalwartTest) findInbox(t *testing.T, accountId AccountId, ctx Context) (string, string) { require := require.New(t) - result, err := s.client.GetAllMailboxes([]AccountId{accountId}, ctx) + result, err := s.Client.GetAllMailboxes([]AccountId{accountId}, ctx) require.NoError(err) require.Equal(ctx.Session.State, result.GetSessionState()) require.Len(result.Payload, 1) @@ -521,7 +521,7 @@ var allKeywords = map[string]imap.Flag{ } func (s *StalwartTest) fillEmailsWithImap(folder string, count int, empty bool, user User) ([]filledMail, int, error) { //NOSONAR - to := fmt.Sprintf("%s <%s>", user.description, user.email) + to := fmt.Sprintf("%s <%s>", user.Description, user.Email) ccEvery := 2 bccEvery := 3 attachmentEvery := 2 @@ -543,7 +543,7 @@ func (s *StalwartTest) fillEmailsWithImap(folder string, count int, empty bool, } }(c) - if err = c.Login(user.email, user.password).Wait(); err != nil { + if err = c.Login(user.Email, user.Password).Wait(); err != nil { return nil, 0, err } @@ -591,7 +591,7 @@ func (s *StalwartTest) fillEmailsWithImap(folder string, count int, empty bool, domain := addressParts[0][2] toName := displayName - toAddress := fmt.Sprintf("%s@%s", user.email, domain) + toAddress := fmt.Sprintf("%s@%s", user.Email, domain) ccName1 := "Team Lead" ccAddress1 := fmt.Sprintf("lead@%s", domain) ccName2 := "Coworker" diff --git a/pkg/jmaptest/integration_ws_test.go b/pkg/jmaptest/integration_ws_test.go index 3700d18886..7407b09b8a 100644 --- a/pkg/jmaptest/integration_ws_test.go +++ b/pkg/jmaptest/integration_ws_test.go @@ -69,7 +69,7 @@ func TestWs(t *testing.T) { defer s.Close() user := pickUser() - session := s.Session(user.email) + session := s.Session(user.Email) ctx := s.Context(session) mailAccountId := session.PrimaryAccounts.Mail @@ -78,8 +78,8 @@ func TestWs(t *testing.T) { _, inboxFolder = s.findInbox(t, mailAccountId, ctx) } - l := &testWsPushListener{t: t, username: user.email, logger: s.logger, mailAccountId: mailAccountId} - s.client.AddWsPushListener(l) + l := &testWsPushListener{t: t, username: user.Email, logger: s.logger, mailAccountId: mailAccountId} + s.Client.AddWsPushListener(l) require.Equal(uint32(0), l.calls.Load()) { @@ -92,7 +92,7 @@ func TestWs(t *testing.T) { var initialState State { - result, err := s.client.GetEmailChanges(mailAccountId, EmptyState, true, 0, 0, ctx) + result, err := s.Client.GetEmailChanges(mailAccountId, EmptyState, true, 0, 0, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) require.NotEmpty(result.GetState()) @@ -106,7 +106,7 @@ func TestWs(t *testing.T) { require.NotEmpty(initialState) { - result, err := s.client.GetEmailChanges(mailAccountId, initialState, true, 0, 0, ctx) + result, err := s.Client.GetEmailChanges(mailAccountId, initialState, true, 0, 0, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) require.Equal(initialState, result.GetState()) @@ -116,7 +116,7 @@ func TestWs(t *testing.T) { require.Empty(result.Payload.Updated) } - wsc, err := s.client.EnablePushNotifications(cotx, initialState, func() (*Session, error) { return session, nil }) + wsc, err := s.Client.EnablePushNotifications(cotx, initialState, func() (*Session, error) { return session, nil }) require.NoError(err) defer wsc.Close() @@ -149,7 +149,7 @@ func TestWs(t *testing.T) { } var lastState State { - result, err := s.client.GetEmailChanges(mailAccountId, initialState, true, 0, 0, ctx) + result, err := s.Client.GetEmailChanges(mailAccountId, initialState, true, 0, 0, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) require.NotEqual(initialState, result.GetState()) @@ -183,7 +183,7 @@ func TestWs(t *testing.T) { l.m.Unlock() } { - result, err := s.client.GetEmailChanges(mailAccountId, lastState, true, 0, 0, ctx) + result, err := s.Client.GetEmailChanges(mailAccountId, lastState, true, 0, 0, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) require.NotEqual(lastState, result.GetState()) @@ -217,7 +217,7 @@ func TestWs(t *testing.T) { l.m.Unlock() } { - result, err := s.client.GetEmailChanges(mailAccountId, lastState, true, 0, 0, ctx) + result, err := s.Client.GetEmailChanges(mailAccountId, lastState, true, 0, 0, ctx) require.NoError(err) require.Equal(session.State, result.GetSessionState()) require.NotEqual(lastState, result.GetState()) diff --git a/pkg/jmaptest/jmap_stalwart.go b/pkg/jmaptest/jmap_stalwart.go index aba3eec0ef..03d4af2d1c 100644 --- a/pkg/jmaptest/jmap_stalwart.go +++ b/pkg/jmaptest/jmap_stalwart.go @@ -14,6 +14,7 @@ import ( "math/rand" "net/http" "net/http/httputil" + "net/netip" "net/url" "os" "reflect" @@ -25,6 +26,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + dockercontainer "github.com/moby/moby/api/types/container" dockernetwork "github.com/moby/moby/api/types/network" "github.com/stretchr/testify/require" @@ -75,11 +77,11 @@ const ( ) type User struct { - name string - description string - alias string - password string - email string + Name string + Description string + Alias string + Password string + Email string } func userpassword() string { @@ -92,7 +94,7 @@ func userpassword() string { func mkuser(name string, description string, alias string) User { parts := strings.Split(alias, "@") - return User{name: name, description: description, alias: alias, email: name + "@" + parts[1], password: userpassword()} + return User{Name: name, Description: description, Alias: alias, Email: name + "@" + parts[1], Password: userpassword()} } var ( @@ -214,16 +216,20 @@ func Skip(t *testing.T) bool { } type StalwartTest struct { - t *testing.T - ip string - imapPort uint16 - container *testcontainers.DockerContainer - ctx context.Context - cancelCtx context.CancelFunc - client *jmap.Client - logger *oclog.Logger - jmapBaseUrl *url.URL - sessionUrl *url.URL + t *testing.T + ip string + imapPort uint16 + container *testcontainers.DockerContainer + ctx context.Context + cancelCtx context.CancelFunc + logger *oclog.Logger + sessionUrl *url.URL + + Client *jmap.Client + JmapBaseUrl *url.URL + MasterUsername string + MasterPassword string + Users []User io.Closer } @@ -249,43 +255,11 @@ func (s *StalwartTest) Context(session *jmap.Session) jmap.Context { } func (s *StalwartTest) Session(username string) *jmap.Session { - session, err := s.client.FetchSession(s.ctx, s.sessionUrl, username, s.logger) + session, err := s.Client.FetchSession(s.ctx, s.sessionUrl, username, s.logger) require.NoError(s.t, err, "failed to authenticate user '%s' and/or retrieve their JMAP session using the URL '%s'", username, s.sessionUrl.String()) require.NotNil(s.t, session.Capabilities.Mail) require.NotNil(s.t, session.Capabilities.Calendars) require.NotNil(s.t, session.Capabilities.Contacts) - - // we have to overwrite the hostname in JMAP URL because the container - // will know its name to be a random Docker container identifier, or - // "localhost" as we defined the hostname in the Stalwart configuration, - // and we also need to overwrite the port number as its not mapped - - session.JmapUrl.Host = s.jmapBaseUrl.Host - session.JmapUrl.Scheme = "http" // replace https with http - session.WebsocketUrl.Host = s.jmapBaseUrl.Host - session.WebsocketUrl.Scheme = "ws" // replace wss with ws - - if v, err := replaceHostProto(session.ApiUrl, "http", s.jmapBaseUrl.Host); err != nil { - require.NoError(s.t, err) - } else { - session.ApiUrl = v - } - if v, err := replaceHostProto(session.DownloadUrl, "http", s.jmapBaseUrl.Host); err != nil { - require.NoError(s.t, err) - } else { - session.DownloadUrl = v - } - if v, err := replaceHostProto(session.UploadUrl, "http", s.jmapBaseUrl.Host); err != nil { - require.NoError(s.t, err) - } else { - session.UploadUrl = v - } - if v, err := replaceHostProto(session.EventSourceUrl, "http", s.jmapBaseUrl.Host); err != nil { - require.NoError(s.t, err) - } else { - session.EventSourceUrl = v - } - return &session } @@ -609,10 +583,10 @@ func createManagementObject(ctx context.Context, h http.Client, url string, admi } } -func createJmapClient(container *testcontainers.DockerContainer, ctx context.Context, auth jmap.HttpJmapClientAuthenticator, logger *oclog.Logger) (jmap.Client, *url.URL, *url.URL, error) { +func createJmapClient(container *testcontainers.DockerContainer, ctx context.Context, auth jmap.HttpJmapClientAuthenticator, logger *oclog.Logger) (*jmap.Client, *url.URL, *url.URL, error) { ip, err := container.Host(ctx) if err != nil { - return jmap.Client{}, nil, nil, err + return nil, nil, nil, err } tlsConfig := &tls.Config{InsecureSkipVerify: true} @@ -630,7 +604,7 @@ func createJmapClient(container *testcontainers.DockerContainer, ctx context.Con jmapPort, err := container.MappedPort(ctx, httpPort) if err != nil { - return jmap.Client{}, nil, nil, err + return nil, nil, nil, err } jmapBaseUrl := &url.URL{ Scheme: "http", @@ -649,11 +623,11 @@ func createJmapClient(container *testcontainers.DockerContainer, ctx context.Con cmd := []string{Wireshark, "-pkSl", "-i", "lo", "-f", fmt.Sprintf("port %d", jmapPort.Num()), "-Y", "http||websocket"} process, err := os.StartProcess(Wireshark, cmd, &attr) if err != nil { - return jmap.Client{}, nil, nil, err + return nil, nil, nil, err } err = process.Release() if err != nil { - return jmap.Client{}, nil, nil, err + return nil, nil, nil, err } time.Sleep(10 * time.Second) @@ -665,7 +639,7 @@ func createJmapClient(container *testcontainers.DockerContainer, ctx context.Con wscf, err := jmap.NewHttpWsClientFactory(wsd, auth, logger, eventListener, true, 8192) if err != nil { - return jmap.Client{}, nil, nil, err + return nil, nil, nil, err } return jmap.NewClient(api, api, api, wscf), jmapBaseUrl, sessionUrl, nil @@ -741,17 +715,17 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT option(&settings) } - var net *testcontainers.DockerNetwork + var containerNetwork *testcontainers.DockerNetwork { if useNetwork { if n, err := network.New(ctx); err != nil { return nil, err } else { testcontainers.CleanupNetwork(t, n) - net = n + containerNetwork = n } } else { - net = nil + containerNetwork = nil } } @@ -791,8 +765,8 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT }, ), } - if net != nil { - opts = append(opts, network.WithNetwork([]string{recoveryAlias}, net)) + if containerNetwork != nil { + opts = append(opts, network.WithNetwork([]string{recoveryAlias}, containerNetwork)) } stalwartImage := fmt.Sprintf(stalwartImageTemplate, StalwartVersion) @@ -805,7 +779,7 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT // now that the container is running in recovery mode, we can use the stalwart CLI to import // the configuration - if output, err := importConfig(t, recovery, ctx, net, recoveryAlias, adminUsername, adminPassword, settings, false); err != nil { + if output, err := importConfig(t, recovery, ctx, containerNetwork, recoveryAlias, adminUsername, adminPassword, settings, false); err != nil { return nil, err } else { t.Logf("Output of applying configuration:\n%s", strings.Join(output, "")) //NOSONAR @@ -819,13 +793,45 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT } } + httpHostPort, err := FreeLocalhostPort() + require.NoError(t, err) + imapsHostPort, err := FreeLocalhostPort() + require.NoError(t, err) + + publicUrl := fmt.Sprintf("http://127.0.0.1:%d", httpHostPort) + // and now we start the container in "proper" mode (not recovery), re-using the same volume for data var container *testcontainers.DockerContainer { + parsedHttpPort, err := dockernetwork.ParsePort(httpPort + "/tcp") + require.NoError(t, err) + parsedImapsPort, err := dockernetwork.ParsePort(imapsPort + "/tcp") + require.NoError(t, err) + address, err := netip.ParseAddr("127.0.0.1") + require.NoError(t, err) + opts := []testcontainers.ContainerCustomizer{ testcontainers.WithLogConsumers(&printingLogConsumer{prefix: "STALWART"}), testcontainers.WithExposedPorts(httpPort+"/tcp", imapsPort+"/tcp"), + testcontainers.WithHostConfigModifier(func(hc *dockercontainer.HostConfig) { + hc.PortBindings = dockernetwork.PortMap{ + dockernetwork.Port(parsedHttpPort): []dockernetwork.PortBinding{ + { + HostIP: address, + HostPort: fmt.Sprintf("%d", httpHostPort), + }, + }, + dockernetwork.Port(parsedImapsPort): []dockernetwork.PortBinding{ + { + HostIP: address, + HostPort: fmt.Sprintf("%d", imapsHostPort), + }, + }, + } + + }), testcontainers.WithEnv(map[string]string{ + "STALWART_PUBLIC_URL": publicUrl, "STALWART_RECOVERY_ADMIN": strings.Join([]string{adminUsername, adminPassword}, ":"), }), testcontainers.WithFiles(testcontainers.ContainerFile{ @@ -850,8 +856,8 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT ), } - if net != nil { - opts = append(opts, network.WithNetwork([]string{containerAlias}, net)) + if containerNetwork != nil { + opts = append(opts, network.WithNetwork([]string{containerAlias}, containerNetwork)) } stalwartImage := fmt.Sprintf(stalwartImageTemplate, StalwartVersion) @@ -866,7 +872,7 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT if !useRecoveryContainer { // we didn't use a recovery container to initialize the configuration, we will use the // regular container to do so - if output, err := importConfig(t, container, ctx, net, containerAlias, adminUsername, adminPassword, settings, true); err != nil { + if output, err := importConfig(t, container, ctx, containerNetwork, containerAlias, adminUsername, adminPassword, settings, true); err != nil { t.Logf("Output of applying configuration:\n%s", strings.Join(output, "")) return nil, fmt.Errorf("failed to import configuration: %w: output: %s", err, strings.Join(output, "")) } else { @@ -887,7 +893,7 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT } domains := structs.Uniq(structs.Map(users[:], func(u User) string { - parts := strings.Split(u.email, "@") + parts := strings.Split(u.Email, "@") return parts[1] })) slices.Sort(domains) @@ -948,26 +954,26 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT } for _, user := range users { - fmt.Printf("Creating individual '%v'\n", user.name) + fmt.Printf("Creating individual '%v'\n", user.Name) localPart := "" domain := "" domainId := "" { - parts := strings.Split(user.alias, "@") + parts := strings.Split(user.Alias, "@") localPart = parts[0] domain = parts[1] if v, ok := domainIds[domain]; ok { domainId = v } else { - require.FailNow(t, "failed to find id for domain '%s' in user email address '%s'", domain, user.email) + require.FailNow(t, "failed to find id for domain '%s' in user email address '%s'", domain, user.Email) } } _, err := createManagementObject(ctx, h, url, adminUsername, adminPassword, "Account", map[string]any{ "@type": "User", - "name": user.name, - "description": user.description, + "name": user.Name, + "description": user.Description, "aliases": map[string]any{ "0": map[string]any{ "enabled": true, @@ -978,7 +984,7 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT "credentials": map[string]any{ "0": map[string]any{ "@type": "Password", - "secret": user.password, + "secret": user.Password, "allowedIps": map[string]any{}, "expiresAt": nil, }, @@ -1012,7 +1018,7 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT }) require.NoError(t, err) - usersByName := structs.Index(users[:], func(u User) string { return u.name }) + usersByName := structs.Index(users[:], func(u User) string { return u.Name }) { mr := result["methodResponses"].([]any) f := mr[0].([]any) @@ -1023,11 +1029,11 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT name := u["name"].(string) email := u["emailAddress"].(string) if match, ok := usersByName[name]; ok { - require.Equal(t, match.email, email) + require.Equal(t, match.Email, email) aliases := u["aliases"].(map[string]any) require.Len(t, aliases, 1) for _, alias := range aliases { - parts := strings.Split(match.alias, "@") + parts := strings.Split(match.Alias, "@") alias := alias.(map[string]any) require.True(t, alias["enabled"].(bool)) require.Equal(t, parts[0], alias["name"].(string)) @@ -1058,12 +1064,18 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT return nil, err } - // check whether we can fetch a session for the provisioned users + // check whether we can fetch a session for the provisioned users, + // but since these are flaky once in a blue moon, we will retry for _, user := range users { - ctx := context.WithValue(ctx, "password", user.password) - session, err := j.FetchSession(ctx, sessionUrl, user.email, oclog.From(logger.With().Str("username", user.email).Str("password", user.password))) - require.NoError(t, err, "failed to retrieve JMAP session for newly created principal '%s'", user.email) - require.Equal(t, user.email, session.Username) + require.Eventually(t, func() bool { + ctx := context.WithValue(ctx, "password", user.Password) + session, err := j.FetchSession(ctx, sessionUrl, user.Email, oclog.From(logger.With().Str("username", user.Email).Str("password", user.Password))) + if err != nil { + return false + } + require.Equal(t, user.Email, session.Username) + return true + }, 5*time.Second, 500*time.Millisecond, "failed to retrieve JMAP session for newly created principal '%s'", user.Email) } } } @@ -1079,16 +1091,19 @@ func NewStalwartTest(t *testing.T, options ...func(*importSettings)) (*StalwartT } return &StalwartTest{ - t: t, - ip: ip, - imapPort: imapPort.Num(), - container: container, - ctx: ctx, - cancelCtx: cancel, - client: &j, - logger: logger, - jmapBaseUrl: jmapBaseUrl, - sessionUrl: sessionUrl, + t: t, + ip: ip, + imapPort: imapPort.Num(), + container: container, + ctx: ctx, + cancelCtx: cancel, + logger: logger, + sessionUrl: sessionUrl, + Client: j, + JmapBaseUrl: jmapBaseUrl, + MasterUsername: masterUsername, + MasterPassword: masterPassword, + Users: users[:], }, nil } @@ -1767,7 +1782,7 @@ func containerTest[OBJ jmap.Idable, RESP jmap.GetResponse[OBJ], BOXES any, CHANG defer s.Close() user := pickUser() - session := s.Session(user.email) + session := s.Session(user.Email) ctx := s.Context(session) accountId := acc(session) @@ -1775,7 +1790,7 @@ func containerTest[OBJ jmap.Idable, RESP jmap.GetResponse[OBJ], BOXES any, CHANG // we first need to retrieve the list of all the Principals in order to be able to use and test sharing principalIds := []jmap.PrincipalId{} { - result, err := s.client.GetPrincipals(accountId, []jmap.PrincipalId{}, ctx) + result, err := s.Client.GetPrincipals(accountId, []jmap.PrincipalId{}, ctx) require.NoError(err) require.NotEmpty(result.Payload.List) principalIds = structs.Map(result.Payload.List, func(p jmap.Principal) jmap.PrincipalId { return jmap.PrincipalId(p.Id) }) diff --git a/pkg/jmaptest/tools.go b/pkg/jmaptest/tools.go index 143f2c1dc1..08d08c40d5 100644 --- a/pkg/jmaptest/tools.go +++ b/pkg/jmaptest/tools.go @@ -1,6 +1,11 @@ package jmaptest -import "github.com/opencloud-eu/opencloud/pkg/jmap" +import ( + "net" + "sync" + + "github.com/opencloud-eu/opencloud/pkg/jmap" +) var ( truep = ptr(true) @@ -26,3 +31,22 @@ func firstKey[K comparable, V any](m map[K]V) (K, bool) { var zero K return zero, false } + +var freeLocalhostPortSync = sync.Mutex{} + +func FreeLocalhostPort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + + freeLocalhostPortSync.Lock() + defer freeLocalhostPortSync.Unlock() + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + + return l.Addr().(*net.TCPAddr).Port, nil +} diff --git a/services/groupware/pkg/groupware/api_index_test.go b/services/groupware/pkg/groupware/api_index_test.go new file mode 100644 index 0000000000..2180873f73 --- /dev/null +++ b/services/groupware/pkg/groupware/api_index_test.go @@ -0,0 +1,33 @@ +package groupware + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGroupwareIndex(t *testing.T) { + require := require.New(t) + g, err := newGroupwareTest(t) + require.NoError(err) + require.NotEmpty(g.Users) + + client := &http.Client{} + for _, u := range g.Users { + req, err := http.NewRequest(http.MethodGet, g.BaseURL, nil) + require.NoError(err) + req.SetBasicAuth(u.Name, u.Password) + resp, err := client.Do(req) + require.NoError(err) + require.Equal(200, resp.StatusCode) + defer resp.Body.Close() + index := IndexResponse{} + err = json.NewDecoder(resp.Body).Decode(&index) + require.NoError(err) + require.Len(index.Accounts, 1) + require.Equal(u.Email, index.Accounts[0].Name) + require.Len(index.Accounts[0].Identities, 2) // email + alias + } +} diff --git a/services/groupware/pkg/groupware/framework.go b/services/groupware/pkg/groupware/framework.go index 099e583b0b..43d3b0ccf0 100644 --- a/services/groupware/pkg/groupware/framework.go +++ b/services/groupware/pkg/groupware/framework.go @@ -179,7 +179,7 @@ func (r groupwareHttpJmapApiClientMetricsRecorder) OnFailedWsHandshakeRequestWit // TODO metrics for WSS } -func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prometheusRegistry prometheus.Registerer) (*Groupware, error) { //NOSONAR +func NewGroupwareUsingJmapClient(jmapClient *jmap.Client, config *config.Config, logger *log.Logger, mux *chi.Mux, prometheusRegistry prometheus.Registerer) (*Groupware, error) { //NOSONAR baseUrl, err := url.Parse(config.Mail.BaseUrl) if err != nil { logger.Error().Err(err).Msgf("failed to parse configured Mail.Baseurl '%v'", config.Mail.BaseUrl) @@ -192,11 +192,9 @@ func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prome maxBodyValueBytes := max(config.Mail.MaxBodyValueBytes, 0) sendDurationsResponse := config.HTTP.SendDurationsResponse defaultContactLimit := ptrIfNot(max(config.Mail.DefaultContactLimit, 0), 0) - responseHeaderTimeout := max(config.Mail.ResponseHeaderTimeout, 0) sessionCacheMaxCapacity := uint64(max(config.Mail.SessionCache.MaxCapacity, 0)) sessionCacheTtl := max(config.Mail.SessionCache.Ttl, 0) sessionFailureCacheTtl := max(config.Mail.SessionCache.FailureTtl, 0) - wsHandshakeTimeout := config.Mail.PushHandshakeTimeout eventChannelSize := 100 // TODO make channel queue buffering size configurable workerQueueSize := 100 // TODO configuration setting @@ -207,8 +205,6 @@ func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prome useDnsForSessionResolution := false // TODO configuration setting, although still experimental, needs proper unit tests first - insecureTls := config.HTTP.Insecure - sanitize := true // TODO make configurable m, err := metrics.New(prometheusRegistry, logger) @@ -218,79 +214,6 @@ func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prome userProvider := newRevaContextUsernameProvider() - jmapMetricsAdapter := groupwareHttpJmapApiClientMetricsRecorder{m: m} - - var jmapClient jmap.Client - { - var auth jmap.HttpJmapClientAuthenticator - { - masterUsername := config.Mail.Master.Username - masterPassword := config.Mail.Master.Password - if masterUsername != "" && masterPassword != "" { - auth = jmap.NewMasterAuthHttpJmapClientAuthenticator(masterUsername, masterPassword) - } else { - auth = newRevaBearerHttpJmapClientAuthenticator() - } - } - - var api *jmap.HttpJmapClient - { - // TODO add timeouts and other meaningful configuration settings for the HTTP client - var httpClient http.Client - { - httpTransport := http.DefaultTransport.(*http.Transport).Clone() - httpTransport.ResponseHeaderTimeout = responseHeaderTimeout - if insecureTls { - tlsConfig := &tls.Config{InsecureSkipVerify: true} // #nosec G402 insecure TLS is a configuration option for development - httpTransport.TLSClientConfig = tlsConfig - } - httpClient = *http.DefaultClient - httpClient.Transport = httpTransport - } - - api = jmap.NewHttpJmapClient( - &httpClient, - auth, - jmapMetricsAdapter, - config.HTTP.TraceRequests, - config.HTTP.TraceMaxRequestBodySize, - config.HTTP.TraceResponses, - config.HTTP.TraceMaxResponseBodySize, - ) - defer func() { - if err := api.Close(); err != nil { - logger.Error().Err(err).Msgf("failed to close HTTP JMAP API client") - } - }() - } - - var wsf *jmap.HttpWsClientFactory - { - wsDialer := &websocket.Dialer{ - HandshakeTimeout: wsHandshakeTimeout, - } - if insecureTls { - wsDialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402 insecure TLS is a configuration option for development - } - - wsf, err = jmap.NewHttpWsClientFactory(wsDialer, auth, logger, jmapMetricsAdapter, - config.HTTP.TraceResponses, config.HTTP.TraceMaxResponseBodySize, - ) - if err != nil { - logger.Error().Err(err).Msg("failed to create websocket client") - return nil, GroupwareInitializationError{Message: "failed to create websocket client", Err: err} - } - } - - // api implements all three interfaces: - jmapClient = jmap.NewClient(api, api, api, wsf) - defer func() { - if err := jmapClient.Close(); err != nil { - logger.Error().Err(err).Msgf("failed to close JMAP client") - } - }() - } - sessionCacheBuilder := newSessionCacheBuilder( sessionUrl, logger, @@ -404,8 +327,8 @@ func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prome var suppliers Suppliers { impls := []Supplier[jmap.Foo]{ - newJmapAddressBookSupplier(&jmapClient), - newJmapContactCardSupplier(&jmapClient), + newJmapAddressBookSupplier(jmapClient), + newJmapContactCardSupplier(jmapClient), } if config.EnableMockData { impls = append(impls, @@ -434,7 +357,7 @@ func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prome logger: logger, sessionCache: sessionCache, userProvider: userProvider, - jmap: &jmapClient, + jmap: jmapClient, defaults: groupwareDefaults{ emailLimit: defaultEmailLimit, contactLimit: defaultContactLimit, @@ -470,6 +393,92 @@ func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prome return g, nil } +func NewGroupware(config *config.Config, logger *log.Logger, mux *chi.Mux, prometheusRegistry prometheus.Registerer) (*Groupware, error) { //NOSONAR + responseHeaderTimeout := max(config.Mail.ResponseHeaderTimeout, 0) + wsHandshakeTimeout := config.Mail.PushHandshakeTimeout + insecureTls := config.HTTP.Insecure + + m, err := metrics.New(prometheusRegistry, logger) + if err != nil { + return nil, GroupwareInitializationError{Message: fmt.Sprintf("failed to configure metrics: %v", err), Err: err} + } + + jmapMetricsAdapter := groupwareHttpJmapApiClientMetricsRecorder{m: m} + + var jmapClient *jmap.Client + { + var auth jmap.HttpJmapClientAuthenticator + { + masterUsername := config.Mail.Master.Username + masterPassword := config.Mail.Master.Password + if masterUsername != "" && masterPassword != "" { + auth = jmap.NewMasterAuthHttpJmapClientAuthenticator(masterUsername, masterPassword) + } else { + auth = newRevaBearerHttpJmapClientAuthenticator() + } + } + + var api *jmap.HttpJmapClient + { + // TODO add timeouts and other meaningful configuration settings for the HTTP client + var httpClient http.Client + { + httpTransport := http.DefaultTransport.(*http.Transport).Clone() + httpTransport.ResponseHeaderTimeout = responseHeaderTimeout + if insecureTls { + tlsConfig := &tls.Config{InsecureSkipVerify: true} // #nosec G402 insecure TLS is a configuration option for development + httpTransport.TLSClientConfig = tlsConfig + } + httpClient = *http.DefaultClient + httpClient.Transport = httpTransport + } + + api = jmap.NewHttpJmapClient( + &httpClient, + auth, + jmapMetricsAdapter, + config.HTTP.TraceRequests, + config.HTTP.TraceMaxRequestBodySize, + config.HTTP.TraceResponses, + config.HTTP.TraceMaxResponseBodySize, + ) + defer func() { + if err := api.Close(); err != nil { + logger.Error().Err(err).Msgf("failed to close HTTP JMAP API client") + } + }() + } + + var wsf *jmap.HttpWsClientFactory + { + wsDialer := &websocket.Dialer{ + HandshakeTimeout: wsHandshakeTimeout, + } + if insecureTls { + wsDialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402 insecure TLS is a configuration option for development + } + + wsf, err = jmap.NewHttpWsClientFactory(wsDialer, auth, logger, jmapMetricsAdapter, + config.HTTP.TraceResponses, config.HTTP.TraceMaxResponseBodySize, + ) + if err != nil { + logger.Error().Err(err).Msg("failed to create websocket client") + return nil, GroupwareInitializationError{Message: "failed to create websocket client", Err: err} + } + } + + // api implements all three interfaces: + jmapClient = jmap.NewClient(api, api, api, wsf) + defer func() { + if err := jmapClient.Close(); err != nil { + logger.Error().Err(err).Msgf("failed to close JMAP client") + } + }() + } + + return NewGroupwareUsingJmapClient(jmapClient, config, logger, mux, prometheusRegistry) +} + func (g *Groupware) worker(jobs <-chan Job, busy *atomic.Int32) { for job := range jobs { busy.Add(1) diff --git a/services/groupware/pkg/groupware/groupware_integration_test.go b/services/groupware/pkg/groupware/groupware_integration_test.go new file mode 100644 index 0000000000..87c41647ca --- /dev/null +++ b/services/groupware/pkg/groupware/groupware_integration_test.go @@ -0,0 +1,172 @@ +package groupware + +import ( + "fmt" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + "github.com/go-chi/chi/v5" + "github.com/opencloud-eu/opencloud/pkg/cors" + "github.com/opencloud-eu/opencloud/pkg/jmaptest" + opencloudmiddleware "github.com/opencloud-eu/opencloud/pkg/middleware" + "github.com/opencloud-eu/opencloud/pkg/shared" + "github.com/opencloud-eu/opencloud/pkg/structs" + "github.com/opencloud-eu/opencloud/services/groupware/pkg/config" + "github.com/opencloud-eu/opencloud/services/groupware/pkg/logging" + groupwaremiddleware "github.com/opencloud-eu/opencloud/services/groupware/pkg/middleware" + revactx "github.com/opencloud-eu/reva/v2/pkg/ctx" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func init() { + chi.RegisterMethod("REPORT") +} + +type GroupwareTest struct { + BaseURL string + Users []jmaptest.User +} + +func newGroupwareTest(t *testing.T) (GroupwareTest, error) { + require := require.New(t) + s, err := jmaptest.NewStalwartTest(t) + t.Cleanup(func() { s.Close() }) + require.NoError(err) + + root := "" + { + const charset = "abcdefghijklmnopqrstuvwxyz" + l := 4 + rand.Intn(27) + var sb strings.Builder + sb.Grow(l) + for range l { + r := rand.Intn(len(charset)) + sb.WriteByte(charset[r]) + } + root = sb.String() + } + + httpPort, err := jmaptest.FreeLocalhostPort() + require.NoError(err) + + config := &config.Config{ + Service: config.Service{ + Name: "TestGroupware", + }, + Log: &config.Log{ + Level: "trace", + Pretty: true, + Color: false, + }, + Mail: config.Mail{ + Master: config.MailMasterAuth{ + Username: s.MasterUsername, + Password: s.MasterPassword, + }, + BaseUrl: s.JmapBaseUrl.String(), + SessionCache: config.MailSessionCache{ + MaxCapacity: 1, + }, + }, + HTTP: config.HTTP{ + Addr: fmt.Sprintf("127.0.0.1:%d", httpPort), + Insecure: true, + Root: "/" + root, + TLS: shared.HTTPServiceTLS{ + Enabled: false, + }, + Namespace: "eu.opencloud.web", + OpenCloudPublicURL: fmt.Sprintf("http://127.0.0.1:%d", httpPort), + SendDurationsResponse: true, + CORS: config.CORS{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "REPORT"}, + AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With", "X-Request-Id", "Trace-Id", "Cache-Control"}, + AllowCredentials: true, + }, + TraceRequests: true, + TraceMaxRequestBodySize: 8 * 1024, + TraceResponses: true, + TraceMaxResponseBodySize: 8 * 1024, + }, + TokenManager: &config.TokenManager{ + JWTSecret: "some-opencloud-jwt-secret", + }, + } + + logger := logging.Configure("TestGroupware", config.Log) + prom := prometheus.NewRegistry() + m := chi.NewMux() + + gw, err := NewGroupwareUsingJmapClient(s.Client, config, &logger, m, prom) + require.NoError(err) + + auth := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + username, password, ok := r.BasicAuth() + if !ok { + w.WriteHeader(http.StatusUnauthorized) + return + } + if match, ok := structs.First(s.Users, func(u jmaptest.User) bool { + return u.Name == username + }); !ok { + w.WriteHeader(http.StatusUnauthorized) + return + } else if match.Password != password { + w.WriteHeader(http.StatusUnauthorized) + return + } else { + u := &userv1beta1.User{ + Username: match.Email, + Mail: match.Email, + } + ctx = revactx.ContextSetUser(ctx, u) + } + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } + + m.Use( + opencloudmiddleware.Cors( + cors.Logger(logger), + cors.AllowedOrigins(config.HTTP.CORS.AllowedOrigins), + cors.AllowedMethods(config.HTTP.CORS.AllowedMethods), + cors.AllowedHeaders(config.HTTP.CORS.AllowedHeaders), + cors.AllowCredentials(config.HTTP.CORS.AllowCredentials), + ), + groupwaremiddleware.GroupwareLogger(logger), + auth, + ) + m.Route(config.HTTP.Root, gw.Route) + + baseURL := "" + { + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", httpPort)) + require.NoError(err) + ts := &httptest.Server{ + Listener: l, + Config: &http.Server{ + Handler: gw, + }, + } + ts.Start() + t.Cleanup(ts.Close) + + baseURL, err = url.JoinPath(ts.URL, config.HTTP.Root) + require.NoError(err) + } + + return GroupwareTest{ + BaseURL: baseURL, + Users: s.Users, + }, nil +} diff --git a/services/groupware/pkg/middleware/auth.go b/services/groupware/pkg/middleware/auth.go index 2f85e62074..4807429edb 100644 --- a/services/groupware/pkg/middleware/auth.go +++ b/services/groupware/pkg/middleware/auth.go @@ -25,6 +25,8 @@ func authOptions(opts ...account.Option) account.Options { return opt } +const RequestIdHeader = "X-Request-ID" + func Auth(opts ...account.Option) func(http.Handler) http.Handler { opt := authOptions(opts...) tokenManager, err := jwt.New(map[string]any{ @@ -39,7 +41,7 @@ func Auth(opts ...account.Option) func(http.Handler) http.Handler { ctx := r.Context() t := r.Header.Get(revactx.TokenHeader) if t == "" { - requestID := r.Header.Get("X-Request-ID") + requestID := r.Header.Get(RequestIdHeader) traceID := GetTraceID(ctx) l := opt.Logger.Error().Str(log.RequestIDString, log.SafeString(requestID)) if traceID != "" { @@ -52,7 +54,7 @@ func Auth(opts ...account.Option) func(http.Handler) http.Handler { u, tokenScope, err := tokenManager.DismantleToken(r.Context(), t) if err != nil { - requestID := r.Header.Get("X-Request-ID") + requestID := r.Header.Get(RequestIdHeader) traceID := GetTraceID(ctx) l := opt.Logger.Error().Str(log.RequestIDString, log.SafeString(requestID)) if traceID != "" { @@ -63,7 +65,7 @@ func Auth(opts ...account.Option) func(http.Handler) http.Handler { return } if ok, err := scope.VerifyScope(ctx, tokenScope, r); err != nil || !ok { - requestID := r.Header.Get("X-Request-ID") + requestID := r.Header.Get(RequestIdHeader) traceID := GetTraceID(ctx) l := opt.Logger.Error().Str(log.RequestIDString, log.SafeString(requestID)) if traceID != "" {