From 77dd2d6979e96b4eb76937b2681d879f28fa9fb6 Mon Sep 17 00:00:00 2001 From: Pascal Bleser Date: Mon, 8 Dec 2025 17:58:47 +0100 Subject: [PATCH] groupware: improve email submission and testing * jmap/EmailCreate: add more attributes that were omitted: Headers, InReplyTo, References, Sender * add jmap GetEmailSubmissionStatus * improve email integration tests by adding a thorough test for email submission * jmap integration tests: provision principals and domains using the Stalwart Management API, switching from an in-memory to an internal directory --- pkg/jmap/jmap_api.go | 2 - pkg/jmap/jmap_api_email.go | 41 ++- pkg/jmap/jmap_integration_contact_test.go | 11 +- pkg/jmap/jmap_integration_email_test.go | 229 +++++++++++---- pkg/jmap/jmap_integration_event_test.go | 13 +- pkg/jmap/jmap_integration_test.go | 337 ++++++++++++++-------- pkg/jmap/jmap_integration_ws_test.go | 37 +-- pkg/jmap/jmap_model.go | 21 +- 8 files changed, 482 insertions(+), 209 deletions(-) diff --git a/pkg/jmap/jmap_api.go b/pkg/jmap/jmap_api.go index cede3f1904..c5578b28fc 100644 --- a/pkg/jmap/jmap_api.go +++ b/pkg/jmap/jmap_api.go @@ -40,12 +40,10 @@ type BlobClient interface { const ( logOperation = "operation" - logMailboxId = "mailbox-id" logFetchBodies = "fetch-bodies" logOffset = "offset" logLimit = "limit" logDownloadUrl = "download-url" logBlobId = "blob-id" - logUploadUrl = "download-url" logSinceState = "since-state" ) diff --git a/pkg/jmap/jmap_api_email.go b/pkg/jmap/jmap_api_email.go index e548b83444..c728dead77 100644 --- a/pkg/jmap/jmap_api_email.go +++ b/pkg/jmap/jmap_api_email.go @@ -845,7 +845,7 @@ func (j *Client) SubmitEmail(accountId string, identityId string, emailId string id: { IdentityId: identityId, EmailId: emailId, - // leaving Envelope empty + Envelope: nil, }, }, OnSuccessUpdateEmail: map[string]PatchObject{ @@ -856,13 +856,6 @@ func (j *Client) SubmitEmail(accountId string, identityId string, emailId string get := EmailSubmissionGetCommand{ AccountId: accountId, Ids: []string{"#" + id}, - /* - IdRef: &ResultReference{ - ResultOf: "0", - Name: CommandEmailSubmissionSet, - Path: ["#"]"/created/" + "#" + id + "/" + EmailPropertyId, - }, - */ } cmd, err := j.request(session, logger, @@ -918,6 +911,38 @@ func (j *Client) SubmitEmail(accountId string, identityId string, emailId string }) } +type emailSubmissionResult struct { + submissions map[string]EmailSubmission + notFound []string +} + +func (j *Client) GetEmailSubmissionStatus(accountId string, submissionIds []string, session *Session, ctx context.Context, logger *log.Logger, acceptLanguage string) (map[string]EmailSubmission, []string, SessionState, State, Language, Error) { + logger = j.logger("GetEmailSubmissionStatus", session, logger) + + cmd, err := j.request(session, logger, invocation(CommandEmailSubmissionGet, EmailSubmissionGetCommand{ + AccountId: accountId, + Ids: submissionIds, + }, "0")) + if err != nil { + return nil, nil, "", "", "", err + } + + result, sessionState, state, lang, err := command(j.api, logger, ctx, session, j.onSessionOutdated, cmd, acceptLanguage, func(body *Response) (emailSubmissionResult, State, Error) { + var response EmailSubmissionGetResponse + err = retrieveResponseMatchParameters(logger, body, CommandEmailSubmissionGet, "0", &response) + if err != nil { + return emailSubmissionResult{}, "", err + } + m := make(map[string]EmailSubmission, len(response.List)) + for _, s := range response.List { + m[s.Id] = s + } + return emailSubmissionResult{submissions: m, notFound: response.NotFound}, response.State, nil + }) + + return result.submissions, result.notFound, sessionState, state, lang, err +} + func (j *Client) EmailsInThread(accountId string, threadId string, session *Session, ctx context.Context, logger *log.Logger, acceptLanguage string, fetchBodies bool, maxBodyValueBytes uint) ([]Email, SessionState, State, Language, Error) { logger = j.loggerParams("EmailsInThread", session, logger, func(z zerolog.Context) zerolog.Context { return z.Bool(logFetchBodies, fetchBodies).Str("threadId", log.SafeString(threadId)) diff --git a/pkg/jmap/jmap_integration_contact_test.go b/pkg/jmap/jmap_integration_contact_test.go index 20a8c771bc..decdbf3e8f 100644 --- a/pkg/jmap/jmap_integration_contact_test.go +++ b/pkg/jmap/jmap_integration_contact_test.go @@ -39,7 +39,10 @@ func TestContacts(t *testing.T) { require.NoError(err) defer s.Close() - accountId, addressbookId, expectedContactCardsById, boxes, err := s.fillContacts(t, count) + user := pickUser() + session := s.Session(user.name) + + accountId, addressbookId, expectedContactCardsById, boxes, err := s.fillContacts(t, count, session, user) require.NoError(err) require.NotEmpty(accountId) require.NotEmpty(addressbookId) @@ -51,7 +54,7 @@ func TestContacts(t *testing.T) { {Property: jscontact.ContactCardPropertyCreated, IsAscending: true}, } - contactsByAccount, _, _, _, err := s.client.QueryContactCards([]string{accountId}, s.session, t.Context(), s.logger, "", filter, sortBy, 0, 0) + contactsByAccount, _, _, _, err := s.client.QueryContactCards([]string{accountId}, session, t.Context(), s.logger, "", filter, sortBy, 0, 0) require.NoError(err) require.Len(contactsByAccount, 1) @@ -97,9 +100,11 @@ var streetNumberRegex = regexp.MustCompile(`^(\d+)\s+(.+)$`) func (s *StalwartTest) fillContacts( t *testing.T, count uint, + session *Session, + user User, ) (string, string, map[string]jscontact.ContactCard, ContactsBoxes, error) { require := require.New(t) - c, err := NewTestJmapClient(s.session, s.username, s.password, true, true) + c, err := NewTestJmapClient(session, user.name, user.password, true, true) require.NoError(err) defer c.Close() diff --git a/pkg/jmap/jmap_integration_email_test.go b/pkg/jmap/jmap_integration_email_test.go index 2a53261bfb..433eb5cf3a 100644 --- a/pkg/jmap/jmap_integration_email_test.go +++ b/pkg/jmap/jmap_integration_email_test.go @@ -25,34 +25,6 @@ import ( "github.com/stretchr/testify/require" ) -func (s *StalwartTest) findInbox(t *testing.T, accountId string) (string, string) { - require := require.New(t) - respByAccountId, sessionState, _, _, err := s.client.GetAllMailboxes([]string{accountId}, s.session, s.ctx, s.logger, "") - require.NoError(err) - require.Equal(s.session.State, sessionState) - require.Len(respByAccountId, 1) - require.Contains(respByAccountId, accountId) - resp := respByAccountId[accountId] - - mailboxesNameByRole := map[string]string{} - mailboxesUnreadByRole := map[string]int{} - for _, m := range resp { - if m.Role != "" { - mailboxesNameByRole[m.Role] = m.Name - mailboxesUnreadByRole[m.Role] = m.UnreadEmails - } - } - require.Contains(mailboxesNameByRole, "inbox") - require.Contains(mailboxesUnreadByRole, "inbox") - require.Zero(mailboxesUnreadByRole["inbox"]) - - inboxId := mailboxId("inbox", resp) - require.NotEmpty(inboxId) - inboxFolder := mailboxesNameByRole["inbox"] - require.NotEmpty(inboxFolder) - return inboxId, inboxFolder -} - func TestEmails(t *testing.T) { if skip(t) { return @@ -66,32 +38,35 @@ func TestEmails(t *testing.T) { require.NoError(err) defer s.Close() - accountId := s.session.PrimaryAccounts.Mail + user := pickUser() + session := s.Session(user.name) - inboxId, inboxFolder := s.findInbox(t, accountId) + accountId := session.PrimaryAccounts.Mail + + inboxId, inboxFolder := s.findInbox(t, accountId, session) var threads int = 0 var mails []filledMail = nil { - mails, threads, err = s.fillEmailsWithImap(inboxFolder, count, false) + mails, threads, err = s.fillEmailsWithImap(inboxFolder, count, false, user) require.NoError(err) } mailsByMessageId := structs.Index(mails, func(mail filledMail) string { return mail.messageId }) { { - resp, sessionState, _, _, err := s.client.GetAllIdentities(accountId, s.session, s.ctx, s.logger, "") + resp, sessionState, _, _, err := s.client.GetAllIdentities(accountId, session, s.ctx, s.logger, "") require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.Len(resp, 1) - require.Equal(s.userEmail, resp[0].Email) - require.Equal(s.userPersonName, resp[0].Name) + require.Equal(user.email, resp[0].Email) + require.Equal(user.description, resp[0].Name) } { - respByAccountId, sessionState, _, _, err := s.client.GetAllMailboxes([]string{accountId}, s.session, s.ctx, s.logger, "") + respByAccountId, sessionState, _, _, err := s.client.GetAllMailboxes([]string{accountId}, session, s.ctx, s.logger, "") require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.Len(respByAccountId, 1) require.Contains(respByAccountId, accountId) resp := respByAccountId[accountId] @@ -105,9 +80,9 @@ func TestEmails(t *testing.T) { } { - resp, sessionState, _, _, err := s.client.GetAllEmailsInMailbox(accountId, s.session, s.ctx, s.logger, "", inboxId, 0, 0, true, false, 0, true) + resp, sessionState, _, _, err := s.client.GetAllEmailsInMailbox(accountId, session, s.ctx, s.logger, "", inboxId, 0, 0, true, false, 0, true) require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.Equalf(threads, len(resp.Emails), "the number of collapsed emails in the inbox is expected to be %v, but is actually %v", threads, len(resp.Emails)) for _, e := range resp.Emails { @@ -119,9 +94,9 @@ func TestEmails(t *testing.T) { } { - resp, sessionState, _, _, err := s.client.GetAllEmailsInMailbox(accountId, s.session, s.ctx, s.logger, "", inboxId, 0, 0, false, false, 0, true) + resp, sessionState, _, _, err := s.client.GetAllEmailsInMailbox(accountId, session, s.ctx, s.logger, "", inboxId, 0, 0, false, false, 0, true) require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.Equalf(count, len(resp.Emails), "the number of emails in the inbox is expected to be %v, but is actually %v", count, len(resp.Emails)) for _, e := range resp.Emails { @@ -145,11 +120,29 @@ func TestSendingEmails(t *testing.T) { require.NoError(err) defer s.Close() - accountId := s.session.PrimaryAccounts.Mail + from := pickUser() + session := s.Session(from.name) + accountId := session.PrimaryAccounts.Mail + + var to User + { + others := structs.Filter(users[:], func(u User) bool { return u.name != from.name }) + to = others[rand.Intn(len(others))] + } + toSession := s.Session(to.name) + toAccountId := toSession.PrimaryAccounts.Mail + + var cc User + { + others := structs.Filter(users[:], func(u User) bool { return u.name != from.name && u.name != to.name }) + cc = others[rand.Intn(len(others))] + } + ccSession := s.Session(cc.name) + ccAccountId := ccSession.PrimaryAccounts.Mail var mailboxPerRole map[string]Mailbox { - mailboxes, _, _, _, err := s.client.GetAllMailboxes([]string{accountId}, s.session, s.ctx, s.logger, "") + mailboxes, _, _, _, err := s.client.GetAllMailboxes([]string{accountId}, session, s.ctx, s.logger, "") require.NoError(err) mailboxPerRole = structs.Index(mailboxes[accountId], func(m Mailbox) string { return m.Role }) require.Contains(mailboxPerRole, JmapMailboxRoleInbox) @@ -159,7 +152,7 @@ func TestSendingEmails(t *testing.T) { } { roles := []string{JmapMailboxRoleDrafts, JmapMailboxRoleSent, JmapMailboxRoleInbox} - m, _, _, _, err := s.client.SearchMailboxIdsPerRole([]string{accountId}, s.session, s.ctx, s.logger, "", roles) + m, _, _, _, err := s.client.SearchMailboxIdsPerRole([]string{accountId}, session, s.ctx, s.logger, "", roles) require.NoError(err) require.Contains(m, accountId) a := m[accountId] @@ -168,10 +161,26 @@ func TestSendingEmails(t *testing.T) { } } + // let's ensure that the recipients have zero emails in their mailboxes before we send them any + for _, u := range []struct { + accountId string + session *Session + }{{toAccountId, toSession}, {ccAccountId, ccSession}} { + mailboxes, _, _, _, err := s.client.GetAllMailboxes([]string{u.accountId}, u.session, s.ctx, s.logger, "") + require.NoError(err) + for _, mailbox := range mailboxes[u.accountId] { + require.Equal(0, mailbox.TotalEmails) + } + } + + subject := fmt.Sprintf("Test Subject %d", 10000+rand.Intn(90000)) + fromName := fmt.Sprintf("%s (test %d)", from.name, 1000+rand.Intn(9000)) + sender := EmailAddress{Email: from.email, Name: from.description} + { var identity Identity { - identities, _, _, _, err := s.client.GetAllIdentities(accountId, s.session, s.ctx, s.logger, "") + identities, _, _, _, err := s.client.GetAllIdentities(accountId, session, s.ctx, s.logger, "") require.NoError(err) require.NotEmpty(identities) identity = identities[0] @@ -179,15 +188,15 @@ func TestSendingEmails(t *testing.T) { create := EmailCreate{ Keywords: toBoolMapS("test"), - Subject: "testing 123", + Subject: subject, MailboxIds: toBoolMapS(mailboxPerRole[JmapMailboxRoleDrafts].Id), } - created, _, _, _, err := s.client.CreateEmail(accountId, create, "", s.session, s.ctx, s.logger, "") + created, _, _, _, err := s.client.CreateEmail(accountId, create, "", session, s.ctx, s.logger, "") require.NoError(err) require.NotEmpty(created.Id) { - emails, notFound, _, _, _, err := s.client.GetEmails(accountId, s.session, s.ctx, s.logger, "", []string{created.Id}, true, 0, false, false) + emails, notFound, _, _, _, err := s.client.GetEmails(accountId, session, s.ctx, s.logger, "", []string{created.Id}, true, 0, false, false) require.NoError(err) require.Len(emails, 1) require.Empty(notFound) @@ -198,19 +207,22 @@ func TestSendingEmails(t *testing.T) { } update := EmailCreate{ - To: []EmailAddress{{Name: identity.Name, Email: identity.Email}}, + From: []EmailAddress{{Name: fromName, Email: from.email}}, + To: []EmailAddress{{Name: to.description, Email: to.email}}, + Cc: []EmailAddress{{Name: cc.description, Email: cc.email}}, + Sender: []EmailAddress{sender}, Keywords: toBoolMapS("test"), - Subject: "testing 1234", + Subject: subject, MailboxIds: toBoolMapS(mailboxPerRole[JmapMailboxRoleDrafts].Id), } - updated, _, _, _, err := s.client.CreateEmail(accountId, update, created.Id, s.session, s.ctx, s.logger, "") + updated, _, _, _, err := s.client.CreateEmail(accountId, update, created.Id, session, s.ctx, s.logger, "") require.NoError(err) require.NotEmpty(updated.Id) require.NotEqual(created.Id, updated.Id) var updatedMailboxId string { - emails, notFound, _, _, _, err := s.client.GetEmails(accountId, s.session, s.ctx, s.logger, "", []string{created.Id, updated.Id}, true, 0, false, false) + emails, notFound, _, _, _, err := s.client.GetEmails(accountId, session, s.ctx, s.logger, "", []string{created.Id, updated.Id}, true, 0, false, false) require.NoError(err) require.Len(emails, 1) require.Len(notFound, 1) @@ -229,8 +241,81 @@ func TestSendingEmails(t *testing.T) { ToMailboxId: mailboxPerRole[JmapMailboxRoleSent].Id, } - sub, _, _, _, err := s.client.SubmitEmail(accountId, identity.Id, updated.Id, &move, s.session, s.ctx, s.logger, "") - fmt.Printf("sub: %v\n", sub) + sub, _, _, _, err := s.client.SubmitEmail(accountId, identity.Id, updated.Id, &move, session, s.ctx, s.logger, "") + require.NoError(err) + require.NotEmpty(sub.Id) + require.NotEmpty(sub.ThreadId) + require.Equal(updated.Id, sub.EmailId) + require.Equal(identity.Id, sub.IdentityId) + require.Equal(sub.UndoStatus, UndoStatusPending) // this *might* be fragile: if the server is fast enough, would we get "final" here? + require.Empty(sub.DsnBlobIds) + require.Empty(sub.MdnBlobIds) + require.Equal(from.email, sub.Envelope.MailFrom.Email) + require.Nil(sub.Envelope.MailFrom.Parameters) + require.Len(sub.Envelope.RcptTo, 2) + require.Contains(sub.Envelope.RcptTo, Address{Email: to.email}) + require.Contains(sub.Envelope.RcptTo, Address{Email: cc.email}) + require.NotZero(sub.SendAt) + require.Len(sub.DeliveryStatus, 2) + require.Contains(sub.DeliveryStatus, to.email) + require.Contains(sub.DeliveryStatus, cc.email) + + a := 0 + maxAttempts := 3 + delivery := sub.DeliveryStatus[to.email].Delivered + + for delivery != DeliveredYes { + require.NotEqual(DeliveredNo, delivery) + a++ + if a >= maxAttempts { + break + } + time.Sleep(1 * time.Second) + + subs, notFound, _, _, _, err := s.client.GetEmailSubmissionStatus(accountId, []string{sub.Id}, session, s.ctx, s.logger, "") + require.NoError(err) + require.Empty(notFound) + require.Contains(subs, sub.Id) + delivery = subs[sub.Id].DeliveryStatus[to.email].Delivered + } + + require.Contains([]DeliveryStatusDelivered{DeliveredYes, DeliveredUnknown}, delivery) + + for _, r := range []struct { + user User + accountId string + session *Session + }{{to, toAccountId, toSession}, {cc, ccAccountId, ccSession}} { + mailboxes, _, _, _, err := s.client.GetAllMailboxes([]string{r.accountId}, r.session, s.ctx, s.logger, "") + require.NoError(err) + inboxId := "" + for _, mailbox := range mailboxes[r.accountId] { + if mailbox.Role == JmapMailboxRoleInbox { + inboxId = mailbox.Id + require.Equal(1, mailbox.TotalEmails) + } + } + require.NotEmpty(inboxId, "failed to find the Mailbox with the 'inbox' role for %v", r.user.name) + + emails, _, _, _, err := s.client.QueryEmails([]string{r.accountId}, EmailFilterCondition{InMailbox: inboxId}, r.session, s.ctx, s.logger, "", 0, 0, true, 0) + require.NoError(err) + require.Contains(emails, r.accountId) + require.Len(emails[r.accountId].Emails, 1) + received := emails[r.accountId].Emails[0] + require.Len(received.From, 1) + require.Equal(from.email, received.From[0].Email) + require.Equal(fromName, received.From[0].Name) + require.Len(received.Sender, 1) + require.Equal(from.email, received.Sender[0].Email) + require.Equal(from.description, received.Sender[0].Name) + require.Len(received.To, 1) + require.Equal(to.email, received.To[0].Email) + require.Equal(to.description, received.To[0].Name) + require.Len(received.Cc, 1) + require.Equal(cc.email, received.Cc[0].Email) + require.Equal(cc.description, received.Cc[0].Name) + require.Equal(subject, received.Subject) + } } } @@ -269,6 +354,34 @@ func matchEmail(t *testing.T, actual Email, expected filledMail, hasBodies bool) } } +func (s *StalwartTest) findInbox(t *testing.T, accountId string, session *Session) (string, string) { + require := require.New(t) + respByAccountId, sessionState, _, _, err := s.client.GetAllMailboxes([]string{accountId}, session, s.ctx, s.logger, "") + require.NoError(err) + require.Equal(session.State, sessionState) + require.Len(respByAccountId, 1) + require.Contains(respByAccountId, accountId) + resp := respByAccountId[accountId] + + mailboxesNameByRole := map[string]string{} + mailboxesUnreadByRole := map[string]int{} + for _, m := range resp { + if m.Role != "" { + mailboxesNameByRole[m.Role] = m.Name + mailboxesUnreadByRole[m.Role] = m.UnreadEmails + } + } + require.Contains(mailboxesNameByRole, "inbox") + require.Contains(mailboxesUnreadByRole, "inbox") + require.Zero(mailboxesUnreadByRole["inbox"]) + + inboxId := mailboxId("inbox", resp) + require.NotEmpty(inboxId) + inboxFolder := mailboxesNameByRole["inbox"] + require.NotEmpty(inboxFolder) + return inboxId, inboxFolder +} + var emailSplitter = regexp.MustCompile("(.+)@(.+)$") func htmlFormat(body string, msg enmime.MailBuilder) enmime.MailBuilder { @@ -371,8 +484,8 @@ var allKeywords = map[string]imap.Flag{ JmapKeywordSeen: imap.FlagSeen, } -func (s *StalwartTest) fillEmailsWithImap(folder string, count int, empty bool) ([]filledMail, int, error) { - to := fmt.Sprintf("%s <%s>", s.userPersonName, s.userEmail) +func (s *StalwartTest) fillEmailsWithImap(folder string, count int, empty bool, user User) ([]filledMail, int, error) { + to := fmt.Sprintf("%s <%s>", user.description, user.email) ccEvery := 2 bccEvery := 3 attachmentEvery := 2 @@ -394,7 +507,7 @@ func (s *StalwartTest) fillEmailsWithImap(folder string, count int, empty bool) } }(c) - if err = c.Login(s.username, s.password).Wait(); err != nil { + if err = c.Login(user.name, user.password).Wait(); err != nil { return nil, 0, err } @@ -442,7 +555,7 @@ func (s *StalwartTest) fillEmailsWithImap(folder string, count int, empty bool) domain := addressParts[0][2] toName := displayName - toAddress := fmt.Sprintf("%s@%s", s.username, domain) + toAddress := fmt.Sprintf("%s@%s", user.name, domain) ccName1 := "Team Lead" ccAddress1 := fmt.Sprintf("lead@%s", domain) ccName2 := "Coworker" diff --git a/pkg/jmap/jmap_integration_event_test.go b/pkg/jmap/jmap_integration_event_test.go index 72c64fbd44..af014efa16 100644 --- a/pkg/jmap/jmap_integration_event_test.go +++ b/pkg/jmap/jmap_integration_event_test.go @@ -38,7 +38,10 @@ func TestEvents(t *testing.T) { require.NoError(err) defer s.Close() - accountId, calendarId, expectedEventsById, boxes, err := s.fillEvents(t, count) + user := pickUser() + session := s.Session(user.name) + + accountId, calendarId, expectedEventsById, boxes, err := s.fillEvents(t, count, session, user) require.NoError(err) require.NotEmpty(accountId) require.NotEmpty(calendarId) @@ -50,7 +53,7 @@ func TestEvents(t *testing.T) { {Property: CalendarEventPropertyCreated, IsAscending: true}, } - contactsByAccount, _, _, _, err := s.client.QueryCalendarEvents([]string{accountId}, s.session, t.Context(), s.logger, "", filter, sortBy, 0, 0) + contactsByAccount, _, _, _, err := s.client.QueryCalendarEvents([]string{accountId}, session, t.Context(), s.logger, "", filter, sortBy, 0, 0) require.NoError(err) require.Len(contactsByAccount, 1) @@ -85,9 +88,11 @@ type EventsBoxes struct { func (s *StalwartTest) fillEvents( t *testing.T, count uint, + session *Session, + user User, ) (string, string, map[string]CalendarEvent, EventsBoxes, error) { require := require.New(t) - c, err := NewTestJmapClient(s.session, s.username, s.password, true, true) + c, err := NewTestJmapClient(session, user.name, user.password, true, true) require.NoError(err) defer c.Close() @@ -245,7 +250,7 @@ func (s *StalwartTest) fillEvents( virtualLocationId: virtualLocationObj, }, Alerts: map[string]jscalendar.Alert{ - alertId: jscalendar.Alert{ + alertId: { Type: jscalendar.AlertType, Trigger: jscalendar.OffsetTrigger{ Type: jscalendar.OffsetTriggerType, diff --git a/pkg/jmap/jmap_integration_test.go b/pkg/jmap/jmap_integration_test.go index 80edf70cae..15691dc453 100644 --- a/pkg/jmap/jmap_integration_test.go +++ b/pkg/jmap/jmap_integration_test.go @@ -27,8 +27,6 @@ import ( "github.com/gorilla/websocket" "github.com/tidwall/pretty" - "golang.org/x/text/cases" - "golang.org/x/text/language" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -50,18 +48,33 @@ const ( Wireshark = "" ) +type User struct { + name string + description string + email string + password string +} + +func userpassword() string { + password, err := pw.Generate(4+rand.Intn(28), 2, 0, false, true) + if err != nil { + panic(err) + } + return password +} + var ( - domains = [...]string{"earth.gov", "mars.mil", "opa.org", "acme.com"} - people = [...]string{ - "Camina Drummer", - "Amos Burton", - "James Holden", - "Anderson Dawes", - "Naomi Nagata", - "Klaes Ashford", - "Fred Johnson", - "Chrisjen Avasarala", - "Bobby Draper", + domains = [...]string{"earth.gov", "mars.mil", "opa.org"} + users = [...]User{ + {"cdrummer", "Camina Drummer", "camina.drummer@opa.org", userpassword()}, + {"aburton", "Amos Burton", "amos.burton@earth.gov", userpassword()}, + {"jholden", "James Holden", "james.holden@earth.gov", userpassword()}, + {"adawes", "Anderson Dawes", "anderson.dawes@opa.org", userpassword()}, + {"nnagata", "Naomi Nagata", "naomi.nagata@opa.org", userpassword()}, + {"kashford", "Klaes Ashford", "klaes.ashford@opa.org", userpassword()}, + {"fjohnson", "Fred Johnson", "fred.johnson@opa.org", userpassword()}, + {"cavasarala", "Chrisjen Avasarala}", "chrissy@earth.gov", userpassword()}, + {"bdraper", "Roberta Draper", "bobby@mars.mil", userpassword()}, } ) @@ -70,22 +83,16 @@ const ( httpPort = "8080" imapsPort = "993" configTemplate = ` -authentication.fallback-admin.secret = "$6$4qPYDVhaUHkKcY7s$bB6qhcukb9oFNYRIvaDZgbwxrMa2RvF5dumCjkBFdX19lSNqrgKltf3aPrFMuQQKkZpK2YNuQ83hB1B3NiWzj." +authentication.fallback-admin.secret = "secret" authentication.fallback-admin.user = "mailadmin" authentication.master.secret = "{{.masterpassword}}" authentication.master.user = "{{.masterusername}}" -directory.memory.principals.0000.class = "admin" -directory.memory.principals.0000.description = "Superuser" -directory.memory.principals.0000.email.0000 = "admin@example.org" -directory.memory.principals.0000.name = "admin" -directory.memory.principals.0000.secret = "secret" -directory.memory.principals.0001.class = "individual" -directory.memory.principals.0001.description = "{{.description}}" -directory.memory.principals.0001.email.0000 = "{{.email}}" -directory.memory.principals.0001.name = "{{.username}}" -directory.memory.principals.0001.secret = "{{.password}}" -directory.memory.principals.0001.storage.directory = "memory" -directory.memory.type = "memory" +directory.test.bind.auth.method = "default" +directory.test.cache.size = 1048576 +directory.test.cache.ttl.negative = "10m" +directory.test.cache.ttl.positive = "1h" +directory.test.store = "rocksdb" +directory.test.type = "internal" metrics.prometheus.enable = false server.listener.http.bind = "[::]:{{.httpPort}}" server.listener.http.protocol = "http" @@ -100,7 +107,7 @@ server.socket.reuse-addr = true server.socket.reuse-port = true storage.blob = "rocksdb" storage.data = "rocksdb" -storage.directory = "memory" +storage.directory = "test" storage.fts = "rocksdb" storage.lookup = "rocksdb" store.rocksdb.compression = "lz4" @@ -114,6 +121,14 @@ tracer.log.lossy = false tracer.log.multiline = false tracer.log.type = "stdout" sharing.allow-directory-query = false +auth.dkim.sign = false +auth.dkim.verify = "disable" +auth.spf.verify.ehlo = "disable" +auth.spf.verify.mail-from = "disable" +auth.arc.verify = "disable" +auth.arc.seal = false +auth.dmarc.verify = "disable" +auth.iprev.verify = "disable" ` ) @@ -134,19 +149,16 @@ func skip(t *testing.T) bool { } type StalwartTest struct { - t *testing.T - ip string - imapPort int - container *testcontainers.DockerContainer - ctx context.Context - cancelCtx context.CancelFunc - client *Client - session *Session - username string - password string - logger *clog.Logger - userPersonName string - userEmail string + t *testing.T + ip string + imapPort int + container *testcontainers.DockerContainer + ctx context.Context + cancelCtx context.CancelFunc + client *Client + logger *clog.Logger + jmapBaseUrl *url.URL + sessionUrl *url.URL io.Closer } @@ -162,6 +174,38 @@ func (s *StalwartTest) Close() error { return nil } +func (s *StalwartTest) Session(username string) *Session { + session, jerr := s.client.FetchSession(s.sessionUrl, username, s.logger) + require.NoError(s.t, jerr) + require.NotNil(s.t, session.Capabilities.Mail) + require.NotNil(s.t, session.Capabilities.Calendars) + require.NotNil(s.t, session.Capabilities.Contacts) + + // we have to overwrite the hostname in JMAP URL because the container + // will know its name to be a random Docker container identifier, or + // "localhost" as we defined the hostname in the Stalwart configuration, + // and we also need to overwrite the port number as its not mapped + session.JmapUrl.Host = s.jmapBaseUrl.Host + session.WebsocketUrl.Host = s.jmapBaseUrl.Host + var err error + session.ApiUrl, err = replaceHost(session.ApiUrl, s.jmapBaseUrl.Host) + require.NoError(s.t, err) + session.DownloadUrl, err = replaceHost(session.DownloadUrl, s.jmapBaseUrl.Host) + require.NoError(s.t, err) + session.UploadUrl, err = replaceHost(session.UploadUrl, s.jmapBaseUrl.Host) + require.NoError(s.t, err) + session.EventSourceUrl, err = replaceHost(session.EventSourceUrl, s.jmapBaseUrl.Host) + require.NoError(s.t, err) + + return &session +} + +type stalwartTestLogConsumer struct{} + +func (lc *stalwartTestLogConsumer) Accept(l testcontainers.Log) { + fmt.Print("STALWART: " + string(l.Content)) +} + func newStalwartTest(t *testing.T) (*StalwartTest, error) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) var _ context.CancelFunc = cancel // ignore context leak warning: it is passed in the struct and called in Close() @@ -189,33 +233,11 @@ func newStalwartTest(t *testing.T) (*StalwartTest, error) { masterPasswordHash = digest.Encode() } - usernameSuffix, err := pw.Generate(8, 2, 0, true, true) - if err != nil { - return nil, err - } - username := "user_" + usernameSuffix - - password, err := pw.Generate(4+rand.Intn(28), 2, 0, false, true) - if err != nil { - return nil, err - } - hostname := "localhost" - userPersonName := people[rand.Intn(len(people))] - var userEmail string - { - domain := domains[rand.Intn(len(domains))] - userEmail = strings.Join(strings.Split(cases.Lower(language.English).String(userPersonName), " "), ".") + "@" + domain - } - configBuf := bytes.NewBufferString("") template.Must(template.New("").Parse(configTemplate)).Execute(configBuf, map[string]any{ "hostname": hostname, - "password": password, - "username": username, - "description": userPersonName, - "email": userEmail, "masterusername": masterUsername, "masterpassword": masterPasswordHash, "httpPort": httpPort, @@ -227,6 +249,7 @@ func newStalwartTest(t *testing.T) (*StalwartTest, error) { container, err := testcontainers.Run( ctx, stalwartImage, + testcontainers.WithLogConsumers(&stalwartTestLogConsumer{}), testcontainers.WithExposedPorts(httpPort+"/tcp", imapsPort+"/tcp"), testcontainers.WithFiles(testcontainers.ContainerFile{ Reader: configReader, @@ -262,7 +285,8 @@ func newStalwartTest(t *testing.T) (*StalwartTest, error) { loggerImpl := clog.NewLogger(clog.Level("trace")) logger := &loggerImpl var j Client - var session *Session + var jmapBaseUrl *url.URL + var sessionUrl *url.URL { tr := http.DefaultTransport.(*http.Transport).Clone() tr.ResponseHeaderTimeout = time.Duration(30 * time.Second) @@ -279,11 +303,12 @@ func newStalwartTest(t *testing.T) (*StalwartTest, error) { if err != nil { return nil, err } - jmapBaseUrl := url.URL{ + jmapBaseUrl = &url.URL{ Scheme: "http", Host: ip + ":" + jmapPort.Port(), Path: "/", } + sessionUrl = jmapBaseUrl.JoinPath(".well-known", "jmap") if Wireshark != "" { fmt.Printf("\x1b[45;37;1m Starting Wireshark on port %v \x1b[0m\n", jmapPort) @@ -301,8 +326,6 @@ func newStalwartTest(t *testing.T) (*StalwartTest, error) { time.Sleep(10 * time.Second) } - sessionUrl := jmapBaseUrl.JoinPath(".well-known", "jmap") - eventListener := nullHttpJmapApiClientEventListener{} api := NewHttpJmapClient( @@ -318,47 +341,139 @@ func newStalwartTest(t *testing.T) (*StalwartTest, error) { } j = NewClient(api, api, api, wscf) - s, err := j.FetchSession(sessionUrl, username, logger) - if err != nil { - return nil, err - } - // we have to overwrite the hostname in JMAP URL because the container - // will know its name to be a random Docker container identifier, or - // "localhost" as we defined the hostname in the Stalwart configuration, - // and we also need to overwrite the port number as its not mapped - s.JmapUrl.Host = jmapBaseUrl.Host - s.WebsocketUrl.Host = jmapBaseUrl.Host - //s.JmapEndpoint = jmapBaseUrl.Host - s.ApiUrl, err = replaceHost(s.ApiUrl, jmapBaseUrl.Host) - require.NoError(t, err) - s.DownloadUrl, err = replaceHost(s.DownloadUrl, jmapBaseUrl.Host) - require.NoError(t, err) - s.UploadUrl, err = replaceHost(s.UploadUrl, jmapBaseUrl.Host) - require.NoError(t, err) - s.EventSourceUrl, err = replaceHost(s.EventSourceUrl, jmapBaseUrl.Host) - require.NoError(t, err) - session = &s } - require.NotNil(t, session.Capabilities.Mail) - require.NotNil(t, session.Capabilities.Calendars) - require.NotNil(t, session.Capabilities.Contacts) + // provision some things using Stalwart's Management API + { + var h http.Client + { + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.ResponseHeaderTimeout = time.Duration(30 * time.Second) + tr.TLSClientConfig = tlsConfig + h = *http.DefaultClient + h.Transport = tr + } + + apiPort, err := container.MappedPort(ctx, httpPort) + require.NoError(t, err) + + url := fmt.Sprintf("http://%s:%d/api/principal", ip, apiPort.Int()) + + for _, domain := range domains { + fmt.Printf("Creating domain '%v'\n", domain) + bb, err := json.Marshal(map[string]any{ + "type": "domain", + "name": domain, + "description": domain, + }) + require.NoError(t, err) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(bb)) + require.NoError(t, err) + req.SetBasicAuth("mailadmin", "secret") + resp, err := h.Do(req) + require.NoError(t, err) + require.Equal(t, "200 OK", resp.Status) + } + + for _, user := range users { + fmt.Printf("Creating individual '%v'\n", user.name) + bb, err := json.Marshal(map[string]any{ + "type": "individual", + "name": user.name, + "description": user.description, + "emails": user.email, + "roles": []string{"user"}, + "secrets": user.password, + "quota": 20000000000, + }) + require.NoError(t, err) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(bb)) + require.NoError(t, err) + req.SetBasicAuth("mailadmin", "secret") + resp, err := h.Do(req) + require.NoError(t, err) + require.Equal(t, "200 OK", resp.Status) + + // fetch the user once with the superadmin credentials to "activate" it, + // it is unclear why that is needed, but without that, we get errors back + // that we are not allowed to access that resource + { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + req.SetBasicAuth("mailadmin", "secret") + resp, err := h.Do(req) + require.NoError(t, err) + require.Equal(t, "200 OK", resp.Status) + } + } + + { + require.NoError(t, err) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + req.SetBasicAuth("mailadmin", "secret") + resp, err := h.Do(req) + require.NoError(t, err) + require.Equal(t, "200 OK", resp.Status) + var list struct { + Data struct { + Total int `json:"total"` + Items []struct { + Type string `json:"type"` + Id int `json:"id"` + Name string `json:"name"` + Emails []string `json:"emails"` + Roles []string `json:"roles"` + } `json:"items"` + } `json:"data"` + } + bb, err := io.ReadAll(resp.Body) + require.NoError(t, err) + defer resp.Body.Close() + err = json.Unmarshal(bb, &list) + require.NoError(t, err) + individuals := []struct { + Id int + Name string + Emails []string + Roles []string + }{} + for _, p := range list.Data.Items { + if p.Type == "individual" { + individuals = append(individuals, struct { + Id int + Name string + Emails []string + Roles []string + }{p.Id, p.Name, p.Emails, p.Roles}) + } + } + + require.Equal(t, len(users), len(individuals)) + } + + { + // check whether we can fetch a session for the provisioned users + for _, user := range users { + session, err := j.FetchSession(sessionUrl, user.name, logger) + require.NoError(t, err, "failed to retrieve JMAP session for newly created principal '%s'", user.name) + require.Equal(t, user.name, session.Username) + } + } + } success = true return &StalwartTest{ - t: t, - ip: ip, - imapPort: imapPort.Int(), - container: container, - ctx: ctx, - cancelCtx: cancel, - client: &j, - session: session, - username: username, - password: password, - logger: logger, - userPersonName: userPersonName, - userEmail: userEmail, + t: t, + ip: ip, + imapPort: imapPort.Int(), + container: container, + ctx: ctx, + cancelCtx: cancel, + client: &j, + logger: logger, + jmapBaseUrl: jmapBaseUrl, + sessionUrl: sessionUrl, }, nil } @@ -372,20 +487,6 @@ func replaceHost(u string, host string) (string, error) { } } -/* -func pickOneRandomlyFromMap[K comparable, V any](m map[K]V) (K, V) { - l := rand.Intn(len(m)) - i := 0 - for k, v := range m { - if i == l { - return k, v - } - i++ - } - panic("map is empty") -} -*/ - func pickRandomlyFromMap[K comparable, V any](m map[K]V, min int, max int) map[K]V { if min < 0 || max < 0 { panic("min and max must be >= 0") @@ -1023,6 +1124,10 @@ func pickRandom[T any](s ...T) T { return s[rand.Intn(len(s))] } +func pickUser() User { + return users[rand.Intn(len(users))] +} + func pickRandoms[T any](s ...T) []T { n := rand.Intn(len(s)) if n == 0 { diff --git a/pkg/jmap/jmap_integration_ws_test.go b/pkg/jmap/jmap_integration_ws_test.go index e01f63e0c6..673949775d 100644 --- a/pkg/jmap/jmap_integration_ws_test.go +++ b/pkg/jmap/jmap_integration_ws_test.go @@ -64,13 +64,16 @@ func TestWs(t *testing.T) { require.NoError(err) defer s.Close() - mailAccountId := s.session.PrimaryAccounts.Mail + user := pickUser() + session := s.Session(user.name) + + mailAccountId := session.PrimaryAccounts.Mail inboxFolder := "" { - _, inboxFolder = s.findInbox(t, mailAccountId) + _, inboxFolder = s.findInbox(t, mailAccountId, session) } - l := &testWsPushListener{t: t, username: s.username, logger: s.logger, mailAccountId: mailAccountId} + l := &testWsPushListener{t: t, username: user.name, logger: s.logger, mailAccountId: mailAccountId} s.client.AddWsPushListener(l) require.Equal(uint32(0), l.calls.Load()) @@ -84,9 +87,9 @@ func TestWs(t *testing.T) { var initialState State { - changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, s.session, s.ctx, s.logger, "", "", 0) + changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, session, s.ctx, s.logger, "", "", 0) require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.NotEmpty(state) //fmt.Printf("\x1b[45;1;4mChanges [%s]:\x1b[0m\n", state) //for _, c := range changes.Created { fmt.Printf("%s %s\n", c.Id, c.Subject) } @@ -98,9 +101,9 @@ func TestWs(t *testing.T) { require.NotEmpty(initialState) { - changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, s.session, s.ctx, s.logger, "", initialState, 0) + changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, session, s.ctx, s.logger, "", initialState, 0) require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.Equal(initialState, state) require.Equal(initialState, changes.NewState) require.Empty(changes.Created) @@ -108,7 +111,7 @@ func TestWs(t *testing.T) { require.Empty(changes.Updated) } - wsc, err := s.client.EnablePushNotifications(initialState, func() (*Session, error) { return s.session, nil }) + wsc, err := s.client.EnablePushNotifications(initialState, func() (*Session, error) { return session, nil }) require.NoError(err) defer wsc.Close() @@ -124,7 +127,7 @@ func TestWs(t *testing.T) { emailIds := []string{} { - _, n, err := s.fillEmailsWithImap(inboxFolder, 1, false) + _, n, err := s.fillEmailsWithImap(inboxFolder, 1, false, user) require.NoError(err) require.Equal(1, n) } @@ -141,9 +144,9 @@ func TestWs(t *testing.T) { } var lastState State { - changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, s.session, s.ctx, s.logger, "", initialState, 0) + changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, session, s.ctx, s.logger, "", initialState, 0) require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.NotEqual(initialState, state) require.NotEqual(initialState, changes.NewState) require.Equal(state, changes.NewState) @@ -156,7 +159,7 @@ func TestWs(t *testing.T) { } { - _, n, err := s.fillEmailsWithImap(inboxFolder, 1, false) + _, n, err := s.fillEmailsWithImap(inboxFolder, 1, false, user) require.NoError(err) require.Equal(1, n) } @@ -175,9 +178,9 @@ func TestWs(t *testing.T) { l.m.Unlock() } { - changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, s.session, s.ctx, s.logger, "", lastState, 0) + changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, session, s.ctx, s.logger, "", lastState, 0) require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.NotEqual(lastState, state) require.NotEqual(lastState, changes.NewState) require.Equal(state, changes.NewState) @@ -190,7 +193,7 @@ func TestWs(t *testing.T) { } { - _, n, err := s.fillEmailsWithImap(inboxFolder, 0, true) + _, n, err := s.fillEmailsWithImap(inboxFolder, 0, true, user) require.NoError(err) require.Equal(0, n) } @@ -209,9 +212,9 @@ func TestWs(t *testing.T) { l.m.Unlock() } { - changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, s.session, s.ctx, s.logger, "", lastState, 0) + changes, sessionState, state, _, err := s.client.GetEmailChanges(mailAccountId, session, s.ctx, s.logger, "", lastState, 0) require.NoError(err) - require.Equal(s.session.State, sessionState) + require.Equal(session.State, sessionState) require.NotEqual(lastState, state) require.NotEqual(lastState, changes.NewState) require.Equal(state, changes.NewState) diff --git a/pkg/jmap/jmap_model.go b/pkg/jmap/jmap_model.go index 603493b868..3863b61bf8 100644 --- a/pkg/jmap/jmap_model.go +++ b/pkg/jmap/jmap_model.go @@ -2358,6 +2358,11 @@ type Email struct { Preview string `json:"preview,omitempty"` } +type AddressParameters struct { + HoldUntil time.Time `json:"HOLDUNTIL,omitzero"` + HoldForSeconds uint `json:"HOLDFOR,omitzero"` +} + type Address struct { // The email address being represented by the object. // @@ -2373,7 +2378,7 @@ type Address struct { // or null if the parameter does not take a value. // // [RFC5321]: https://datatracker.ietf.org/doc/html/rfc5321 - Parameters map[string]any `json:"parameters,omitempty"` // TODO RFC5321 + Parameters *AddressParameters `json:"parameters,omitempty"` } // Information for use when sending via SMTP. @@ -2987,6 +2992,20 @@ type EmailCreate struct { // The value for each key in the object MUST be true. Keywords map[string]bool `json:"keywords,omitempty"` + // This is a list of all header fields [RFC5322], in the same order they appear in the message. + // + // [RFC5322]: https://www.rfc-editor.org/rfc/rfc5322.html + Headers []EmailHeader `json:"headers,omitempty"` + + // The value is identical to the value of header:In-Reply-To:asMessageIds. + InReplyTo []string `json:"inReplyTo,omitempty"` + + // The value is identical to the value of header:References:asMessageIds. + References []string `json:"references,omitempty"` + + // The value is identical to the value of header:Sender:asAddresses. + Sender []EmailAddress `json:"sender,omitempty"` + // The ["From:" field] specifies the author(s) of the message, that is, the mailbox(es) // of the person(s) or system(s) responsible for the writing of the message //