From b7a7c9526b0830d70dcd10fb6d96bdfb7233ecd7 Mon Sep 17 00:00:00 2001 From: Pascal Bleser Date: Tue, 5 Aug 2025 16:28:31 +0200 Subject: [PATCH] groupware: implement message search with snippets --- pkg/jmap/jmap.go | 81 ++++++++- pkg/jmap/jmap_model.go | 92 ++++++++-- pkg/jmap/jmap_tools.go | 10 +- .../pkg/groupware/groupware_api_messages.go | 167 +++++++++++++++--- .../pkg/groupware/groupware_framework.go | 40 +++++ .../pkg/groupware/groupware_route.go | 35 ++-- 6 files changed, 363 insertions(+), 62 deletions(-) diff --git a/pkg/jmap/jmap.go b/pkg/jmap/jmap.go index bbec7c9623..933ec12bc5 100644 --- a/pkg/jmap/jmap.go +++ b/pkg/jmap/jmap.go @@ -250,10 +250,10 @@ func (j *Client) GetAllMailboxes(accountId string, session *Session, ctx context } // https://jmap.io/spec-mail.html#mailboxquery -func (j *Client) QueryMailbox(accountId string, session *Session, ctx context.Context, logger *log.Logger, filter MailboxFilterCondition) (MailboxQueryResponse, Error) { +func (j *Client) QueryMailbox(accountId string, session *Session, ctx context.Context, logger *log.Logger, filter MailboxFilterElement) (MailboxQueryResponse, Error) { aid := session.MailAccountId(accountId) logger = j.logger(aid, "QueryMailbox", session, logger) - cmd, err := request(invocation(MailboxQuery, SimpleMailboxQueryCommand{AccountId: aid, Filter: filter}, "0")) + cmd, err := request(invocation(MailboxQuery, MailboxQueryCommand{AccountId: aid, Filter: filter}, "0")) if err != nil { return MailboxQueryResponse{}, SimpleError{code: JmapErrorInvalidJmapRequestPayload, err: err} } @@ -269,12 +269,12 @@ type Mailboxes struct { State string `json:"state,omitempty"` } -func (j *Client) SearchMailboxes(accountId string, session *Session, ctx context.Context, logger *log.Logger, filter MailboxFilterCondition) (Mailboxes, Error) { +func (j *Client) SearchMailboxes(accountId string, session *Session, ctx context.Context, logger *log.Logger, filter MailboxFilterElement) (Mailboxes, Error) { aid := session.MailAccountId(accountId) logger = j.logger(aid, "SearchMailboxes", session, logger) cmd, err := request( - invocation(MailboxQuery, SimpleMailboxQueryCommand{AccountId: aid, Filter: filter}, "0"), + invocation(MailboxQuery, MailboxQueryCommand{AccountId: aid, Filter: filter}, "0"), invocation(MailboxGet, MailboxGetRefCommand{ AccountId: aid, IdRef: &ResultReference{Name: MailboxQuery, Path: "/ids/*", ResultOf: "0"}, @@ -330,7 +330,7 @@ func (j *Client) GetAllEmails(accountId string, session *Session, ctx context.Co query := EmailQueryCommand{ AccountId: aid, - Filter: &MessageFilter{InMailbox: mailboxId}, + Filter: &EmailFilterCondition{InMailbox: mailboxId}, Sort: []Sort{{Property: emailSortByReceivedAt, IsAscending: false}}, CollapseThreads: true, CalculateTotal: false, @@ -518,6 +518,77 @@ func (j *Client) GetEmailsSince(accountId string, session *Session, ctx context. }) } +type EmailQueryResult struct { + Snippets []SearchSnippet `json:"snippets,omitempty"` + QueryState string `json:"queryState"` + Total int `json:"total"` + Limit int `json:"limit,omitzero"` + Position int `json:"position,omitzero"` +} + +func (j *Client) QueryEmails(accountId string, filter EmailFilterElement, session *Session, ctx context.Context, logger *log.Logger, offset int, limit int, fetchBodies bool, maxBodyValueBytes int) (EmailQueryResult, Error) { + aid := session.MailAccountId(accountId) + logger = j.loggerParams(aid, "QueryEmails", session, logger, func(z zerolog.Context) zerolog.Context { + return z.Bool(logFetchBodies, fetchBodies) + }) + + query := EmailQueryCommand{ + AccountId: aid, + Filter: filter, + Sort: []Sort{{Property: emailSortByReceivedAt, IsAscending: false}}, + CollapseThreads: true, + CalculateTotal: true, + } + if offset >= 0 { + query.Position = offset + } + if limit >= 0 { + query.Limit = limit + } + + snippet := SearchSnippetRefCommand{ + AccountId: aid, + Filter: filter, + EmailIdRef: &ResultReference{ + ResultOf: "0", + Name: EmailQuery, + Path: "/ids/*", + }, + } + + cmd, err := request( + invocation(EmailQuery, query, "0"), + invocation(SearchSnippetGet, snippet, "1"), + ) + + if err != nil { + return EmailQueryResult{}, SimpleError{code: JmapErrorInvalidJmapRequestPayload, err: err} + } + + return command(j.api, logger, ctx, session, j.onSessionOutdated, cmd, func(body *Response) (EmailQueryResult, Error) { + var queryResponse EmailQueryResponse + err = retrieveResponseMatchParameters(body, EmailQuery, "0", &queryResponse) + if err != nil { + return EmailQueryResult{}, SimpleError{code: JmapErrorInvalidJmapResponsePayload, err: err} + } + + var snippetResponse SearchSnippetGetResponse + err = retrieveResponseMatchParameters(body, SearchSnippetGet, "1", &snippetResponse) + if err != nil { + return EmailQueryResult{}, SimpleError{code: JmapErrorInvalidJmapResponsePayload, err: err} + } + + return EmailQueryResult{ + Snippets: snippetResponse.List, + QueryState: queryResponse.QueryState, + Total: queryResponse.Total, + Limit: queryResponse.Limit, + Position: queryResponse.Position, + }, nil + }) + +} + func (j *Client) GetBlob(accountId string, session *Session, ctx context.Context, logger *log.Logger, id string) (*Blob, Error) { aid := session.BlobAccountId(accountId) diff --git a/pkg/jmap/jmap_model.go b/pkg/jmap/jmap_model.go index 3f90b3e3d3..9299af6093 100644 --- a/pkg/jmap/jmap_model.go +++ b/pkg/jmap/jmap_model.go @@ -244,6 +244,14 @@ type SetError struct { Description string `json:"description,omitempty"` } +type FilterOperatorTerm string + +const ( + And FilterOperatorTerm = "AND" + Or FilterOperatorTerm = "OR" + Not FilterOperatorTerm = "NOT" +) + type Mailbox struct { Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` @@ -274,7 +282,12 @@ type MailboxChangesCommand struct { MaxChanges int `json:"maxChanges,omitzero"` } +type MailboxFilterElement interface { + _isAMailboxFilterElement() // marker method +} + type MailboxFilterCondition struct { + MailboxFilterElement ParentId string `json:"parentId,omitempty"` Name string `json:"name,omitempty"` Role string `json:"role,omitempty"` @@ -282,11 +295,16 @@ type MailboxFilterCondition struct { IsSubscribed *bool `json:"isSubscribed,omitempty"` } +var _ MailboxFilterElement = &MailboxFilterCondition{} + type MailboxFilterOperator struct { - Operator string `json:"operator"` - Conditions []MailboxFilterCondition `json:"conditions"` + MailboxFilterElement + Operator FilterOperatorTerm `json:"operator"` + Conditions []MailboxFilterElement `json:"conditions,omitempty"` } +var _ MailboxFilterElement = &MailboxFilterOperator{} + type MailboxComparator struct { Property string `json:"property"` IsAscending bool `json:"isAscending,omitempty"` @@ -294,15 +312,20 @@ type MailboxComparator struct { CalculateTotal bool `json:"calculateTotal,omitempty"` } -type SimpleMailboxQueryCommand struct { - AccountId string `json:"accountId"` - Filter MailboxFilterCondition `json:"filter,omitempty"` - Sort []MailboxComparator `json:"sort,omitempty"` - SortAsTree bool `json:"sortAsTree,omitempty"` - FilterAsTree bool `json:"filterAsTree,omitempty"` +type MailboxQueryCommand struct { + AccountId string `json:"accountId"` + Filter MailboxFilterElement `json:"filter,omitempty"` + Sort []MailboxComparator `json:"sort,omitempty"` + SortAsTree bool `json:"sortAsTree,omitempty"` + FilterAsTree bool `json:"filterAsTree,omitempty"` } -type MessageFilter struct { +type EmailFilterElement interface { + _isAnEmailFilterElement() // marker method +} + +type EmailFilterCondition struct { + EmailFilterElement InMailbox string `json:"inMailbox,omitempty"` InMailboxOtherThan []string `json:"inMailboxOtherThan,omitempty"` Before time.Time `json:"before,omitzero"` // omitzero requires Go 1.24 @@ -316,8 +339,25 @@ type MessageFilter struct { NotKeyword string `json:"notKeyword,omitempty"` HasAttachment bool `json:"hasAttachment,omitempty"` Text string `json:"text,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + Cc string `json:"cc,omitempty"` + Bcc string `json:"bcc,omitempty"` + Subject string `json:"subject,omitempty"` + Body string `json:"body,omitempty"` + Header []string `json:"header,omitempty"` } +var _ EmailFilterElement = &EmailFilterCondition{} + +type EmailFilterOperator struct { + EmailFilterElement + Operator FilterOperatorTerm `json:"operator"` + Conditions []EmailFilterElement `json:"conditions,omitempty"` +} + +var _ EmailFilterElement = &EmailFilterOperator{} + type Sort struct { Property string `json:"property,omitempty"` IsAscending bool `json:"isAscending,omitempty"` @@ -326,13 +366,13 @@ type Sort struct { } type EmailQueryCommand struct { - AccountId string `json:"accountId"` - Filter *MessageFilter `json:"filter,omitempty"` - Sort []Sort `json:"sort,omitempty"` - CollapseThreads bool `json:"collapseThreads,omitempty"` - Position int `json:"position,omitempty"` - Limit int `json:"limit,omitempty"` - CalculateTotal bool `json:"calculateTotal,omitempty"` + AccountId string `json:"accountId"` + Filter EmailFilterElement `json:"filter,omitempty"` + Sort []Sort `json:"sort,omitempty"` + CollapseThreads bool `json:"collapseThreads,omitempty"` + Position int `json:"position,omitempty"` + Limit int `json:"limit,omitempty"` + CalculateTotal bool `json:"calculateTotal,omitempty"` } type EmailGetCommand struct { @@ -1306,6 +1346,24 @@ type BlobDownload struct { CacheControl string } +type SearchSnippet struct { + EmailId string `json:"emailId"` + Subject string `json:"subject,omitempty"` + Preview string `json:"preview,omitempty"` +} + +type SearchSnippetRefCommand struct { + AccountId string `json:"accountId"` + Filter EmailFilterElement `json:"filter,omitempty"` + EmailIdRef *ResultReference `json:"#emailIds,omitempty"` +} + +type SearchSnippetGetResponse struct { + AccountId string `json:"accountId"` + List []SearchSnippet `json:"list,omitempty"` + NotFound []string `json:"notFound,omitempty"` +} + const ( BlobGet Command = "Blob/get" BlobUpload Command = "Blob/upload" @@ -1320,6 +1378,7 @@ const ( MailboxChanges Command = "Mailbox/changes" IdentityGet Command = "Identity/get" VacationResponseGet Command = "VacationResponse/get" + SearchSnippetGet Command = "SearchSnippet/get" ) var CommandResponseTypeMap = map[Command]func() any{ @@ -1334,4 +1393,5 @@ var CommandResponseTypeMap = map[Command]func() any{ ThreadGet: func() any { return ThreadGetResponse{} }, IdentityGet: func() any { return IdentityGetResponse{} }, VacationResponseGet: func() any { return VacationResponseGetResponse{} }, + SearchSnippetGet: func() any { return SearchSnippetGetResponse{} }, } diff --git a/pkg/jmap/jmap_tools.go b/pkg/jmap/jmap_tools.go index a975b7f09c..3402ba6d34 100644 --- a/pkg/jmap/jmap_tools.go +++ b/pkg/jmap/jmap_tools.go @@ -51,15 +51,15 @@ func command[T any](api ApiClient, return zero, jmapErr } - var data Response - err := json.Unmarshal(responseBody, &data) + var response Response + err := json.Unmarshal(responseBody, &response) if err != nil { logger.Error().Err(err).Msg("failed to deserialize body JSON payload") var zero T return zero, SimpleError{code: JmapErrorDecodingResponseBody, err: err} } - if data.SessionState != session.State { + if response.SessionState != session.State { if sessionOutdatedHandler != nil { sessionOutdatedHandler(session) } @@ -67,7 +67,7 @@ func command[T any](api ApiClient, // search for an "error" response // https://jmap.io/spec-core.html#method-level-errors - for _, mr := range data.MethodResponses { + for _, mr := range response.MethodResponses { if mr.Command == "error" { err := fmt.Errorf("found method level error in response '%v'", mr.Tag) if payload, ok := mr.Parameters.(map[string]any); ok { @@ -80,7 +80,7 @@ func command[T any](api ApiClient, } } - return mapper(&data) + return mapper(&response) } func mapstructStringToTimeHook() mapstructure.DecodeHookFunc { diff --git a/services/groupware/pkg/groupware/groupware_api_messages.go b/services/groupware/pkg/groupware/groupware_api_messages.go index e8f1120f7f..ab1cf0853a 100644 --- a/services/groupware/pkg/groupware/groupware_api_messages.go +++ b/services/groupware/pkg/groupware/groupware_api_messages.go @@ -7,6 +7,7 @@ import ( "github.com/go-chi/chi/v5" + "github.com/opencloud-eu/opencloud/pkg/jmap" "github.com/opencloud-eu/opencloud/pkg/log" ) @@ -37,6 +38,7 @@ func (g Groupware) GetAllMessages(w http.ResponseWriter, r *http.Request) { }) } else { g.respond(w, r, func(req Request) (any, string, *Error) { + l := req.logger.With() if mailboxId == "" { errorId := req.errorId() msg := fmt.Sprintf("Missing required mailbox ID path parameter '%v'", UriParamMailboxId) @@ -45,28 +47,23 @@ func (g Groupware) GetAllMessages(w http.ResponseWriter, r *http.Request) { withSource(&ErrorSource{Parameter: UriParamMailboxId}), ) } - page, ok, err := req.parseNumericParam(QueryParamPage, -1) - if err != nil { - return nil, "", err - } - logger := req.logger - if ok { - logger = &log.Logger{Logger: logger.With().Int(QueryParamPage, page).Logger()} - } - - size, ok, err := req.parseNumericParam(QueryParamSize, -1) + offset, ok, err := req.parseNumericParam(QueryParamOffset, 0) if err != nil { return nil, "", err } if ok { - logger = &log.Logger{Logger: logger.With().Int(QueryParamSize, size).Logger()} + l = l.Int(QueryParamOffset, offset) } - offset := page * size - limit := size - if limit < 0 { - limit = g.defaultEmailLimit + limit, ok, err := req.parseNumericParam(QueryParamLimit, g.defaultEmailLimit) + if err != nil { + return nil, "", err } + if ok { + l = l.Int(QueryParamLimit, limit) + } + + logger := &log.Logger{Logger: l.Logger()} emails, jerr := g.jmap.GetAllEmails(req.GetAccountId(), req.session, req.ctx, logger, mailboxId, offset, limit, true, g.maxBodyValueBytes) if jerr != nil { @@ -101,21 +98,141 @@ func (g Groupware) GetMessagesById(w http.ResponseWriter, r *http.Request) { }) } -func (g Groupware) GetMessageUpdates(w http.ResponseWriter, r *http.Request) { +func (g Groupware) GetMessages(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() since := q.Get(QueryParamSince) if since == "" { since = r.Header.Get("If-None-Match") } - maxChanges := -1 - g.respond(w, r, func(req Request) (any, string, *Error) { - logger := &log.Logger{Logger: req.logger.With().Str(HeaderSince, since).Logger()} + if since != "" { + // get messages changes since a given state + maxChanges := -1 + g.respond(w, r, func(req Request) (any, string, *Error) { + logger := &log.Logger{Logger: req.logger.With().Str(HeaderSince, since).Logger()} - emails, jerr := g.jmap.GetEmailsSince(req.GetAccountId(), req.session, req.ctx, logger, since, true, g.maxBodyValueBytes, maxChanges) - if jerr != nil { - return nil, "", req.apiErrorFromJmap(jerr) - } + emails, jerr := g.jmap.GetEmailsSince(req.GetAccountId(), req.session, req.ctx, logger, since, true, g.maxBodyValueBytes, maxChanges) + if jerr != nil { + return nil, "", req.apiErrorFromJmap(jerr) + } - return emails, emails.State, nil - }) + return emails, emails.State, nil + }) + } else { + // do a search + g.respond(w, r, func(req Request) (any, string, *Error) { + mailboxId := q.Get(QueryParamMailboxId) + notInMailboxIds := q[QueryParamNotInMailboxId] + text := q.Get(QueryParamSearchText) + from := q.Get(QueryParamSearchFrom) + to := q.Get(QueryParamSearchTo) + cc := q.Get(QueryParamSearchCc) + bcc := q.Get(QueryParamSearchBcc) + subject := q.Get(QueryParamSearchSubject) + body := q.Get(QueryParamSearchBody) + + l := req.logger.With() + + offset, ok, err := req.parseNumericParam(QueryParamOffset, 0) + if err != nil { + return nil, "", err + } + if ok { + l = l.Int(QueryParamOffset, offset) + } + + limit, ok, err := req.parseNumericParam(QueryParamLimit, g.defaultEmailLimit) + if err != nil { + return nil, "", err + } + if ok { + l = l.Int(QueryParamLimit, limit) + } + + before, ok, err := req.parseDateParam(QueryParamSearchBefore) + if err != nil { + return nil, "", err + } + if ok { + l = l.Time(QueryParamSearchBefore, before) + } + + after, ok, err := req.parseDateParam(QueryParamSearchAfter) + if err != nil { + return nil, "", err + } + if ok { + l = l.Time(QueryParamSearchAfter, after) + } + + if mailboxId != "" { + l = l.Str(QueryParamMailboxId, logstr(mailboxId)) + } + if len(notInMailboxIds) > 0 { + l = l.Array(QueryParamNotInMailboxId, logstrarray(notInMailboxIds)) + } + if text != "" { + l = l.Str(QueryParamSearchText, logstr(text)) + } + if from != "" { + l = l.Str(QueryParamSearchFrom, logstr(from)) + } + if to != "" { + l = l.Str(QueryParamSearchTo, logstr(to)) + } + if cc != "" { + l = l.Str(QueryParamSearchCc, logstr(cc)) + } + if bcc != "" { + l = l.Str(QueryParamSearchBcc, logstr(bcc)) + } + if subject != "" { + l = l.Str(QueryParamSearchSubject, logstr(subject)) + } + if body != "" { + l = l.Str(QueryParamSearchBody, logstr(body)) + } + + minSize, ok, err := req.parseNumericParam(QueryParamSearchMinSize, 0) + if err != nil { + return nil, "", err + } + if ok { + l = l.Int(QueryParamSearchMinSize, minSize) + } + + maxSize, ok, err := req.parseNumericParam(QueryParamSearchMaxSize, 0) + if err != nil { + return nil, "", err + } + if ok { + l = l.Int(QueryParamSearchMaxSize, maxSize) + } + + logger := &log.Logger{Logger: l.Logger()} + + filter := jmap.EmailFilterCondition{ + Text: text, + InMailbox: mailboxId, + InMailboxOtherThan: notInMailboxIds, + From: from, + To: to, + Cc: cc, + Bcc: bcc, + Subject: subject, + Body: body, + Before: before, + After: after, + MinSize: minSize, + MaxSize: maxSize, + //HasKeyword: "", + } + + emails, jerr := g.jmap.QueryEmails(req.GetAccountId(), &filter, req.session, req.ctx, logger, offset, limit, false, 0) + if jerr != nil { + return nil, "", req.apiErrorFromJmap(jerr) + } + + return emails, emails.QueryState, nil + }) + } } diff --git a/services/groupware/pkg/groupware/groupware_framework.go b/services/groupware/pkg/groupware/groupware_framework.go index cb19123671..4930546748 100644 --- a/services/groupware/pkg/groupware/groupware_framework.go +++ b/services/groupware/pkg/groupware/groupware_framework.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "strconv" + "time" "github.com/go-chi/chi/v5" "github.com/go-chi/render" @@ -222,6 +223,29 @@ func (r Request) parseNumericParam(param string, defaultValue int) (int, bool, * return int(value), true, nil } +func (r Request) parseDateParam(param string) (time.Time, bool, *Error) { + q := r.r.URL.Query() + if !q.Has(param) { + return time.Time{}, false, nil + } + + str := q.Get(param) + if str == "" { + return time.Time{}, false, nil + } + + t, err := time.Parse(time.RFC3339, str) + if err != nil { + errorId := r.errorId() + msg := fmt.Sprintf("Invalid RFC3339 value for query parameter '%v': '%s': %s", param, logstr(str), err.Error()) + return time.Time{}, true, apiError(errorId, ErrorInvalidRequestParameter, + withDetail(msg), + withSource(&ErrorSource{Parameter: param}), + ) + } + return t, true, nil +} + // Safely caps a string to a given size to avoid log bombing. // Use this function to wrap strings that are user input (HTTP headers, path parameters, URI parameters, HTTP body, ...). func logstr(text string) string { @@ -234,6 +258,22 @@ func logstr(text string) string { } } +type SafeLogStringArrayMarshaller struct { + array []string +} + +func (m SafeLogStringArrayMarshaller) MarshalZerologArray(a *zerolog.Array) { + for _, elem := range m.array { + a.Str(logstr(elem)) + } +} + +var _ zerolog.LogArrayMarshaler = SafeLogStringArrayMarshaller{} + +func logstrarray(array []string) SafeLogStringArrayMarshaller { + return SafeLogStringArrayMarshaller{array: array} +} + func (g Groupware) log(error *Error) { var level *zerolog.Event if error.NumStatus < 300 { diff --git a/services/groupware/pkg/groupware/groupware_route.go b/services/groupware/pkg/groupware/groupware_route.go index 144ffdbd74..7f661a3ffa 100644 --- a/services/groupware/pkg/groupware/groupware_route.go +++ b/services/groupware/pkg/groupware/groupware_route.go @@ -5,16 +5,29 @@ import ( ) const ( - UriParamAccount = "account" - UriParamMailboxId = "mailbox" - QueryParamPage = "page" - QueryParamSize = "size" - UriParamMessagesId = "id" - UriParamBlobId = "blobid" - UriParamBlobName = "blobname" - QueryParamBlobType = "type" - QueryParamSince = "since" - HeaderSince = "if-none-match" + UriParamAccount = "account" + UriParamMailboxId = "mailbox" + UriParamMessagesId = "id" + UriParamBlobId = "blobid" + UriParamBlobName = "blobname" + QueryParamBlobType = "type" + QueryParamSince = "since" + QueryParamMailboxId = "mailbox" + QueryParamNotInMailboxId = "notmailbox" + QueryParamSearchText = "text" + QueryParamSearchFrom = "from" + QueryParamSearchTo = "to" + QueryParamSearchCc = "cc" + QueryParamSearchBcc = "bcc" + QueryParamSearchSubject = "subject" + QueryParamSearchBody = "body" + QueryParamSearchBefore = "before" + QueryParamSearchAfter = "after" + QueryParamSearchMinSize = "minsize" + QueryParamSearchMaxSize = "maxsize" + QueryParamOffset = "offset" + QueryParamLimit = "limit" + HeaderSince = "if-none-match" ) func (g Groupware) Route(r chi.Router) { @@ -30,7 +43,7 @@ func (g Groupware) Route(r chi.Router) { r.Get("/{mailbox}/messages", g.GetAllMessages) }) r.Route("/messages", func(r chi.Router) { - r.Get("/", g.GetMessageUpdates) + r.Get("/", g.GetMessages) r.Get("/{id}", g.GetMessagesById) }) r.Route("/blobs", func(r chi.Router) {