package jmap import ( "bytes" "context" "crypto/tls" "encoding/json" "fmt" "io" "log" "maps" "math/rand" "net/http" "net/http/httputil" "net/url" "os" "reflect" "regexp" "slices" "strings" "testing" "text/template" "time" "github.com/stretchr/testify/require" "github.com/gorilla/websocket" "github.com/tidwall/pretty" "golang.org/x/text/cases" "golang.org/x/text/language" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" "github.com/brianvoe/gofakeit/v7" pw "github.com/sethvargo/go-password/password" "github.com/opencloud-eu/opencloud/pkg/jscontact" clog "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/pkg/structs" "github.com/go-crypt/crypt/algorithm/shacrypt" ) var ( domains = [...]string{"earth.gov", "mars.mil", "opa.org", "acme.com"} people = [...]string{ "Camina Drummer", "Amos Burton", "James Holden", "Anderson Dawes", "Naomi Nagata", "Klaes Ashford", "Fred Johnson", "Chrisjen Avasarala", "Bobby Draper", } ) const ( stalwartImage = "ghcr.io/stalwartlabs/stalwart:v0.14.0-alpine" httpPort = "8080" imapsPort = "993" configTemplate = ` authentication.fallback-admin.secret = "$6$4qPYDVhaUHkKcY7s$bB6qhcukb9oFNYRIvaDZgbwxrMa2RvF5dumCjkBFdX19lSNqrgKltf3aPrFMuQQKkZpK2YNuQ83hB1B3NiWzj." authentication.fallback-admin.user = "mailadmin" authentication.master.secret = "{{.masterpassword}}" authentication.master.user = "{{.masterusername}}" directory.memory.principals.0000.class = "admin" directory.memory.principals.0000.description = "Superuser" directory.memory.principals.0000.email.0000 = "admin@example.org" directory.memory.principals.0000.name = "admin" directory.memory.principals.0000.secret = "secret" directory.memory.principals.0001.class = "individual" directory.memory.principals.0001.description = "{{.description}}" directory.memory.principals.0001.email.0000 = "{{.email}}" directory.memory.principals.0001.name = "{{.username}}" directory.memory.principals.0001.secret = "{{.password}}" directory.memory.principals.0001.storage.directory = "memory" directory.memory.type = "memory" metrics.prometheus.enable = false server.listener.http.bind = "[::]:{{.httpPort}}" server.listener.http.protocol = "http" server.listener.imaptls.bind = "[::]:{{.imapsPort}}" server.listener.imaptls.protocol = "imap" server.listener.imaptls.tls.implicit = true server.hostname = "{{.hostname}}" server.max-connections = 8192 server.socket.backlog = 1024 server.socket.nodelay = true server.socket.reuse-addr = true server.socket.reuse-port = true storage.blob = "rocksdb" storage.data = "rocksdb" storage.directory = "memory" storage.fts = "rocksdb" storage.lookup = "rocksdb" store.rocksdb.compression = "lz4" store.rocksdb.path = "/opt/stalwart/data" store.rocksdb.type = "rocksdb" tracer.log.ansi = false tracer.log.buffered = false tracer.log.enable = true tracer.log.level = "trace" tracer.log.lossy = false tracer.log.multiline = false tracer.log.type = "stdout" ` ) func skip(t *testing.T) bool { if os.Getenv("CI") == "woodpecker" { t.Skip("Skipping tests because CI==wookpecker") return true } if os.Getenv("CI_SYSTEM_NAME") == "woodpecker" { t.Skip("Skipping tests because CI_SYSTEM_NAME==wookpecker") return true } if os.Getenv("USE_TESTCONTAINERS") == "false" { t.Skip("Skipping tests because USE_TESTCONTAINERS==false") return true } return false } type StalwartTest struct { t *testing.T ip string imapPort int container *testcontainers.DockerContainer ctx context.Context cancelCtx context.CancelFunc client *Client session *Session username string password string logger *clog.Logger userPersonName string userEmail string io.Closer } func (s *StalwartTest) Close() error { if s.container != nil { var c testcontainers.Container = s.container testcontainers.CleanupContainer(s.t, c) } if s.cancelCtx != nil { s.cancelCtx() } return nil } func newStalwartTest(t *testing.T) (*StalwartTest, error) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) var _ context.CancelFunc = cancel // ignore context leak warning: it is passed in the struct and called in Close() // A master user name different from "master" does not seem to work as of the current Stalwart version //masterUsernameSuffix, err := pw.Generate(4+rand.Intn(28), 2, 0, false, true) //require.NoError(err) masterUsername := "master" //"master_" + masterUsernameSuffix masterPassword, err := pw.Generate(4+rand.Intn(28), 2, 0, false, true) if err != nil { return nil, err } masterPasswordHash := "" { hasher, err := shacrypt.New(shacrypt.WithSHA512(), shacrypt.WithIterations(shacrypt.IterationsDefaultOmitted)) if err != nil { return nil, err } digest, err := hasher.Hash(masterPassword) if err != nil { return nil, err } masterPasswordHash = digest.Encode() } usernameSuffix, err := pw.Generate(8, 2, 0, true, true) if err != nil { return nil, err } username := "user_" + usernameSuffix password, err := pw.Generate(4+rand.Intn(28), 2, 0, false, true) if err != nil { return nil, err } hostname := "localhost" userPersonName := people[rand.Intn(len(people))] var userEmail string { domain := domains[rand.Intn(len(domains))] userEmail = strings.Join(strings.Split(cases.Lower(language.English).String(userPersonName), " "), ".") + "@" + domain } configBuf := bytes.NewBufferString("") template.Must(template.New("").Parse(configTemplate)).Execute(configBuf, map[string]any{ "hostname": hostname, "password": password, "username": username, "description": userPersonName, "email": userEmail, "masterusername": masterUsername, "masterpassword": masterPasswordHash, "httpPort": httpPort, "imapsPort": imapsPort, }) config := configBuf.String() configReader := strings.NewReader(config) container, err := testcontainers.Run( ctx, stalwartImage, testcontainers.WithExposedPorts(httpPort+"/tcp", imapsPort+"/tcp"), testcontainers.WithFiles(testcontainers.ContainerFile{ Reader: configReader, ContainerFilePath: "/opt/stalwart/etc/config.toml", FileMode: 0o700, }), testcontainers.WithWaitStrategyAndDeadline( 30*time.Second, wait.ForLog(`Network listener started (network.listen-start) listenerId = "imaptls"`), wait.ForLog(`Network listener started (network.listen-start) listenerId = "http"`), ), ) success := false defer func() { if !success { testcontainers.CleanupContainer(t, container) } }() ip, err := container.Host(ctx) if err != nil { return nil, err } imapPort, err := container.MappedPort(ctx, "993") if err != nil { return nil, err } tlsConfig := &tls.Config{InsecureSkipVerify: true} loggerImpl := clog.NewLogger(clog.Level("trace")) logger := &loggerImpl var j Client var session *Session { tr := http.DefaultTransport.(*http.Transport).Clone() tr.ResponseHeaderTimeout = time.Duration(30 * time.Second) tr.TLSClientConfig = tlsConfig jh := *http.DefaultClient jh.Transport = tr wsd := &websocket.Dialer{ TLSClientConfig: tlsConfig, HandshakeTimeout: time.Duration(10) * time.Second, } jmapPort, err := container.MappedPort(ctx, httpPort) if err != nil { return nil, err } jmapBaseUrl := url.URL{ Scheme: "http", Host: ip + ":" + jmapPort.Port(), Path: "/", } sessionUrl := jmapBaseUrl.JoinPath(".well-known", "jmap") api := NewHttpJmapClient( &jh, masterUsername, masterPassword, nullHttpJmapApiClientEventListener{}, ) wscf, err := NewHttpWsClientFactory(wsd, masterUsername, masterPassword, logger) if err != nil { return nil, err } j = NewClient(api, api, api, wscf) s, err := j.FetchSession(sessionUrl, username, logger) if err != nil { return nil, err } // we have to overwrite the hostname in JMAP URL because the container // will know its name to be a random Docker container identifier, or // "localhost" as we defined the hostname in the Stalwart configuration, // and we also need to overwrite the port number as its not mapped s.JmapUrl.Host = jmapBaseUrl.Host s.WebsocketUrl.Host = jmapBaseUrl.Host //s.JmapEndpoint = jmapBaseUrl.Host s.ApiUrl, err = replaceHost(s.ApiUrl, jmapBaseUrl.Host) require.NoError(t, err) s.DownloadUrl, err = replaceHost(s.DownloadUrl, jmapBaseUrl.Host) require.NoError(t, err) s.UploadUrl, err = replaceHost(s.UploadUrl, jmapBaseUrl.Host) require.NoError(t, err) s.EventSourceUrl, err = replaceHost(s.EventSourceUrl, jmapBaseUrl.Host) require.NoError(t, err) session = &s } require.NotNil(t, session.Capabilities.Mail) require.NotNil(t, session.Capabilities.Calendars) require.NotNil(t, session.Capabilities.Contacts) success = true return &StalwartTest{ t: t, ip: ip, imapPort: imapPort.Int(), container: container, ctx: ctx, cancelCtx: cancel, client: &j, session: session, username: username, password: password, logger: logger, userPersonName: userPersonName, userEmail: userEmail, }, nil } var urlHostRegex = regexp.MustCompile(`^(https?://)(.+?)/(.+)$`) func replaceHost(u string, host string) (string, error) { if m := urlHostRegex.FindAllStringSubmatch(u, -1); m != nil { return fmt.Sprintf("%s%s/%s", m[0][1], host, m[0][3]), nil } else { return "", fmt.Errorf("'%v' does not match '%v'", u, urlHostRegex) } } /* func pickOneRandomlyFromMap[K comparable, V any](m map[K]V) (K, V) { l := rand.Intn(len(m)) i := 0 for k, v := range m { if i == l { return k, v } i++ } panic("map is empty") } */ func pickRandomlyFromMap[K comparable, V any](m map[K]V, min int, max int) map[K]V { if min < 0 || max < 0 { panic("min and max must be >= 0") } l := len(m) if min > l || max > l { panic(fmt.Sprintf("min and max must be <= %d", l)) } n := min + rand.Intn(max-min+1) if n == l { return m } // let's use a deep copy so we can remove elements as we pick them c := make(map[K]V, l) maps.Copy(c, m) // r will hold the results r := make(map[K]V, n) for range n { pick := rand.Intn(len(c)) j := 0 for k, v := range m { if j == pick { delete(c, k) r[k] = v break } j++ } } return r } var productName = "jmaptest" func untype[S any](s S, t bool) S { if t { reflect.ValueOf(&s).Elem().FieldByName("Type").SetString("") } return s } type TestJmapClient struct { h *http.Client username string password string session *Session u *url.URL trace bool color bool } func NewTestJmapClient(session *Session, username string, password string, trace bool, color bool) (*TestJmapClient, error) { httpTransport := http.DefaultTransport.(*http.Transport).Clone() tlsConfig := &tls.Config{InsecureSkipVerify: true} httpTransport.TLSClientConfig = tlsConfig h := http.DefaultClient h.Transport = httpTransport u, err := url.Parse(session.ApiUrl) if err != nil { return nil, err } return &TestJmapClient{ h: h, trace: trace, color: color, username: username, password: password, session: session, u: u, }, nil } func (j *TestJmapClient) Close() error { return nil } type uploadedBlob struct { BlobId string `json:"blobId"` Size int `json:"size"` Type string `json:"type"` Sha512 string `json:"sha:512"` } func (j *TestJmapClient) uploadBlob(accountId string, data []byte, mimetype string) (uploadedBlob, error) { uploadUrl := strings.ReplaceAll(j.session.UploadUrl, "{accountId}", accountId) req, err := http.NewRequest(http.MethodPost, uploadUrl, bytes.NewReader(data)) if err != nil { return uploadedBlob{}, err } req.Header.Add("Content-Type", mimetype) req.SetBasicAuth(j.username, j.password) res, err := j.h.Do(req) if err != nil { return uploadedBlob{}, err } defer res.Body.Close() var response []byte = nil if j.trace { if b, err := httputil.DumpResponse(res, false); err == nil { response, err = io.ReadAll(res.Body) if err != nil { return uploadedBlob{}, err } p := pretty.Pretty(response) if j.color { p = pretty.Color(p, nil) } log.Printf("<== %s%s\n", b, p) } } if res.StatusCode < 200 || res.StatusCode > 299 { return uploadedBlob{}, fmt.Errorf("blob uploading to '%v': status is %s", uploadUrl, res.Status) } if response == nil { response, err = io.ReadAll(res.Body) if err != nil { return uploadedBlob{}, err } } var result uploadedBlob err = json.Unmarshal(response, &result) if err != nil { return uploadedBlob{}, err } return result, nil } func (j *TestJmapClient) command(body map[string]any) ([]any, error) { payload, err := json.Marshal(body) if err != nil { return nil, err } req, err := http.NewRequest(http.MethodPost, j.u.String(), bytes.NewReader(payload)) if err != nil { return nil, err } if j.trace { if b, err := httputil.DumpRequestOut(req, false); err == nil { p := pretty.Pretty(payload) if j.color { p = pretty.Color(p, nil) } log.Printf("==> %s%s\n", b, p) } } req.SetBasicAuth(j.username, j.password) resp, err := j.h.Do(req) if err != nil { return nil, err } defer resp.Body.Close() var response []byte = nil if j.trace { if b, err := httputil.DumpResponse(resp, false); err == nil { response, err = io.ReadAll(resp.Body) if err != nil { return nil, err } p := pretty.Pretty(response) if j.color { p = pretty.Color(p, nil) } log.Printf("<== %s%s\n", b, p) } } if resp.StatusCode >= 300 { return nil, fmt.Errorf("JMAP command HTTP response status is %s", resp.Status) } if response == nil { response, err = io.ReadAll(resp.Body) if err != nil { return nil, err } } r := map[string]any{} err = json.Unmarshal(response, &r) if err != nil { return nil, err } return r["methodResponses"].([]any), nil } type Commander[T any] struct { j *TestJmapClient closure func([]any) (T, error) } func newCommander[T any](j *TestJmapClient, closure func([]any) (T, error)) Commander[T] { return Commander[T]{j: j, closure: closure} } func (c Commander[T]) command(body map[string]any) (T, error) { var zero T methodResponses, err := c.j.command(body) if err != nil { return zero, err } return c.closure(methodResponses) } func (j *TestJmapClient) create(id string, objectType ObjectType, body map[string]any) (string, error) { return newCommander(j, func(methodResponses []any) (string, error) { z := methodResponses[0].([]any) f := z[1].(map[string]any) if x, ok := f["created"]; ok { created := x.(map[string]any) if c, ok := created[id].(map[string]any); ok { return c["id"].(string), nil } else { return "", fmt.Errorf("failed to create %v", objectType) } } else { if ncx, ok := f["notCreated"]; ok { nc := ncx.(map[string]any) c := nc[id].(map[string]any) return "", fmt.Errorf("failed to create %v: %v", objectType, c["description"]) } else { return "", fmt.Errorf("failed to create %v", objectType) } } }).command(body) } func (j *TestJmapClient) objectsById(accountId string, objectType ObjectType, scope string) (map[string]map[string]any, error) { m := map[string]map[string]any{} { body := map[string]any{ "using": []string{JmapCore, scope}, "methodCalls": []any{ []any{ objectType + "/get", map[string]any{ "accountId": accountId, }, "0", }, }, } result, err := newCommander(j, func(methodResponses []any) ([]any, error) { z := methodResponses[0].([]any) f := z[1].(map[string]any) if list, ok := f["list"]; ok { return list.([]any), nil } else { return nil, fmt.Errorf("methodResponse[1] has no 'list' attribute: %v", f) } }).command(body) if err != nil { return nil, err } for _, a := range result { obj := a.(map[string]any) id := obj["id"].(string) m[id] = obj } } return m, nil } func createName(person *gofakeit.PersonInfo, u bool) (map[string]any, jscontact.Name) { o := jscontact.Name{ Type: jscontact.NameType, } m := map[string]any{ "@type": "Name", } mComps := make([]map[string]string, 2) oComps := make([]jscontact.NameComponent, 2) mComps[0] = map[string]string{ "kind": "given", "value": person.FirstName, } oComps[0] = untype(jscontact.NameComponent{ Type: jscontact.NameComponentType, Kind: jscontact.NameComponentKindGiven, Value: person.FirstName, }, u) mComps[1] = map[string]string{ "kind": "surname", "value": person.LastName, } oComps[1] = untype(jscontact.NameComponent{ Type: jscontact.NameComponentType, Kind: jscontact.NameComponentKindSurname, Value: person.LastName, }, u) m["components"] = mComps o.Components = oComps m["isOrdered"] = true o.IsOrdered = true m["defaultSeparator"] = " " o.DefaultSeparator = " " full := fmt.Sprintf("%s %s", person.FirstName, person.LastName) m["full"] = full o.Full = full return m, untype(o, u) } func createNickName(_ *gofakeit.PersonInfo, u bool) (map[string]any, jscontact.Nickname) { name := gofakeit.PetName() contexts := pickRandoms(jscontact.NicknameContextPrivate, jscontact.NicknameContextWork) return map[string]any{ "@type": "Nickname", "name": name, "contexts": toBoolMap(structs.Map(contexts, func(s jscontact.NicknameContext) string { return string(s) })), }, untype(jscontact.Nickname{ Type: jscontact.NicknameType, Name: name, Contexts: orNilMap(toBoolMap(contexts)), }, u) } func createEmail(person *gofakeit.PersonInfo, pref int, u bool) (map[string]any, jscontact.EmailAddress) { email := person.Contact.Email contexts := pickRandoms1(jscontact.EmailAddressContextWork, jscontact.EmailAddressContextPrivate) label := strings.ToLower(person.FirstName) return map[string]any{ "@type": "EmailAddress", "address": email, "contexts": toBoolMap(structs.Map(contexts, func(s jscontact.EmailAddressContext) string { return string(s) })), "label": label, "pref": pref, }, untype(jscontact.EmailAddress{ Type: jscontact.EmailAddressType, Address: email, Contexts: orNilMap(toBoolMap(contexts)), Label: label, Pref: uint(pref), }, u) } func createSecondaryEmail(email string, pref int, u bool) (map[string]any, jscontact.EmailAddress) { contexts := pickRandoms(jscontact.EmailAddressContextWork, jscontact.EmailAddressContextPrivate) return map[string]any{ "@type": "EmailAddress", "address": email, "contexts": toBoolMap(structs.Map(contexts, func(s jscontact.EmailAddressContext) string { return string(s) })), "pref": pref, }, untype(jscontact.EmailAddress{ Type: jscontact.EmailAddressType, Address: email, Contexts: orNilMap(toBoolMap(contexts)), Pref: uint(pref), }, u) } var idFirstLetters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") var idOtherLetters = append(idFirstLetters, []rune("0123456789")...) func id() string { n := 4 + rand.Intn(12-4+1) b := make([]rune, n) b[0] = idFirstLetters[rand.Intn(len(idFirstLetters))] for i := 1; i < n; i++ { b[i] = idOtherLetters[rand.Intn(len(idOtherLetters))] } return string(b) } var timezones = []string{ "America/Adak", "America/Anchorage", "America/Chicago", "America/Denver", "America/Detroit", "America/Indiana/Knox", "America/Kentucky/Louisville", "America/Los_Angeles", "America/New_York", } func propmap[T any](enabled bool, min int, max int, container map[string]any, name string, cardProperty *map[string]T, generator func(int, string) (map[string]any, T, error)) error { if !enabled { return nil } n := min + rand.Intn(max-min+1) m := make(map[string]map[string]any, n) o := make(map[string]T, n) for i := range n { id := id() itemForMap, itemForCard, err := generator(i, id) if err != nil { return err } if itemForMap != nil { m[id] = itemForMap o[id] = itemForCard } } if len(m) > 0 { container[name] = m *cardProperty = o } return nil } func picsum(w, h int) string { return fmt.Sprintf("https://picsum.photos/id/%d/%d/%d", 1+rand.Intn(200), h, w) } func orNilMap[K comparable, V any](m map[K]V) map[K]V { if len(m) < 1 { return nil } else { return m } } func orNilSlice[E any](s []E) []E { if len(s) < 1 { return nil } else { return s } } func toBoolMap[K comparable](s []K) map[K]bool { m := make(map[K]bool, len(s)) for _, e := range s { m[e] = true } return m } func toBoolMapS[K comparable](s ...K) map[K]bool { m := make(map[K]bool, len(s)) for _, e := range s { m[e] = true } return m } func pickRandom[T any](s ...T) T { return s[rand.Intn(len(s))] } func pickRandoms[T any](s ...T) []T { n := rand.Intn(len(s)) if n == 0 { return []T{} } result := make([]T, n) o := make([]T, len(s)) copy(o, s) for i := range n { p := rand.Intn(len(o)) result[i] = slices.Delete(o, p, p)[0] } return result } func pickRandoms1[T any](s ...T) []T { n := 1 + rand.Intn(len(s)-1) result := make([]T, n) o := make([]T, len(s)) copy(o, s) for i := range n { p := rand.Intn(len(o)) result[i] = slices.Delete(o, p, p)[0] } return result } func pickLanguage() string { return pickRandom("en-US", "en-GB", "en-AU") } func allTrue[S any](t *testing.T, s S, exceptions ...string) { v := reflect.ValueOf(s) typ := v.Type() for i := range v.NumField() { name := typ.Field(i).Name if slices.Contains(exceptions, name) { continue } value := v.Field(i).Bool() require.True(t, value, "should be true: %v", name) } }