Merge branch 'origin/main' into 'next-release/main'

.woodpecker.star

@@ -8,7 +8,7 @@ docker_repo_slug = "opencloudeu/opencloud"
 
 # images
 ALPINE_GIT = "alpine/git:latest"
-APACHE_TIKA = "apache/tika:3.2.3.0-full"
+APACHE_TIKA = "apache/tika:4.0.0-full"
 CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1"
 CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:14.1.3"
 COLLABORA_CODE = "collabora/code:24.04.5.1.1"
@@ -3442,6 +3442,9 @@ def tikaService():
     return [{
         "name": "tika",
         "image": APACHE_TIKA,
+        # tika 4 discovers its plugins relative to the image working directory,
+        # the workspace default would leave the pipes fetchers empty
+        "directory": "/opt/tika-server",
         "detach": True,
     }, {
         "name": "wait-for-tika-service",

services/search/README.md

@@ -99,7 +99,7 @@ It does not do any further content analysis.
 
 The main difference is that this extractor is able to analyze and extract data from more advanced file types like PDF, DOCX, PPTX, etc.
 However, [Apache Tika](https://tika.apache.org/) is required for this task.
-Read the [Getting Started with Apache Tika](https://tika.apache.org/2.6.0/gettingstarted.html) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika).
+Read the [Getting Started with Apache Tika](https://tika.apache.org/) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika).
 See the [Tika container usage document](https://github.com/apache/tika-docker#usage) for a quickstart.
 
 As soon as Tika is installed and configured, the search service needs to be told to use it.

services/search/pkg/content/extractor.go

@@ -23,7 +23,7 @@ func getFirstValue(m map[string][]string, key string) (string, error) {
 		return "", fmt.Errorf("unknown key: %v", key)
 	}
 
-	if len(m) == 0 {
+	if len(v) == 0 {
 		return "", fmt.Errorf("no values for: %v", key)
 	}
 

services/search/pkg/content/tika.go

@@ -3,15 +3,13 @@ package content
 import (
 	"context"
 	"fmt"
-	"math"
-	"strconv"
+	"io"
+	"net/http"
 	"strings"
-	"time"
 
 	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	"github.com/google/go-tika/tika"
-	libregraph "github.com/opencloud-eu/libre-graph-api-go"
 	"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
 
 	"github.com/opencloud-eu/opencloud/pkg/log"
@@ -24,6 +22,7 @@ type Tika struct {
 	*Basic
 	Retriever
 	tika                       *tika.Client
+	tikaURL                    string
 	ContentExtractionSizeLimit uint64
 	CleanStopWords             bool
 }
@@ -46,6 +45,7 @@ func NewTikaExtractor(gatewaySelector pool.Selectable[gateway.GatewayAPIClient],
 		Basic:                      basic,
 		Retriever:                  newCS3Retriever(gatewaySelector, logger, cfg.Extractor.CS3AllowInsecure),
 		tika:                       tika.NewClient(nil, cfg.Extractor.Tika.TikaURL),
+		tikaURL:                    cfg.Extractor.Tika.TikaURL,
 		ContentExtractionSizeLimit: cfg.ContentExtractionSizeLimit,
 		CleanStopWords:             cfg.Extractor.Tika.CleanStopWords,
 	}, nil
@@ -91,218 +91,54 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
 			doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title))
 		}
 
-		if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
+		// tika 4 renamed the meta prefix from X-TIKA: to tk:
+		if content, err := getFirstValue(meta, "tk:content"); err == nil {
+			doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
+		} else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
 			doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
 		}
 
-		doc.Location = t.getLocation(meta)
-		doc.Image = t.getImage(meta)
-		doc.Photo = t.getPhoto(meta)
-
-		if contentType, err := getFirstValue(meta, "Content-Type"); err == nil && strings.HasPrefix(contentType, "audio/") {
-			doc.Audio = t.getAudio(meta)
+		// keep facets from earlier entries, an embedded resource's meta
+		// (e.g. cover art) must not reset them
+		if v := t.getLocation(meta); v != nil {
+			doc.Location = v
+		}
+		if v := t.getImage(meta); v != nil {
+			doc.Image = v
+		}
+		if v := t.getPhoto(meta); v != nil {
+			doc.Photo = v
+		}
+		if v := t.getAudio(meta); v != nil {
+			doc.Audio = v
 		}
 	}
 
-	if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords {
+	if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords {
 		doc.Content = CleanString(doc.Content, langCode)
 	}
 
 	return doc, nil
 }
 
-func (t Tika) getImage(meta map[string][]string) *libregraph.Image {
-	var image *libregraph.Image
-	initImage := func() {
-		if image == nil {
-			image = libregraph.NewImage()
+// detectLanguage asks tika for the language of content. Tika 4 moved the
+// endpoint from /language/string to /language, so try the new path first and
+// fall back for an older tika.
+func (t Tika) detectLanguage(ctx context.Context, content string) string {
+	for _, path := range []string{"/language", "/language/string"} {
+		req, err := http.NewRequestWithContext(ctx, http.MethodPut, t.tikaURL+path, strings.NewReader(content))
+		if err != nil {
+			return ""
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return ""
+		}
+		lang, err := io.ReadAll(res.Body)
+		_ = res.Body.Close()
+		if err == nil && res.StatusCode == http.StatusOK && len(lang) > 0 {
+			return string(lang)
 		}
 	}
-
-	if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initImage()
-			image.SetWidth(int32(i))
-		}
-	}
-
-	if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initImage()
-			image.SetHeight(int32(i))
-		}
-	}
-
-	return image
-}
-
-func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates {
-	var location *libregraph.GeoCoordinates
-	initLocation := func() {
-		if location == nil {
-			location = libregraph.NewGeoCoordinates()
-		}
-	}
-
-	// TODO: location.Altitute: transform the following data to … feet above sea level.
-	// "GPS:GPS Altitude":                          []string{"227.4 metres"},
-	// "GPS:GPS Altitude Ref":                      []string{"Sea level"},
-
-	if v, err := getFirstValue(meta, "geo:lat"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initLocation()
-			location.SetLatitude(i)
-		}
-	}
-
-	if v, err := getFirstValue(meta, "geo:long"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initLocation()
-			location.SetLongitude(i)
-		}
-	}
-
-	return location
-}
-
-func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo {
-	var photo *libregraph.Photo
-	initPhoto := func() {
-		if photo == nil {
-			photo = libregraph.NewPhoto()
-		}
-	}
-
-	if v, err := getFirstValue(meta, "tiff:Make"); err == nil {
-		initPhoto()
-		photo.SetCameraMake(v)
-	}
-
-	if v, err := getFirstValue(meta, "tiff:Model"); err == nil {
-		initPhoto()
-		photo.SetCameraModel(v)
-	}
-
-	if v, err := getFirstValue(meta, "exif:FNumber"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initPhoto()
-			photo.SetFNumber(i)
-		}
-	}
-
-	if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initPhoto()
-			photo.SetFocalLength(i)
-		}
-	}
-
-	if v, err := getFirstValue(meta, "Base ISO"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initPhoto()
-			photo.SetIso(int32(i))
-		}
-	}
-
-	if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initPhoto()
-			photo.SetOrientation(int32(i))
-		}
-	}
-
-	if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil {
-		layout := "2006-01-02T15:04:05"
-		if t, err := time.Parse(layout, v); err == nil {
-			initPhoto()
-			photo.SetTakenDateTime(t)
-		}
-	}
-
-	if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initPhoto()
-			photo.SetExposureNumerator(1)
-			photo.SetExposureDenominator(math.Round(1 / i))
-		}
-	}
-
-	return photo
-}
-
-func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio {
-	var audio *libregraph.Audio
-	initAudio := func() {
-		if audio == nil {
-			audio = libregraph.NewAudio()
-		}
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:album"); err == nil {
-		initAudio()
-		audio.SetAlbum(v)
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil {
-		initAudio()
-		audio.SetAlbumArtist(v)
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil {
-		initAudio()
-		audio.SetArtist(v)
-	}
-
-	// TODO: audio.Bitrate: not provided by tika
-	// TODO: audio.Composers: not provided by tika
-	// TODO: audio.Copyright: not provided by tika for audio files?
-
-	if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil {
-		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
-			initAudio()
-			audio.SetDisc(int32(i))
-		}
-
-	}
-
-	//  TODO: audio.DiscCount: not provided by tika
-
-	if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil {
-		// Tika emits fractional seconds.
-		if f, err := strconv.ParseFloat(v, 64); err == nil {
-			initAudio()
-			audio.SetDuration(int64(math.Round(f * 1000)))
-		}
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil {
-		initAudio()
-		audio.SetGenre(v)
-	}
-
-	// TODO: audio.HasDrm: not provided by tika
-	// TODO: audio.IsVariableBitrate: not provided by tika
-
-	if v, err := getFirstValue(meta, "dc:title"); err == nil {
-		initAudio()
-		audio.SetTitle(v)
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil {
-		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
-			initAudio()
-			audio.SetTrack(int32(i))
-		}
-	}
-
-	// TODO: audio.TrackCount: not provided by tika
-
-	if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil {
-		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
-			initAudio()
-			audio.SetYear(int32(i))
-		}
-	}
-
-	return audio
+	return ""
 }

services/search/pkg/content/tika_audio.go

@@ -0,0 +1,130 @@
+package content
+
+import (
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio {
+	// generic keys like dc:title show up on any document, only audio
+	// content types carry the audio facet
+	if v, err := getFirstValue(meta, "Content-Type"); err != nil || !strings.HasPrefix(v, "audio/") {
+		return nil
+	}
+
+	var audio *libregraph.Audio
+	initAudio := func() {
+		if audio == nil {
+			audio = libregraph.NewAudio()
+		}
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:album"); err == nil {
+		initAudio()
+		audio.SetAlbum(v)
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil {
+		initAudio()
+		audio.SetAlbumArtist(v)
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil {
+		initAudio()
+		audio.SetArtist(v)
+	}
+
+	if v, err := getFirstValue(meta, "audio:bitrate"); err == nil {
+		// tika emits bits per second, graph wants kbps
+		if bps, err := strconv.ParseInt(v, 10, 64); err == nil && bps > 0 {
+			initAudio()
+			audio.SetBitrate(int64(math.Round(float64(bps) / 1000)))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:composer"); err == nil {
+		initAudio()
+		audio.SetComposers(v)
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:copyright"); err == nil {
+		initAudio()
+		audio.SetCopyright(v)
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil {
+		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
+			initAudio()
+			audio.SetDisc(int32(i))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "audio:disc-count"); err == nil {
+		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
+			initAudio()
+			audio.SetDiscCount(int32(i))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil {
+		// Tika emits fractional seconds.
+		if f, err := strconv.ParseFloat(v, 64); err == nil {
+			initAudio()
+			audio.SetDuration(int64(math.Round(f * 1000)))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil {
+		initAudio()
+		audio.SetGenre(v)
+	}
+
+	if v, err := getFirstValue(meta, "audio:has-drm"); err == nil {
+		if b, err := strconv.ParseBool(v); err == nil {
+			initAudio()
+			audio.SetHasDrm(b)
+		}
+	}
+
+	if v, err := getFirstValue(meta, "audio:is-variable-bitrate"); err == nil {
+		if b, err := strconv.ParseBool(v); err == nil {
+			initAudio()
+			audio.SetIsVariableBitrate(b)
+		}
+	}
+
+	if v, err := getFirstValue(meta, "dc:title"); err == nil {
+		initAudio()
+		audio.SetTitle(v)
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil {
+		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
+			initAudio()
+			audio.SetTrack(int32(i))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "audio:track-count"); err == nil {
+		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
+			initAudio()
+			audio.SetTrackCount(int32(i))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil {
+		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
+			initAudio()
+			audio.SetYear(int32(i))
+		} else if d, err := time.Parse(time.DateOnly, v); err == nil {
+			initAudio()
+			audio.SetYear(int32(d.Year()))
+		}
+	}
+
+	return audio
+}

services/search/pkg/content/tika_audio_test.go

@@ -0,0 +1,80 @@
+package content
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+var _ = Describe("getAudio", func() {
+	It("maps the audio metadata to the audio facet", func() {
+		meta := map[string][]string{
+			"Content-Type":              {"audio/mpeg"},
+			"xmpDM:genre":               {"Some Genre"},
+			"xmpDM:album":               {"Some Album"},
+			"xmpDM:trackNumber":         {"7"},
+			"xmpDM:discNumber":          {"4"},
+			"xmpDM:releaseDate":         {"2004"},
+			"xmpDM:artist":              {"Some Artist"},
+			"xmpDM:albumArtist":         {"Some AlbumArtist"},
+			"dc:title":                  {"Some Title"},
+			"xmpDM:duration":            {"225.5"},
+			"xmpDM:composer":            {"Some Composers"},
+			"xmpDM:copyright":           {"Some Copyright"},
+			"audio:bitrate":             {"192000"},
+			"audio:is-variable-bitrate": {"true"},
+			"audio:has-drm":             {"false"},
+			"audio:track-count":         {"9"},
+			"audio:disc-count":          {"5"},
+		}
+
+		audio := Tika{}.getAudio(meta)
+		Expect(audio).ToNot(BeNil())
+
+		Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
+		Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
+		Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
+		Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
+		Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers")))
+		Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright")))
+		Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4)))
+		Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5)))
+		Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500)))
+		Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
+		Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false)))
+		Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true)))
+		Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
+		Expect(audio.Track).To(Equal(libregraph.PtrInt32(7)))
+		Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9)))
+		Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
+	})
+
+	It("returns nil when no audio metadata is present", func() {
+		Expect(Tika{}.getAudio(map[string][]string{})).To(BeNil())
+	})
+
+	It("returns nil for non-audio content types", func() {
+		Expect(Tika{}.getAudio(map[string][]string{
+			"Content-Type": {"application/pdf"},
+			"dc:title":     {"quarterly report"},
+		})).To(BeNil())
+	})
+
+	It("takes the year from a full release date", func() {
+		audio := Tika{}.getAudio(map[string][]string{
+			"Content-Type":      {"audio/mpeg"},
+			"xmpDM:releaseDate": {"2004-06-01"},
+		})
+		Expect(audio).ToNot(BeNil())
+		Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
+	})
+
+	It("rounds the bitrate to the nearest kbps", func() {
+		audio := Tika{}.getAudio(map[string][]string{
+			"Content-Type":  {"audio/mpeg"},
+			"audio:bitrate": {"191999"},
+		})
+		Expect(audio).ToNot(BeNil())
+		Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
+	})
+})

services/search/pkg/content/tika_image.go

@@ -0,0 +1,32 @@
+package content
+
+import (
+	"strconv"
+
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+func (t Tika) getImage(meta map[string][]string) *libregraph.Image {
+	var image *libregraph.Image
+	initImage := func() {
+		if image == nil {
+			image = libregraph.NewImage()
+		}
+	}
+
+	if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil {
+		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
+			initImage()
+			image.SetWidth(int32(i))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil {
+		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
+			initImage()
+			image.SetHeight(int32(i))
+		}
+	}
+
+	return image
+}

services/search/pkg/content/tika_image_test.go

@@ -0,0 +1,23 @@
+package content
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+var _ = Describe("getImage", func() {
+	It("maps the image dimensions to the image facet", func() {
+		image := Tika{}.getImage(map[string][]string{
+			"tiff:ImageWidth":  {"100"},
+			"tiff:ImageLength": {"200"},
+		})
+		Expect(image).ToNot(BeNil())
+		Expect(image.Width).To(Equal(libregraph.PtrInt32(100)))
+		Expect(image.Height).To(Equal(libregraph.PtrInt32(200)))
+	})
+
+	It("returns nil when no image metadata is present", func() {
+		Expect(Tika{}.getImage(map[string][]string{})).To(BeNil())
+	})
+})

services/search/pkg/content/tika_location.go

@@ -0,0 +1,52 @@
+package content
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+// graph geoCoordinates.altitude is in feet, exif GPS altitude (geo:alt) in metres.
+const metresToFeet = 3.280839895
+
+func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates {
+	// the facet needs a sane coordinate pair, an altitude alone is useless
+	lat, latErr := parseCoordinate(meta, "geo:lat", 90)
+	long, longErr := parseCoordinate(meta, "geo:long", 180)
+	if latErr != nil || longErr != nil {
+		return nil
+	}
+
+	location := libregraph.NewGeoCoordinates()
+	location.SetLatitude(lat)
+	location.SetLongitude(long)
+
+	// tika emits metres (negative below sea level), graph wants feet
+	if v, err := getFirstValue(meta, "geo:alt"); err == nil {
+		if metres, err := strconv.ParseFloat(v, 64); err == nil {
+			location.SetAltitude(metres * metresToFeet)
+		}
+	}
+
+	return location
+}
+
+func parseCoordinate(meta map[string][]string, key string, limit float64) (float64, error) {
+	v, err := getFirstValue(meta, key)
+	if err != nil {
+		return 0, err
+	}
+
+	f, err := strconv.ParseFloat(v, 64)
+	if err != nil {
+		return 0, err
+	}
+	// ParseFloat accepts "NaN", which json cannot marshal
+	if math.IsNaN(f) || math.Abs(f) > limit {
+		return 0, fmt.Errorf("%s out of range: %v", key, f)
+	}
+
+	return f, nil
+}

services/search/pkg/content/tika_location_test.go

@@ -0,0 +1,48 @@
+package content
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+var _ = Describe("getLocation", func() {
+	It("maps lat/long and converts altitude from metres to feet", func() {
+		metres := 227.4
+		location := Tika{}.getLocation(map[string][]string{
+			"geo:lat":  {"49.48675890884328"},
+			"geo:long": {"11.103870357204285"},
+			"geo:alt":  {"227.4"},
+		})
+		Expect(location).ToNot(BeNil())
+		Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
+		Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
+		Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet)))
+	})
+
+	It("keeps below-sea-level altitude negative", func() {
+		metres := -227.4
+		location := Tika{}.getLocation(map[string][]string{
+			"geo:lat":  {"31.5"},
+			"geo:long": {"35.47"},
+			"geo:alt":  {"-227.4"},
+		})
+		Expect(location).ToNot(BeNil())
+		Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet)))
+	})
+
+	It("returns nil for an incomplete coordinate pair", func() {
+		Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}})).To(BeNil())
+		Expect(Tika{}.getLocation(map[string][]string{"geo:alt": {"227.4"}})).To(BeNil())
+	})
+
+	It("returns nil for out-of-range coordinates", func() {
+		Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"91"}, "geo:long": {"11.1"}})).To(BeNil())
+		Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}, "geo:long": {"-180.5"}})).To(BeNil())
+		Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"NaN"}, "geo:long": {"11.1"}})).To(BeNil())
+	})
+
+	It("returns nil when no location metadata is present", func() {
+		Expect(Tika{}.getLocation(map[string][]string{})).To(BeNil())
+	})
+})

services/search/pkg/content/tika_photo.go

@@ -0,0 +1,74 @@
+package content
+
+import (
+	"math"
+	"strconv"
+	"time"
+
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo {
+	var photo *libregraph.Photo
+	initPhoto := func() {
+		if photo == nil {
+			photo = libregraph.NewPhoto()
+		}
+	}
+
+	if v, err := getFirstValue(meta, "tiff:Make"); err == nil {
+		initPhoto()
+		photo.SetCameraMake(v)
+	}
+
+	if v, err := getFirstValue(meta, "tiff:Model"); err == nil {
+		initPhoto()
+		photo.SetCameraModel(v)
+	}
+
+	if v, err := getFirstValue(meta, "exif:FNumber"); err == nil {
+		if i, err := strconv.ParseFloat(v, 64); err == nil {
+			initPhoto()
+			photo.SetFNumber(i)
+		}
+	}
+
+	if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil {
+		if i, err := strconv.ParseFloat(v, 64); err == nil {
+			initPhoto()
+			photo.SetFocalLength(i)
+		}
+	}
+
+	if v, err := getFirstValue(meta, "Base ISO"); err == nil {
+		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
+			initPhoto()
+			photo.SetIso(int32(i))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil {
+		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
+			initPhoto()
+			photo.SetOrientation(int32(i))
+		}
+	}
+
+	if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil {
+		layout := "2006-01-02T15:04:05"
+		if t, err := time.Parse(layout, v); err == nil {
+			initPhoto()
+			photo.SetTakenDateTime(t)
+		}
+	}
+
+	if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil {
+		if i, err := strconv.ParseFloat(v, 64); err == nil && i > 0 {
+			initPhoto()
+			photo.SetExposureNumerator(1)
+			photo.SetExposureDenominator(math.Round(1 / i))
+		}
+	}
+
+	return photo
+}

services/search/pkg/content/tika_photo_test.go

@@ -0,0 +1,38 @@
+package content
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	libregraph "github.com/opencloud-eu/libre-graph-api-go"
+)
+
+var _ = Describe("getPhoto", func() {
+	It("maps the exif metadata to the photo facet", func() {
+		photo := Tika{}.getPhoto(map[string][]string{
+			"tiff:Make":             {"Canon"},
+			"tiff:Model":            {"Canon EOS 5D"},
+			"exif:ExposureTime":     {"0.001"},
+			"exif:FNumber":          {"1.8"},
+			"exif:FocalLength":      {"50"},
+			"Base ISO":              {"100"},
+			"tiff:Orientation":      {"1"},
+			"exif:DateTimeOriginal": {"2018-01-01T12:34:56"},
+		})
+		Expect(photo).ToNot(BeNil())
+		Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon")))
+		Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D")))
+		Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1)))
+		Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000)))
+		Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8)))
+		Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50)))
+		Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100)))
+		Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1)))
+		Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC))))
+	})
+
+	It("returns nil when no photo metadata is present", func() {
+		Expect(Tika{}.getPhoto(map[string][]string{})).To(BeNil())
+	})
+})

services/search/pkg/content/tika_test.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
-	"time"
 
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	. "github.com/onsi/ginkgo/v2"
@@ -24,17 +23,19 @@ import (
 var _ = Describe("Tika", func() {
 	Describe("extract", func() {
 		var (
-			body         string
-			fullResponse string
-			language     string
-			version      string
-			srv          *httptest.Server
-			tika         *content.Tika
+			body          string
+			fullResponse  string
+			language      string
+			tika4Language bool
+			version       string
+			srv           *httptest.Server
+			tika          *content.Tika
 		)
 
 		BeforeEach(func() {
 			body = ""
 			language = ""
+			tika4Language = false
 			version = ""
 			fullResponse = ""
 			srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
@@ -42,6 +43,13 @@ var _ = Describe("Tika", func() {
 				switch req.URL.Path {
 				case "/version":
 					out = version
+				case "/language":
+					if tika4Language {
+						out = language
+					} else {
+						w.WriteHeader(http.StatusNotFound)
+						return
+					}
 				case "/language/string":
 					out = language
 				case "/rmeta/text":
@@ -102,6 +110,17 @@ var _ = Describe("Tika", func() {
 			Expect(doc.Title).To(Equal("quarterly report"))
 		})
 
+		It("adds the content of a tika 4", func() {
+			fullResponse = `[{"tk:content": "some data"}]`
+
+			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
+				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
+				Size: 1,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.Content).To(Equal("some data"))
+		})
+
 		It("adds the title of an older tika", func() {
 			fullResponse = `[{"title": "quarterly report"}]`
 
@@ -113,129 +132,17 @@ var _ = Describe("Tika", func() {
 			Expect(doc.Title).To(Equal("quarterly report"))
 		})
 
-		It("adds audio content", func() {
-			fullResponse = `[
-				{
-					"xmpDM:genre": "Some Genre",
-					"xmpDM:album": "Some Album",
-					"xmpDM:trackNumber": "7",
-					"xmpDM:discNumber": "4",
-					"xmpDM:releaseDate": "2004",
-					"xmpDM:artist": "Some Artist",
-					"xmpDM:albumArtist": "Some AlbumArtist",
-					"xmpDM:audioCompressor": "MP3",
-					"xmpDM:audioChannelType": "Stereo",
-					"version": "MPEG 3 Layer III Version 1",
-					"xmpDM:logComment": "some comment",
-					"xmpDM:audioSampleRate": "44100",
-					"channels": "2",
-					"dc:title": "Some Title",
-					"xmpDM:duration": "225.5",
-					"Content-Type": "audio/mpeg",
-					"samplerate": "44100"
-				}
-			]`
+		It("removes stop words with a tika 4 language endpoint", func() {
+			body = "body to test stop words!!! against almost everyone"
+			language = "en"
+			tika4Language = true
+
 			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
 				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
 				Size: 1,
 			})
 			Expect(err).ToNot(HaveOccurred())
-
-			audio := doc.Audio
-			Expect(audio).ToNot(BeNil())
-
-			Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
-			Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
-			Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
-			// Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
-			// Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers")))
-			// Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright")))
-			Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4)))
-			// Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5)))
-			Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500)))
-			Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
-			// Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false)))
-			// Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true)))
-			Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
-			Expect(audio.Track).To(Equal(libregraph.PtrInt32(7)))
-			// Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9)))
-			Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
-
-		})
-
-		It("adds location content", func() {
-			fullResponse = `[
-				{
-					"geo:lat": "49.48675890884328",
-					"geo:long": "11.103870357204285"
-				}
-			]`
-			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
-				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
-				Size: 1,
-			})
-			Expect(err).ToNot(HaveOccurred())
-
-			location := doc.Location
-			Expect(location).ToNot(BeNil())
-
-			// TODO: Altitude is not supported right now
-			Expect(location.Altitude).To(BeNil())
-			Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
-			Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
-		})
-
-		It("adds image content", func() {
-			fullResponse = `[
-				{
-					"tiff:ImageWidth": "100",
-					"tiff:ImageLength": "100"
-				}
-			]`
-			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
-				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
-				Size: 1,
-			})
-			Expect(err).ToNot(HaveOccurred())
-
-			image := doc.Image
-			Expect(image).ToNot(BeNil())
-
-			Expect(image.Width).To(Equal(libregraph.PtrInt32(100)))
-			Expect(image.Height).To(Equal(libregraph.PtrInt32(100)))
-		})
-
-		It("adds photo content", func() {
-			fullResponse = `[
-				{
-					"tiff:Make": "Canon",
-					"tiff:Model": "Canon EOS 5D",
-					"exif:ExposureTime": "0.001",
-					"exif:FNumber": "1.8",
-					"exif:FocalLength": "50",
-					"Base ISO": "100",
-					"tiff:Orientation": "1",
-					"exif:DateTimeOriginal": "2018-01-01T12:34:56"
-				}
-			]`
-			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
-				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
-				Size: 1,
-			})
-			Expect(err).ToNot(HaveOccurred())
-
-			photo := doc.Photo
-			Expect(photo).ToNot(BeNil())
-
-			Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon")))
-			Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D")))
-			Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1)))
-			Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000)))
-			Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8)))
-			Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50)))
-			Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100)))
-			Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1)))
-			Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC))))
+			Expect(doc.Content).To(Equal("body test stop words!!!"))
 		})
 
 		It("removes stop words", func() {
@@ -250,6 +157,53 @@ var _ = Describe("Tika", func() {
 			Expect(doc.Content).To(Equal("body test stop words!!!"))
 		})
 
+		It("keeps the audio facet when an embedded resource follows", func() {
+			fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]`
+
+			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
+				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
+				Size: 1,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.Audio).ToNot(BeNil())
+			Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker")))
+			Expect(doc.Image).ToNot(BeNil())
+		})
+
+		It("adds no audio facet to non-audio documents", func() {
+			fullResponse = `[{"Content-Type": "application/pdf", "dc:title": "quarterly report"}]`
+
+			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
+				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
+				Size: 1,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.Audio).To(BeNil())
+			Expect(doc.Title).To(Equal("quarterly report"))
+		})
+
+		It("prefers the tika 4 content key over the legacy one", func() {
+			fullResponse = `[{"tk:content": "new", "X-TIKA:content": "old"}]`
+
+			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
+				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
+				Size: 1,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.Content).To(Equal("new"))
+		})
+
+		It("joins the content of all meta entries", func() {
+			fullResponse = `[{"tk:content": "one"}, {"X-TIKA:content": "two"}]`
+
+			doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
+				Type: provider.ResourceType_RESOURCE_TYPE_FILE,
+				Size: 1,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.Content).To(Equal("one two"))
+		})
+
 		It("keeps stop words", func() {
 			body = "body to test stop words!!! against almost everyone"
 			language = "en"

tests/acceptance/docker/src/tika.yml

@@ -5,4 +5,4 @@ services:
       - tika
     command: tika:9998
   tika:
-    image: apache/tika:3.2.3.0-full
+    image: apache/tika:4.0.0-full