From 7e0f536b833a9b8e52d3df355fa3c4063487af94 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Thu, 30 Jul 2026 00:07:23 +0200 Subject: [PATCH 1/8] refactor(search): split the tika extractor and its tests into per-facet files --- services/search/pkg/content/tika.go | 200 ------------------ services/search/pkg/content/tika_audio.go | 85 ++++++++ .../search/pkg/content/tika_audio_test.go | 40 ++++ services/search/pkg/content/tika_image.go | 32 +++ .../search/pkg/content/tika_image_test.go | 23 ++ services/search/pkg/content/tika_location.go | 36 ++++ .../search/pkg/content/tika_location_test.go | 23 ++ services/search/pkg/content/tika_photo.go | 74 +++++++ .../search/pkg/content/tika_photo_test.go | 38 ++++ services/search/pkg/content/tika_test.go | 128 ----------- 10 files changed, 351 insertions(+), 328 deletions(-) create mode 100644 services/search/pkg/content/tika_audio.go create mode 100644 services/search/pkg/content/tika_audio_test.go create mode 100644 services/search/pkg/content/tika_image.go create mode 100644 services/search/pkg/content/tika_image_test.go create mode 100644 services/search/pkg/content/tika_location.go create mode 100644 services/search/pkg/content/tika_location_test.go create mode 100644 services/search/pkg/content/tika_photo.go create mode 100644 services/search/pkg/content/tika_photo_test.go diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 2b475efb17..d9e45c41cc 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -3,15 +3,11 @@ package content import ( "context" "fmt" - "math" - "strconv" "strings" - "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/google/go-tika/tika" - libregraph "github.com/opencloud-eu/libre-graph-api-go" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/opencloud/pkg/log" @@ -110,199 +106,3 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, return doc, nil } - -func (t Tika) getImage(meta map[string][]string) *libregraph.Image { - var image *libregraph.Image - initImage := func() { - if image == nil { - image = libregraph.NewImage() - } - } - - if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initImage() - image.SetWidth(int32(i)) - } - } - - if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initImage() - image.SetHeight(int32(i)) - } - } - - return image -} - -func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { - var location *libregraph.GeoCoordinates - initLocation := func() { - if location == nil { - location = libregraph.NewGeoCoordinates() - } - } - - // TODO: location.Altitute: transform the following data to … feet above sea level. - // "GPS:GPS Altitude": []string{"227.4 metres"}, - // "GPS:GPS Altitude Ref": []string{"Sea level"}, - - if v, err := getFirstValue(meta, "geo:lat"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLatitude(i) - } - } - - if v, err := getFirstValue(meta, "geo:long"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLongitude(i) - } - } - - return location -} - -func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo { - var photo *libregraph.Photo - initPhoto := func() { - if photo == nil { - photo = libregraph.NewPhoto() - } - } - - if v, err := getFirstValue(meta, "tiff:Make"); err == nil { - initPhoto() - photo.SetCameraMake(v) - } - - if v, err := getFirstValue(meta, "tiff:Model"); err == nil { - initPhoto() - photo.SetCameraModel(v) - } - - if v, err := getFirstValue(meta, "exif:FNumber"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetFNumber(i) - } - } - - if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetFocalLength(i) - } - } - - if v, err := getFirstValue(meta, "Base ISO"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initPhoto() - photo.SetIso(int32(i)) - } - } - - if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initPhoto() - photo.SetOrientation(int32(i)) - } - } - - if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil { - layout := "2006-01-02T15:04:05" - if t, err := time.Parse(layout, v); err == nil { - initPhoto() - photo.SetTakenDateTime(t) - } - } - - if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetExposureNumerator(1) - photo.SetExposureDenominator(math.Round(1 / i)) - } - } - - return photo -} - -func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { - var audio *libregraph.Audio - initAudio := func() { - if audio == nil { - audio = libregraph.NewAudio() - } - } - - if v, err := getFirstValue(meta, "xmpDM:album"); err == nil { - initAudio() - audio.SetAlbum(v) - } - - if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil { - initAudio() - audio.SetAlbumArtist(v) - } - - if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil { - initAudio() - audio.SetArtist(v) - } - - // TODO: audio.Bitrate: not provided by tika - // TODO: audio.Composers: not provided by tika - // TODO: audio.Copyright: not provided by tika for audio files? - - if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetDisc(int32(i)) - } - - } - - // TODO: audio.DiscCount: not provided by tika - - if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil { - // Tika emits fractional seconds. - if f, err := strconv.ParseFloat(v, 64); err == nil { - initAudio() - audio.SetDuration(int64(math.Round(f * 1000))) - } - } - - if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil { - initAudio() - audio.SetGenre(v) - } - - // TODO: audio.HasDrm: not provided by tika - // TODO: audio.IsVariableBitrate: not provided by tika - - if v, err := getFirstValue(meta, "dc:title"); err == nil { - initAudio() - audio.SetTitle(v) - } - - if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetTrack(int32(i)) - } - } - - // TODO: audio.TrackCount: not provided by tika - - if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetYear(int32(i)) - } - } - - return audio -} diff --git a/services/search/pkg/content/tika_audio.go b/services/search/pkg/content/tika_audio.go new file mode 100644 index 0000000000..7a89e3e225 --- /dev/null +++ b/services/search/pkg/content/tika_audio.go @@ -0,0 +1,85 @@ +package content + +import ( + "math" + "strconv" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { + var audio *libregraph.Audio + initAudio := func() { + if audio == nil { + audio = libregraph.NewAudio() + } + } + + if v, err := getFirstValue(meta, "xmpDM:album"); err == nil { + initAudio() + audio.SetAlbum(v) + } + + if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil { + initAudio() + audio.SetAlbumArtist(v) + } + + if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil { + initAudio() + audio.SetArtist(v) + } + + // TODO: audio.Bitrate: not provided by tika + // TODO: audio.Composers: not provided by tika + // TODO: audio.Copyright: not provided by tika for audio files? + + if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetDisc(int32(i)) + } + + } + + // TODO: audio.DiscCount: not provided by tika + + if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil { + // Tika emits fractional seconds. + if f, err := strconv.ParseFloat(v, 64); err == nil { + initAudio() + audio.SetDuration(int64(math.Round(f * 1000))) + } + } + + if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil { + initAudio() + audio.SetGenre(v) + } + + // TODO: audio.HasDrm: not provided by tika + // TODO: audio.IsVariableBitrate: not provided by tika + + if v, err := getFirstValue(meta, "dc:title"); err == nil { + initAudio() + audio.SetTitle(v) + } + + if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetTrack(int32(i)) + } + } + + // TODO: audio.TrackCount: not provided by tika + + if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetYear(int32(i)) + } + } + + return audio +} diff --git a/services/search/pkg/content/tika_audio_test.go b/services/search/pkg/content/tika_audio_test.go new file mode 100644 index 0000000000..7c81f3b6c9 --- /dev/null +++ b/services/search/pkg/content/tika_audio_test.go @@ -0,0 +1,40 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getAudio", func() { + It("maps the audio metadata to the audio facet", func() { + meta := map[string][]string{ + "xmpDM:genre": {"Some Genre"}, + "xmpDM:album": {"Some Album"}, + "xmpDM:trackNumber": {"7"}, + "xmpDM:discNumber": {"4"}, + "xmpDM:releaseDate": {"2004"}, + "xmpDM:artist": {"Some Artist"}, + "xmpDM:albumArtist": {"Some AlbumArtist"}, + "dc:title": {"Some Title"}, + "xmpDM:duration": {"225.5"}, + } + + audio := Tika{}.getAudio(meta) + Expect(audio).ToNot(BeNil()) + + Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) + Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) + Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) + Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4))) + Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500))) + Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) + Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) + Expect(audio.Track).To(Equal(libregraph.PtrInt32(7))) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + + It("returns nil when no audio metadata is present", func() { + Expect(Tika{}.getAudio(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_image.go b/services/search/pkg/content/tika_image.go new file mode 100644 index 0000000000..b5ce2a4037 --- /dev/null +++ b/services/search/pkg/content/tika_image.go @@ -0,0 +1,32 @@ +package content + +import ( + "strconv" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getImage(meta map[string][]string) *libregraph.Image { + var image *libregraph.Image + initImage := func() { + if image == nil { + image = libregraph.NewImage() + } + } + + if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initImage() + image.SetWidth(int32(i)) + } + } + + if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initImage() + image.SetHeight(int32(i)) + } + } + + return image +} diff --git a/services/search/pkg/content/tika_image_test.go b/services/search/pkg/content/tika_image_test.go new file mode 100644 index 0000000000..91456ea97c --- /dev/null +++ b/services/search/pkg/content/tika_image_test.go @@ -0,0 +1,23 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getImage", func() { + It("maps the image dimensions to the image facet", func() { + image := Tika{}.getImage(map[string][]string{ + "tiff:ImageWidth": {"100"}, + "tiff:ImageLength": {"200"}, + }) + Expect(image).ToNot(BeNil()) + Expect(image.Width).To(Equal(libregraph.PtrInt32(100))) + Expect(image.Height).To(Equal(libregraph.PtrInt32(200))) + }) + + It("returns nil when no image metadata is present", func() { + Expect(Tika{}.getImage(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_location.go b/services/search/pkg/content/tika_location.go new file mode 100644 index 0000000000..2334c53afe --- /dev/null +++ b/services/search/pkg/content/tika_location.go @@ -0,0 +1,36 @@ +package content + +import ( + "strconv" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { + var location *libregraph.GeoCoordinates + initLocation := func() { + if location == nil { + location = libregraph.NewGeoCoordinates() + } + } + + // TODO: location.Altitute: transform the following data to … feet above sea level. + // "GPS:GPS Altitude": []string{"227.4 metres"}, + // "GPS:GPS Altitude Ref": []string{"Sea level"}, + + if v, err := getFirstValue(meta, "geo:lat"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initLocation() + location.SetLatitude(i) + } + } + + if v, err := getFirstValue(meta, "geo:long"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initLocation() + location.SetLongitude(i) + } + } + + return location +} diff --git a/services/search/pkg/content/tika_location_test.go b/services/search/pkg/content/tika_location_test.go new file mode 100644 index 0000000000..38bb9ef705 --- /dev/null +++ b/services/search/pkg/content/tika_location_test.go @@ -0,0 +1,23 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getLocation", func() { + It("maps latitude and longitude to the location facet", func() { + location := Tika{}.getLocation(map[string][]string{ + "geo:lat": {"49.48675890884328"}, + "geo:long": {"11.103870357204285"}, + }) + Expect(location).ToNot(BeNil()) + Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) + Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) + }) + + It("returns nil when no location metadata is present", func() { + Expect(Tika{}.getLocation(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_photo.go b/services/search/pkg/content/tika_photo.go new file mode 100644 index 0000000000..a833df3b1d --- /dev/null +++ b/services/search/pkg/content/tika_photo.go @@ -0,0 +1,74 @@ +package content + +import ( + "math" + "strconv" + "time" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo { + var photo *libregraph.Photo + initPhoto := func() { + if photo == nil { + photo = libregraph.NewPhoto() + } + } + + if v, err := getFirstValue(meta, "tiff:Make"); err == nil { + initPhoto() + photo.SetCameraMake(v) + } + + if v, err := getFirstValue(meta, "tiff:Model"); err == nil { + initPhoto() + photo.SetCameraModel(v) + } + + if v, err := getFirstValue(meta, "exif:FNumber"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initPhoto() + photo.SetFNumber(i) + } + } + + if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initPhoto() + photo.SetFocalLength(i) + } + } + + if v, err := getFirstValue(meta, "Base ISO"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initPhoto() + photo.SetIso(int32(i)) + } + } + + if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initPhoto() + photo.SetOrientation(int32(i)) + } + } + + if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil { + layout := "2006-01-02T15:04:05" + if t, err := time.Parse(layout, v); err == nil { + initPhoto() + photo.SetTakenDateTime(t) + } + } + + if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initPhoto() + photo.SetExposureNumerator(1) + photo.SetExposureDenominator(math.Round(1 / i)) + } + } + + return photo +} diff --git a/services/search/pkg/content/tika_photo_test.go b/services/search/pkg/content/tika_photo_test.go new file mode 100644 index 0000000000..9995d59587 --- /dev/null +++ b/services/search/pkg/content/tika_photo_test.go @@ -0,0 +1,38 @@ +package content + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getPhoto", func() { + It("maps the exif metadata to the photo facet", func() { + photo := Tika{}.getPhoto(map[string][]string{ + "tiff:Make": {"Canon"}, + "tiff:Model": {"Canon EOS 5D"}, + "exif:ExposureTime": {"0.001"}, + "exif:FNumber": {"1.8"}, + "exif:FocalLength": {"50"}, + "Base ISO": {"100"}, + "tiff:Orientation": {"1"}, + "exif:DateTimeOriginal": {"2018-01-01T12:34:56"}, + }) + Expect(photo).ToNot(BeNil()) + Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon"))) + Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D"))) + Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1))) + Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000))) + Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8))) + Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50))) + Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100))) + Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1))) + Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC)))) + }) + + It("returns nil when no photo metadata is present", func() { + Expect(Tika{}.getPhoto(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index 286ffa8b9c..2d9a34936f 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -7,12 +7,10 @@ import ( "net/http" "net/http/httptest" "strings" - "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - libregraph "github.com/opencloud-eu/libre-graph-api-go" "github.com/stretchr/testify/mock" "github.com/opencloud-eu/opencloud/pkg/log" @@ -112,132 +110,6 @@ var _ = Describe("Tika", func() { Expect(err).ToNot(HaveOccurred()) Expect(doc.Title).To(Equal("quarterly report")) }) - - It("adds audio content", func() { - fullResponse = `[ - { - "xmpDM:genre": "Some Genre", - "xmpDM:album": "Some Album", - "xmpDM:trackNumber": "7", - "xmpDM:discNumber": "4", - "xmpDM:releaseDate": "2004", - "xmpDM:artist": "Some Artist", - "xmpDM:albumArtist": "Some AlbumArtist", - "xmpDM:audioCompressor": "MP3", - "xmpDM:audioChannelType": "Stereo", - "version": "MPEG 3 Layer III Version 1", - "xmpDM:logComment": "some comment", - "xmpDM:audioSampleRate": "44100", - "channels": "2", - "dc:title": "Some Title", - "xmpDM:duration": "225.5", - "Content-Type": "audio/mpeg", - "samplerate": "44100" - } - ]` - doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ - Type: provider.ResourceType_RESOURCE_TYPE_FILE, - Size: 1, - }) - Expect(err).ToNot(HaveOccurred()) - - audio := doc.Audio - Expect(audio).ToNot(BeNil()) - - Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) - Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) - Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) - // Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) - // Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) - // Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright"))) - Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4))) - // Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) - Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500))) - Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) - // Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) - // Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) - Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) - Expect(audio.Track).To(Equal(libregraph.PtrInt32(7))) - // Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9))) - Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) - - }) - - It("adds location content", func() { - fullResponse = `[ - { - "geo:lat": "49.48675890884328", - "geo:long": "11.103870357204285" - } - ]` - doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ - Type: provider.ResourceType_RESOURCE_TYPE_FILE, - Size: 1, - }) - Expect(err).ToNot(HaveOccurred()) - - location := doc.Location - Expect(location).ToNot(BeNil()) - - // TODO: Altitude is not supported right now - Expect(location.Altitude).To(BeNil()) - Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) - Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) - }) - - It("adds image content", func() { - fullResponse = `[ - { - "tiff:ImageWidth": "100", - "tiff:ImageLength": "100" - } - ]` - doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ - Type: provider.ResourceType_RESOURCE_TYPE_FILE, - Size: 1, - }) - Expect(err).ToNot(HaveOccurred()) - - image := doc.Image - Expect(image).ToNot(BeNil()) - - Expect(image.Width).To(Equal(libregraph.PtrInt32(100))) - Expect(image.Height).To(Equal(libregraph.PtrInt32(100))) - }) - - It("adds photo content", func() { - fullResponse = `[ - { - "tiff:Make": "Canon", - "tiff:Model": "Canon EOS 5D", - "exif:ExposureTime": "0.001", - "exif:FNumber": "1.8", - "exif:FocalLength": "50", - "Base ISO": "100", - "tiff:Orientation": "1", - "exif:DateTimeOriginal": "2018-01-01T12:34:56" - } - ]` - doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ - Type: provider.ResourceType_RESOURCE_TYPE_FILE, - Size: 1, - }) - Expect(err).ToNot(HaveOccurred()) - - photo := doc.Photo - Expect(photo).ToNot(BeNil()) - - Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon"))) - Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D"))) - Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1))) - Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000))) - Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8))) - Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50))) - Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100))) - Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1))) - Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC)))) - }) - It("removes stop words", func() { body = "body to test stop words!!! against almost everyone" language = "en" From 2b4a6fd739227fddcd9d21dd0c7adf972d354cef Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Thu, 30 Jul 2026 00:11:01 +0200 Subject: [PATCH 2/8] feat(search): populate audio and gps-altitude facets from tika, drop the audio-only gate --- services/search/pkg/content/tika.go | 5 +- services/search/pkg/content/tika_audio.go | 49 ++++++++++++++++--- .../search/pkg/content/tika_audio_test.go | 32 ++++++++---- services/search/pkg/content/tika_location.go | 15 ++++-- .../search/pkg/content/tika_location_test.go | 12 ++++- 5 files changed, 88 insertions(+), 25 deletions(-) diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index d9e45c41cc..078152cb5d 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -94,10 +94,7 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, doc.Location = t.getLocation(meta) doc.Image = t.getImage(meta) doc.Photo = t.getPhoto(meta) - - if contentType, err := getFirstValue(meta, "Content-Type"); err == nil && strings.HasPrefix(contentType, "audio/") { - doc.Audio = t.getAudio(meta) - } + doc.Audio = t.getAudio(meta) } if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords { diff --git a/services/search/pkg/content/tika_audio.go b/services/search/pkg/content/tika_audio.go index 7a89e3e225..59e79404ab 100644 --- a/services/search/pkg/content/tika_audio.go +++ b/services/search/pkg/content/tika_audio.go @@ -30,9 +30,23 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { audio.SetArtist(v) } - // TODO: audio.Bitrate: not provided by tika - // TODO: audio.Composers: not provided by tika - // TODO: audio.Copyright: not provided by tika for audio files? + if v, err := getFirstValue(meta, "audio:bitrate"); err == nil { + // tika emits bits per second, graph wants kbps + if bps, err := strconv.ParseInt(v, 10, 64); err == nil { + initAudio() + audio.SetBitrate(bps / 1000) + } + } + + if v, err := getFirstValue(meta, "xmpDM:composer"); err == nil { + initAudio() + audio.SetComposers(v) + } + + if v, err := getFirstValue(meta, "xmpDM:copyright"); err == nil { + initAudio() + audio.SetCopyright(v) + } if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil { if i, err := strconv.ParseInt(v, 10, 32); err == nil { @@ -42,7 +56,12 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { } - // TODO: audio.DiscCount: not provided by tika + if v, err := getFirstValue(meta, "audio:disc-count"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetDiscCount(int32(i)) + } + } if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil { // Tika emits fractional seconds. @@ -57,8 +76,19 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { audio.SetGenre(v) } - // TODO: audio.HasDrm: not provided by tika - // TODO: audio.IsVariableBitrate: not provided by tika + if v, err := getFirstValue(meta, "audio:has-drm"); err == nil { + if b, err := strconv.ParseBool(v); err == nil { + initAudio() + audio.SetHasDrm(b) + } + } + + if v, err := getFirstValue(meta, "audio:is-variable-bitrate"); err == nil { + if b, err := strconv.ParseBool(v); err == nil { + initAudio() + audio.SetIsVariableBitrate(b) + } + } if v, err := getFirstValue(meta, "dc:title"); err == nil { initAudio() @@ -72,7 +102,12 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { } } - // TODO: audio.TrackCount: not provided by tika + if v, err := getFirstValue(meta, "audio:track-count"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetTrackCount(int32(i)) + } + } if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil { if i, err := strconv.ParseInt(v, 10, 32); err == nil { diff --git a/services/search/pkg/content/tika_audio_test.go b/services/search/pkg/content/tika_audio_test.go index 7c81f3b6c9..65576fcc46 100644 --- a/services/search/pkg/content/tika_audio_test.go +++ b/services/search/pkg/content/tika_audio_test.go @@ -9,15 +9,22 @@ import ( var _ = Describe("getAudio", func() { It("maps the audio metadata to the audio facet", func() { meta := map[string][]string{ - "xmpDM:genre": {"Some Genre"}, - "xmpDM:album": {"Some Album"}, - "xmpDM:trackNumber": {"7"}, - "xmpDM:discNumber": {"4"}, - "xmpDM:releaseDate": {"2004"}, - "xmpDM:artist": {"Some Artist"}, - "xmpDM:albumArtist": {"Some AlbumArtist"}, - "dc:title": {"Some Title"}, - "xmpDM:duration": {"225.5"}, + "xmpDM:genre": {"Some Genre"}, + "xmpDM:album": {"Some Album"}, + "xmpDM:trackNumber": {"7"}, + "xmpDM:discNumber": {"4"}, + "xmpDM:releaseDate": {"2004"}, + "xmpDM:artist": {"Some Artist"}, + "xmpDM:albumArtist": {"Some AlbumArtist"}, + "dc:title": {"Some Title"}, + "xmpDM:duration": {"225.5"}, + "xmpDM:composer": {"Some Composers"}, + "xmpDM:copyright": {"Some Copyright"}, + "audio:bitrate": {"192000"}, + "audio:is-variable-bitrate": {"true"}, + "audio:has-drm": {"false"}, + "audio:track-count": {"9"}, + "audio:disc-count": {"5"}, } audio := Tika{}.getAudio(meta) @@ -26,11 +33,18 @@ var _ = Describe("getAudio", func() { Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) + Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright"))) Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4))) + Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500))) Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) + Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) + Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) Expect(audio.Track).To(Equal(libregraph.PtrInt32(7))) + Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9))) Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) }) diff --git a/services/search/pkg/content/tika_location.go b/services/search/pkg/content/tika_location.go index 2334c53afe..d7b489daab 100644 --- a/services/search/pkg/content/tika_location.go +++ b/services/search/pkg/content/tika_location.go @@ -6,6 +6,9 @@ import ( libregraph "github.com/opencloud-eu/libre-graph-api-go" ) +// graph geoCoordinates.altitude is in feet, exif GPS altitude (geo:alt) in metres. +const metresToFeet = 3.280839895 + func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { var location *libregraph.GeoCoordinates initLocation := func() { @@ -14,10 +17,6 @@ func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { } } - // TODO: location.Altitute: transform the following data to … feet above sea level. - // "GPS:GPS Altitude": []string{"227.4 metres"}, - // "GPS:GPS Altitude Ref": []string{"Sea level"}, - if v, err := getFirstValue(meta, "geo:lat"); err == nil { if i, err := strconv.ParseFloat(v, 64); err == nil { initLocation() @@ -32,5 +31,13 @@ func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { } } + // tika emits metres (negative below sea level), graph wants feet + if v, err := getFirstValue(meta, "geo:alt"); err == nil { + if metres, err := strconv.ParseFloat(v, 64); err == nil { + initLocation() + location.SetAltitude(metres * metresToFeet) + } + } + return location } diff --git a/services/search/pkg/content/tika_location_test.go b/services/search/pkg/content/tika_location_test.go index 38bb9ef705..067d06e4f8 100644 --- a/services/search/pkg/content/tika_location_test.go +++ b/services/search/pkg/content/tika_location_test.go @@ -7,14 +7,24 @@ import ( ) var _ = Describe("getLocation", func() { - It("maps latitude and longitude to the location facet", func() { + It("maps lat/long and converts altitude from metres to feet", func() { + metres := 227.4 location := Tika{}.getLocation(map[string][]string{ "geo:lat": {"49.48675890884328"}, "geo:long": {"11.103870357204285"}, + "geo:alt": {"227.4"}, }) Expect(location).ToNot(BeNil()) Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet))) + }) + + It("keeps below-sea-level altitude negative", func() { + metres := -227.4 + location := Tika{}.getLocation(map[string][]string{"geo:alt": {"-227.4"}}) + Expect(location).ToNot(BeNil()) + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet))) }) It("returns nil when no location metadata is present", func() { From 1be01ae50cf37f67dd41335177a9e3297b3fb6a8 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 16:51:34 +0200 Subject: [PATCH 3/8] fix(search): read the tika 4 content key, keep the legacy one Tika 4.0.0 renamed the meta prefix from X-TIKA: to tk:. --- services/search/pkg/content/tika.go | 5 ++++- services/search/pkg/content/tika_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 078152cb5d..1e7d8674e2 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -87,7 +87,10 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title)) } - if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil { + // tika 4 renamed the meta prefix from X-TIKA: to tk: + if content, err := getFirstValue(meta, "tk:content"); err == nil { + doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content)) + } else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil { doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content)) } diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index 2d9a34936f..8d0a70d440 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -100,6 +100,17 @@ var _ = Describe("Tika", func() { Expect(doc.Title).To(Equal("quarterly report")) }) + It("adds the content of a tika 4", func() { + fullResponse = `[{"tk:content": "some data"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("some data")) + }) + It("adds the title of an older tika", func() { fullResponse = `[{"title": "quarterly report"}]` From 77bb974f9be421a3488e079ae75537b2db91c4cd Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 17:14:33 +0200 Subject: [PATCH 4/8] test(search): run acceptance and CI against the released tika 4.0.0 Both tika pins (local compose and the CI service) point at apache/tika:4.0.0-full; the search README links tika version-less. --- .woodpecker.star | 5 ++++- services/search/README.md | 2 +- tests/acceptance/docker/src/tika.yml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.woodpecker.star b/.woodpecker.star index 978a1dfdfa..e5cd56303f 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -8,7 +8,7 @@ docker_repo_slug = "opencloudeu/opencloud" # images ALPINE_GIT = "alpine/git:latest" -APACHE_TIKA = "apache/tika:3.2.3.0-full" +APACHE_TIKA = "apache/tika:4.0.0-full" CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1" CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:14.1.3" COLLABORA_CODE = "collabora/code:24.04.5.1.1" @@ -3442,6 +3442,9 @@ def tikaService(): return [{ "name": "tika", "image": APACHE_TIKA, + # tika 4 discovers its plugins relative to the image working directory, + # the workspace default would leave the pipes fetchers empty + "directory": "/opt/tika-server", "detach": True, }, { "name": "wait-for-tika-service", diff --git a/services/search/README.md b/services/search/README.md index 026f4fc5ae..96bdc00c21 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -99,7 +99,7 @@ It does not do any further content analysis. The main difference is that this extractor is able to analyze and extract data from more advanced file types like PDF, DOCX, PPTX, etc. However, [Apache Tika](https://tika.apache.org/) is required for this task. -Read the [Getting Started with Apache Tika](https://tika.apache.org/2.6.0/gettingstarted.html) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika). +Read the [Getting Started with Apache Tika](https://tika.apache.org/) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika). See the [Tika container usage document](https://github.com/apache/tika-docker#usage) for a quickstart. As soon as Tika is installed and configured, the search service needs to be told to use it. diff --git a/tests/acceptance/docker/src/tika.yml b/tests/acceptance/docker/src/tika.yml index 4c9768a090..fe8eff86e8 100644 --- a/tests/acceptance/docker/src/tika.yml +++ b/tests/acceptance/docker/src/tika.yml @@ -5,4 +5,4 @@ services: - tika command: tika:9998 tika: - image: apache/tika:3.2.3.0-full + image: apache/tika:4.0.0-full From c03b9c72291971f5bf0293d29cc7ff295e23c7ab Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 19:45:53 +0200 Subject: [PATCH 5/8] fix(search): gate the audio facet inside getAudio, keep facets across meta entries --- services/search/pkg/content/tika.go | 18 +++++-- services/search/pkg/content/tika_audio.go | 14 +++++- .../search/pkg/content/tika_audio_test.go | 26 ++++++++++ services/search/pkg/content/tika_test.go | 49 +++++++++++++++++++ 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 1e7d8674e2..73eb9d603c 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -94,10 +94,20 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content)) } - doc.Location = t.getLocation(meta) - doc.Image = t.getImage(meta) - doc.Photo = t.getPhoto(meta) - doc.Audio = t.getAudio(meta) + // keep facets from earlier entries, an embedded resource's meta + // (e.g. cover art) must not reset them + if v := t.getLocation(meta); v != nil { + doc.Location = v + } + if v := t.getImage(meta); v != nil { + doc.Image = v + } + if v := t.getPhoto(meta); v != nil { + doc.Photo = v + } + if v := t.getAudio(meta); v != nil { + doc.Audio = v + } } if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords { diff --git a/services/search/pkg/content/tika_audio.go b/services/search/pkg/content/tika_audio.go index 59e79404ab..c6f121db0d 100644 --- a/services/search/pkg/content/tika_audio.go +++ b/services/search/pkg/content/tika_audio.go @@ -3,11 +3,19 @@ package content import ( "math" "strconv" + "strings" + "time" libregraph "github.com/opencloud-eu/libre-graph-api-go" ) func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { + // generic keys like dc:title show up on any document, only audio + // content types carry the audio facet + if v, err := getFirstValue(meta, "Content-Type"); err != nil || !strings.HasPrefix(v, "audio/") { + return nil + } + var audio *libregraph.Audio initAudio := func() { if audio == nil { @@ -34,7 +42,7 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { // tika emits bits per second, graph wants kbps if bps, err := strconv.ParseInt(v, 10, 64); err == nil { initAudio() - audio.SetBitrate(bps / 1000) + audio.SetBitrate(int64(math.Round(float64(bps) / 1000))) } } @@ -53,7 +61,6 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { initAudio() audio.SetDisc(int32(i)) } - } if v, err := getFirstValue(meta, "audio:disc-count"); err == nil { @@ -113,6 +120,9 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { if i, err := strconv.ParseInt(v, 10, 32); err == nil { initAudio() audio.SetYear(int32(i)) + } else if d, err := time.Parse(time.DateOnly, v); err == nil { + initAudio() + audio.SetYear(int32(d.Year())) } } diff --git a/services/search/pkg/content/tika_audio_test.go b/services/search/pkg/content/tika_audio_test.go index 65576fcc46..02f85719f1 100644 --- a/services/search/pkg/content/tika_audio_test.go +++ b/services/search/pkg/content/tika_audio_test.go @@ -9,6 +9,7 @@ import ( var _ = Describe("getAudio", func() { It("maps the audio metadata to the audio facet", func() { meta := map[string][]string{ + "Content-Type": {"audio/mpeg"}, "xmpDM:genre": {"Some Genre"}, "xmpDM:album": {"Some Album"}, "xmpDM:trackNumber": {"7"}, @@ -51,4 +52,29 @@ var _ = Describe("getAudio", func() { It("returns nil when no audio metadata is present", func() { Expect(Tika{}.getAudio(map[string][]string{})).To(BeNil()) }) + + It("returns nil for non-audio content types", func() { + Expect(Tika{}.getAudio(map[string][]string{ + "Content-Type": {"application/pdf"}, + "dc:title": {"quarterly report"}, + })).To(BeNil()) + }) + + It("takes the year from a full release date", func() { + audio := Tika{}.getAudio(map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "xmpDM:releaseDate": {"2004-06-01"}, + }) + Expect(audio).ToNot(BeNil()) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + + It("rounds the bitrate to the nearest kbps", func() { + audio := Tika{}.getAudio(map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "audio:bitrate": {"191999"}, + }) + Expect(audio).ToNot(BeNil()) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + }) }) diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index 8d0a70d440..b759cf6a7c 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -11,6 +11,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" "github.com/stretchr/testify/mock" "github.com/opencloud-eu/opencloud/pkg/log" @@ -121,6 +122,7 @@ var _ = Describe("Tika", func() { Expect(err).ToNot(HaveOccurred()) Expect(doc.Title).To(Equal("quarterly report")) }) + It("removes stop words", func() { body = "body to test stop words!!! against almost everyone" language = "en" @@ -133,6 +135,53 @@ var _ = Describe("Tika", func() { Expect(doc.Content).To(Equal("body test stop words!!!")) }) + It("keeps the audio facet when an embedded resource follows", func() { + fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Audio).ToNot(BeNil()) + Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker"))) + Expect(doc.Image).ToNot(BeNil()) + }) + + It("adds no audio facet to non-audio documents", func() { + fullResponse = `[{"Content-Type": "application/pdf", "dc:title": "quarterly report"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Audio).To(BeNil()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + + It("prefers the tika 4 content key over the legacy one", func() { + fullResponse = `[{"tk:content": "new", "X-TIKA:content": "old"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("new")) + }) + + It("joins the content of all meta entries", func() { + fullResponse = `[{"tk:content": "one"}, {"X-TIKA:content": "two"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("one two")) + }) + It("keeps stop words", func() { body = "body to test stop words!!! against almost everyone" language = "en" From ae9ec4ce70fb647024ae525fd2d92351973eae48 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 19:45:53 +0200 Subject: [PATCH 6/8] fix(search): harden facet value parsing --- services/search/pkg/content/extractor.go | 2 +- services/search/pkg/content/tika_location.go | 46 +++++++++++-------- .../search/pkg/content/tika_location_test.go | 16 ++++++- services/search/pkg/content/tika_photo.go | 2 +- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/services/search/pkg/content/extractor.go b/services/search/pkg/content/extractor.go index a91be6cb4e..3d4e457288 100644 --- a/services/search/pkg/content/extractor.go +++ b/services/search/pkg/content/extractor.go @@ -23,7 +23,7 @@ func getFirstValue(m map[string][]string, key string) (string, error) { return "", fmt.Errorf("unknown key: %v", key) } - if len(m) == 0 { + if len(v) == 0 { return "", fmt.Errorf("no values for: %v", key) } diff --git a/services/search/pkg/content/tika_location.go b/services/search/pkg/content/tika_location.go index d7b489daab..936ba9b911 100644 --- a/services/search/pkg/content/tika_location.go +++ b/services/search/pkg/content/tika_location.go @@ -1,6 +1,8 @@ package content import ( + "fmt" + "math" "strconv" libregraph "github.com/opencloud-eu/libre-graph-api-go" @@ -10,34 +12,40 @@ import ( const metresToFeet = 3.280839895 func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { - var location *libregraph.GeoCoordinates - initLocation := func() { - if location == nil { - location = libregraph.NewGeoCoordinates() - } + // the facet needs a sane coordinate pair, an altitude alone is useless + lat, latErr := parseCoordinate(meta, "geo:lat", 90) + long, longErr := parseCoordinate(meta, "geo:long", 180) + if latErr != nil || longErr != nil { + return nil } - if v, err := getFirstValue(meta, "geo:lat"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLatitude(i) - } - } - - if v, err := getFirstValue(meta, "geo:long"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLongitude(i) - } - } + location := libregraph.NewGeoCoordinates() + location.SetLatitude(lat) + location.SetLongitude(long) // tika emits metres (negative below sea level), graph wants feet if v, err := getFirstValue(meta, "geo:alt"); err == nil { if metres, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() location.SetAltitude(metres * metresToFeet) } } return location } + +func parseCoordinate(meta map[string][]string, key string, limit float64) (float64, error) { + v, err := getFirstValue(meta, key) + if err != nil { + return 0, err + } + + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0, err + } + if math.Abs(f) > limit { + return 0, fmt.Errorf("%s out of range: %v", key, f) + } + + return f, nil +} diff --git a/services/search/pkg/content/tika_location_test.go b/services/search/pkg/content/tika_location_test.go index 067d06e4f8..8b79810de5 100644 --- a/services/search/pkg/content/tika_location_test.go +++ b/services/search/pkg/content/tika_location_test.go @@ -22,11 +22,25 @@ var _ = Describe("getLocation", func() { It("keeps below-sea-level altitude negative", func() { metres := -227.4 - location := Tika{}.getLocation(map[string][]string{"geo:alt": {"-227.4"}}) + location := Tika{}.getLocation(map[string][]string{ + "geo:lat": {"31.5"}, + "geo:long": {"35.47"}, + "geo:alt": {"-227.4"}, + }) Expect(location).ToNot(BeNil()) Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet))) }) + It("returns nil for an incomplete coordinate pair", func() { + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:alt": {"227.4"}})).To(BeNil()) + }) + + It("returns nil for out-of-range coordinates", func() { + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"91"}, "geo:long": {"11.1"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}, "geo:long": {"-180.5"}})).To(BeNil()) + }) + It("returns nil when no location metadata is present", func() { Expect(Tika{}.getLocation(map[string][]string{})).To(BeNil()) }) diff --git a/services/search/pkg/content/tika_photo.go b/services/search/pkg/content/tika_photo.go index a833df3b1d..2ee961a300 100644 --- a/services/search/pkg/content/tika_photo.go +++ b/services/search/pkg/content/tika_photo.go @@ -63,7 +63,7 @@ func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo { } if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil && i > 0 { initPhoto() photo.SetExposureNumerator(1) photo.SetExposureDenominator(math.Round(1 / i)) From a1e8aa783d19247bbbd8f3824d058c2605ce006c Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 20:05:36 +0200 Subject: [PATCH 7/8] fix(search): reject NaN coordinates and non-positive bitrates --- services/search/pkg/content/tika_audio.go | 2 +- services/search/pkg/content/tika_location.go | 3 ++- services/search/pkg/content/tika_location_test.go | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/search/pkg/content/tika_audio.go b/services/search/pkg/content/tika_audio.go index c6f121db0d..6b6b088472 100644 --- a/services/search/pkg/content/tika_audio.go +++ b/services/search/pkg/content/tika_audio.go @@ -40,7 +40,7 @@ func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { if v, err := getFirstValue(meta, "audio:bitrate"); err == nil { // tika emits bits per second, graph wants kbps - if bps, err := strconv.ParseInt(v, 10, 64); err == nil { + if bps, err := strconv.ParseInt(v, 10, 64); err == nil && bps > 0 { initAudio() audio.SetBitrate(int64(math.Round(float64(bps) / 1000))) } diff --git a/services/search/pkg/content/tika_location.go b/services/search/pkg/content/tika_location.go index 936ba9b911..586a4731c7 100644 --- a/services/search/pkg/content/tika_location.go +++ b/services/search/pkg/content/tika_location.go @@ -43,7 +43,8 @@ func parseCoordinate(meta map[string][]string, key string, limit float64) (float if err != nil { return 0, err } - if math.Abs(f) > limit { + // ParseFloat accepts "NaN", which json cannot marshal + if math.IsNaN(f) || math.Abs(f) > limit { return 0, fmt.Errorf("%s out of range: %v", key, f) } diff --git a/services/search/pkg/content/tika_location_test.go b/services/search/pkg/content/tika_location_test.go index 8b79810de5..413dae0c73 100644 --- a/services/search/pkg/content/tika_location_test.go +++ b/services/search/pkg/content/tika_location_test.go @@ -39,6 +39,7 @@ var _ = Describe("getLocation", func() { It("returns nil for out-of-range coordinates", func() { Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"91"}, "geo:long": {"11.1"}})).To(BeNil()) Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}, "geo:long": {"-180.5"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"NaN"}, "geo:long": {"11.1"}})).To(BeNil()) }) It("returns nil when no location metadata is present", func() { From 5e4f124e78b5968b61f5b83a4b81e1355b3630dc Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 20:37:10 +0200 Subject: [PATCH 8/8] fix(search): find the language endpoint on tika 4, keep the legacy path --- services/search/pkg/content/tika.go | 28 ++++++++++++++++++- services/search/pkg/content/tika_test.go | 34 +++++++++++++++++++----- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 73eb9d603c..619744fed6 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -3,6 +3,8 @@ package content import ( "context" "fmt" + "io" + "net/http" "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" @@ -20,6 +22,7 @@ type Tika struct { *Basic Retriever tika *tika.Client + tikaURL string ContentExtractionSizeLimit uint64 CleanStopWords bool } @@ -42,6 +45,7 @@ func NewTikaExtractor(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], Basic: basic, Retriever: newCS3Retriever(gatewaySelector, logger, cfg.Extractor.CS3AllowInsecure), tika: tika.NewClient(nil, cfg.Extractor.Tika.TikaURL), + tikaURL: cfg.Extractor.Tika.TikaURL, ContentExtractionSizeLimit: cfg.ContentExtractionSizeLimit, CleanStopWords: cfg.Extractor.Tika.CleanStopWords, }, nil @@ -110,9 +114,31 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, } } - if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords { + if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords { doc.Content = CleanString(doc.Content, langCode) } return doc, nil } + +// detectLanguage asks tika for the language of content. Tika 4 moved the +// endpoint from /language/string to /language, so try the new path first and +// fall back for an older tika. +func (t Tika) detectLanguage(ctx context.Context, content string) string { + for _, path := range []string{"/language", "/language/string"} { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, t.tikaURL+path, strings.NewReader(content)) + if err != nil { + return "" + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return "" + } + lang, err := io.ReadAll(res.Body) + _ = res.Body.Close() + if err == nil && res.StatusCode == http.StatusOK && len(lang) > 0 { + return string(lang) + } + } + return "" +} diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index b759cf6a7c..772ed185b3 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -23,17 +23,19 @@ import ( var _ = Describe("Tika", func() { Describe("extract", func() { var ( - body string - fullResponse string - language string - version string - srv *httptest.Server - tika *content.Tika + body string + fullResponse string + language string + tika4Language bool + version string + srv *httptest.Server + tika *content.Tika ) BeforeEach(func() { body = "" language = "" + tika4Language = false version = "" fullResponse = "" srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -41,6 +43,13 @@ var _ = Describe("Tika", func() { switch req.URL.Path { case "/version": out = version + case "/language": + if tika4Language { + out = language + } else { + w.WriteHeader(http.StatusNotFound) + return + } case "/language/string": out = language case "/rmeta/text": @@ -123,6 +132,19 @@ var _ = Describe("Tika", func() { Expect(doc.Title).To(Equal("quarterly report")) }) + It("removes stop words with a tika 4 language endpoint", func() { + body = "body to test stop words!!! against almost everyone" + language = "en" + tika4Language = true + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("body test stop words!!!")) + }) + It("removes stop words", func() { body = "body to test stop words!!! against almost everyone" language = "en"