refactor(search): split the tika extractor and its tests into per-facet files

This commit is contained in:
Dominik Schmidt
2026-07-30 00:07:23 +02:00
parent 62d0aadc42
commit b82fe225da
10 changed files with 351 additions and 327 deletions

View File

@@ -3,16 +3,12 @@ package content
import (
"context"
"fmt"
"math"
"strconv"
"strings"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/google/go-tika/tika"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/config"
@@ -106,199 +102,3 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
return doc, nil
}
func (t Tika) getImage(meta map[string][]string) *libregraph.Image {
var image *libregraph.Image
initImage := func() {
if image == nil {
image = libregraph.NewImage()
}
}
if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initImage()
image.SetWidth(int32(i))
}
}
if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initImage()
image.SetHeight(int32(i))
}
}
return image
}
func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates {
var location *libregraph.GeoCoordinates
initLocation := func() {
if location == nil {
location = libregraph.NewGeoCoordinates()
}
}
// TODO: location.Altitute: transform the following data to … feet above sea level.
// "GPS:GPS Altitude": []string{"227.4 metres"},
// "GPS:GPS Altitude Ref": []string{"Sea level"},
if v, err := getFirstValue(meta, "geo:lat"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initLocation()
location.SetLatitude(i)
}
}
if v, err := getFirstValue(meta, "geo:long"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initLocation()
location.SetLongitude(i)
}
}
return location
}
func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo {
var photo *libregraph.Photo
initPhoto := func() {
if photo == nil {
photo = libregraph.NewPhoto()
}
}
if v, err := getFirstValue(meta, "tiff:Make"); err == nil {
initPhoto()
photo.SetCameraMake(v)
}
if v, err := getFirstValue(meta, "tiff:Model"); err == nil {
initPhoto()
photo.SetCameraModel(v)
}
if v, err := getFirstValue(meta, "exif:FNumber"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initPhoto()
photo.SetFNumber(i)
}
}
if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initPhoto()
photo.SetFocalLength(i)
}
}
if v, err := getFirstValue(meta, "Base ISO"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initPhoto()
photo.SetIso(int32(i))
}
}
if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initPhoto()
photo.SetOrientation(int32(i))
}
}
if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil {
layout := "2006-01-02T15:04:05"
if t, err := time.Parse(layout, v); err == nil {
initPhoto()
photo.SetTakenDateTime(t)
}
}
if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initPhoto()
photo.SetExposureNumerator(1)
photo.SetExposureDenominator(math.Round(1 / i))
}
}
return photo
}
func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio {
var audio *libregraph.Audio
initAudio := func() {
if audio == nil {
audio = libregraph.NewAudio()
}
}
if v, err := getFirstValue(meta, "xmpDM:album"); err == nil {
initAudio()
audio.SetAlbum(v)
}
if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil {
initAudio()
audio.SetAlbumArtist(v)
}
if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil {
initAudio()
audio.SetArtist(v)
}
// TODO: audio.Bitrate: not provided by tika
// TODO: audio.Composers: not provided by tika
// TODO: audio.Copyright: not provided by tika for audio files?
if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil {
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
initAudio()
audio.SetDisc(int32(i))
}
}
// TODO: audio.DiscCount: not provided by tika
if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil {
// Tika emits fractional seconds.
if f, err := strconv.ParseFloat(v, 64); err == nil {
initAudio()
audio.SetDuration(int64(math.Round(f * 1000)))
}
}
if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil {
initAudio()
audio.SetGenre(v)
}
// TODO: audio.HasDrm: not provided by tika
// TODO: audio.IsVariableBitrate: not provided by tika
if v, err := getFirstValue(meta, "dc:title"); err == nil {
initAudio()
audio.SetTitle(v)
}
if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil {
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
initAudio()
audio.SetTrack(int32(i))
}
}
// TODO: audio.TrackCount: not provided by tika
if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil {
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
initAudio()
audio.SetYear(int32(i))
}
}
return audio
}

View File

@@ -0,0 +1,85 @@
package content
import (
"math"
"strconv"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio {
var audio *libregraph.Audio
initAudio := func() {
if audio == nil {
audio = libregraph.NewAudio()
}
}
if v, err := getFirstValue(meta, "xmpDM:album"); err == nil {
initAudio()
audio.SetAlbum(v)
}
if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil {
initAudio()
audio.SetAlbumArtist(v)
}
if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil {
initAudio()
audio.SetArtist(v)
}
// TODO: audio.Bitrate: not provided by tika
// TODO: audio.Composers: not provided by tika
// TODO: audio.Copyright: not provided by tika for audio files?
if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil {
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
initAudio()
audio.SetDisc(int32(i))
}
}
// TODO: audio.DiscCount: not provided by tika
if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil {
// Tika emits fractional seconds.
if f, err := strconv.ParseFloat(v, 64); err == nil {
initAudio()
audio.SetDuration(int64(math.Round(f * 1000)))
}
}
if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil {
initAudio()
audio.SetGenre(v)
}
// TODO: audio.HasDrm: not provided by tika
// TODO: audio.IsVariableBitrate: not provided by tika
if v, err := getFirstValue(meta, "dc:title"); err == nil {
initAudio()
audio.SetTitle(v)
}
if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil {
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
initAudio()
audio.SetTrack(int32(i))
}
}
// TODO: audio.TrackCount: not provided by tika
if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil {
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
initAudio()
audio.SetYear(int32(i))
}
}
return audio
}

View File

@@ -0,0 +1,40 @@
package content
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
var _ = Describe("getAudio", func() {
It("maps the audio metadata to the audio facet", func() {
meta := map[string][]string{
"xmpDM:genre": {"Some Genre"},
"xmpDM:album": {"Some Album"},
"xmpDM:trackNumber": {"7"},
"xmpDM:discNumber": {"4"},
"xmpDM:releaseDate": {"2004"},
"xmpDM:artist": {"Some Artist"},
"xmpDM:albumArtist": {"Some AlbumArtist"},
"dc:title": {"Some Title"},
"xmpDM:duration": {"225.5"},
}
audio := Tika{}.getAudio(meta)
Expect(audio).ToNot(BeNil())
Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4)))
Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500)))
Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
Expect(audio.Track).To(Equal(libregraph.PtrInt32(7)))
Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
})
It("returns nil when no audio metadata is present", func() {
Expect(Tika{}.getAudio(map[string][]string{})).To(BeNil())
})
})

View File

@@ -0,0 +1,32 @@
package content
import (
"strconv"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
func (t Tika) getImage(meta map[string][]string) *libregraph.Image {
var image *libregraph.Image
initImage := func() {
if image == nil {
image = libregraph.NewImage()
}
}
if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initImage()
image.SetWidth(int32(i))
}
}
if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initImage()
image.SetHeight(int32(i))
}
}
return image
}

View File

@@ -0,0 +1,23 @@
package content
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
var _ = Describe("getImage", func() {
It("maps the image dimensions to the image facet", func() {
image := Tika{}.getImage(map[string][]string{
"tiff:ImageWidth": {"100"},
"tiff:ImageLength": {"200"},
})
Expect(image).ToNot(BeNil())
Expect(image.Width).To(Equal(libregraph.PtrInt32(100)))
Expect(image.Height).To(Equal(libregraph.PtrInt32(200)))
})
It("returns nil when no image metadata is present", func() {
Expect(Tika{}.getImage(map[string][]string{})).To(BeNil())
})
})

View File

@@ -0,0 +1,36 @@
package content
import (
"strconv"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates {
var location *libregraph.GeoCoordinates
initLocation := func() {
if location == nil {
location = libregraph.NewGeoCoordinates()
}
}
// TODO: location.Altitute: transform the following data to … feet above sea level.
// "GPS:GPS Altitude": []string{"227.4 metres"},
// "GPS:GPS Altitude Ref": []string{"Sea level"},
if v, err := getFirstValue(meta, "geo:lat"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initLocation()
location.SetLatitude(i)
}
}
if v, err := getFirstValue(meta, "geo:long"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initLocation()
location.SetLongitude(i)
}
}
return location
}

View File

@@ -0,0 +1,23 @@
package content
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
var _ = Describe("getLocation", func() {
It("maps latitude and longitude to the location facet", func() {
location := Tika{}.getLocation(map[string][]string{
"geo:lat": {"49.48675890884328"},
"geo:long": {"11.103870357204285"},
})
Expect(location).ToNot(BeNil())
Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
})
It("returns nil when no location metadata is present", func() {
Expect(Tika{}.getLocation(map[string][]string{})).To(BeNil())
})
})

View File

@@ -0,0 +1,74 @@
package content
import (
"math"
"strconv"
"time"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo {
var photo *libregraph.Photo
initPhoto := func() {
if photo == nil {
photo = libregraph.NewPhoto()
}
}
if v, err := getFirstValue(meta, "tiff:Make"); err == nil {
initPhoto()
photo.SetCameraMake(v)
}
if v, err := getFirstValue(meta, "tiff:Model"); err == nil {
initPhoto()
photo.SetCameraModel(v)
}
if v, err := getFirstValue(meta, "exif:FNumber"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initPhoto()
photo.SetFNumber(i)
}
}
if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initPhoto()
photo.SetFocalLength(i)
}
}
if v, err := getFirstValue(meta, "Base ISO"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initPhoto()
photo.SetIso(int32(i))
}
}
if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil {
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
initPhoto()
photo.SetOrientation(int32(i))
}
}
if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil {
layout := "2006-01-02T15:04:05"
if t, err := time.Parse(layout, v); err == nil {
initPhoto()
photo.SetTakenDateTime(t)
}
}
if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil {
if i, err := strconv.ParseFloat(v, 64); err == nil {
initPhoto()
photo.SetExposureNumerator(1)
photo.SetExposureDenominator(math.Round(1 / i))
}
}
return photo
}

View File

@@ -0,0 +1,38 @@
package content
import (
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
var _ = Describe("getPhoto", func() {
It("maps the exif metadata to the photo facet", func() {
photo := Tika{}.getPhoto(map[string][]string{
"tiff:Make": {"Canon"},
"tiff:Model": {"Canon EOS 5D"},
"exif:ExposureTime": {"0.001"},
"exif:FNumber": {"1.8"},
"exif:FocalLength": {"50"},
"Base ISO": {"100"},
"tiff:Orientation": {"1"},
"exif:DateTimeOriginal": {"2018-01-01T12:34:56"},
})
Expect(photo).ToNot(BeNil())
Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon")))
Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D")))
Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1)))
Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000)))
Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8)))
Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50)))
Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100)))
Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1)))
Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC))))
})
It("returns nil when no photo metadata is present", func() {
Expect(Tika{}.getPhoto(map[string][]string{})).To(BeNil())
})
})

View File

@@ -7,12 +7,10 @@ import (
"net/http"
"net/http/httptest"
"strings"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/stretchr/testify/mock"
"github.com/opencloud-eu/opencloud/pkg/log"
@@ -91,131 +89,6 @@ var _ = Describe("Tika", func() {
Expect(doc.Content).To(Equal(body))
})
It("adds audio content", func() {
fullResponse = `[
{
"xmpDM:genre": "Some Genre",
"xmpDM:album": "Some Album",
"xmpDM:trackNumber": "7",
"xmpDM:discNumber": "4",
"xmpDM:releaseDate": "2004",
"xmpDM:artist": "Some Artist",
"xmpDM:albumArtist": "Some AlbumArtist",
"xmpDM:audioCompressor": "MP3",
"xmpDM:audioChannelType": "Stereo",
"version": "MPEG 3 Layer III Version 1",
"xmpDM:logComment": "some comment",
"xmpDM:audioSampleRate": "44100",
"channels": "2",
"dc:title": "Some Title",
"xmpDM:duration": "225.5",
"Content-Type": "audio/mpeg",
"samplerate": "44100"
}
]`
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Size: 1,
})
Expect(err).ToNot(HaveOccurred())
audio := doc.Audio
Expect(audio).ToNot(BeNil())
Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
// Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
// Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers")))
// Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright")))
Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4)))
// Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5)))
Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500)))
Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
// Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false)))
// Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true)))
Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
Expect(audio.Track).To(Equal(libregraph.PtrInt32(7)))
// Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9)))
Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
})
It("adds location content", func() {
fullResponse = `[
{
"geo:lat": "49.48675890884328",
"geo:long": "11.103870357204285"
}
]`
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Size: 1,
})
Expect(err).ToNot(HaveOccurred())
location := doc.Location
Expect(location).ToNot(BeNil())
// TODO: Altitude is not supported right now
Expect(location.Altitude).To(BeNil())
Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
})
It("adds image content", func() {
fullResponse = `[
{
"tiff:ImageWidth": "100",
"tiff:ImageLength": "100"
}
]`
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Size: 1,
})
Expect(err).ToNot(HaveOccurred())
image := doc.Image
Expect(image).ToNot(BeNil())
Expect(image.Width).To(Equal(libregraph.PtrInt32(100)))
Expect(image.Height).To(Equal(libregraph.PtrInt32(100)))
})
It("adds photo content", func() {
fullResponse = `[
{
"tiff:Make": "Canon",
"tiff:Model": "Canon EOS 5D",
"exif:ExposureTime": "0.001",
"exif:FNumber": "1.8",
"exif:FocalLength": "50",
"Base ISO": "100",
"tiff:Orientation": "1",
"exif:DateTimeOriginal": "2018-01-01T12:34:56"
}
]`
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Size: 1,
})
Expect(err).ToNot(HaveOccurred())
photo := doc.Photo
Expect(photo).ToNot(BeNil())
Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon")))
Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D")))
Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1)))
Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000)))
Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8)))
Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50)))
Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100)))
Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1)))
Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC))))
})
It("removes stop words", func() {
body = "body to test stop words!!! against almost everyone"
language = "en"