diff --git a/.woodpecker.star b/.woodpecker.star index 978a1dfdfa..e5cd56303f 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -8,7 +8,7 @@ docker_repo_slug = "opencloudeu/opencloud" # images ALPINE_GIT = "alpine/git:latest" -APACHE_TIKA = "apache/tika:3.2.3.0-full" +APACHE_TIKA = "apache/tika:4.0.0-full" CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1" CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:14.1.3" COLLABORA_CODE = "collabora/code:24.04.5.1.1" @@ -3442,6 +3442,9 @@ def tikaService(): return [{ "name": "tika", "image": APACHE_TIKA, + # tika 4 discovers its plugins relative to the image working directory, + # the workspace default would leave the pipes fetchers empty + "directory": "/opt/tika-server", "detach": True, }, { "name": "wait-for-tika-service", diff --git a/services/search/README.md b/services/search/README.md index 026f4fc5ae..96bdc00c21 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -99,7 +99,7 @@ It does not do any further content analysis. The main difference is that this extractor is able to analyze and extract data from more advanced file types like PDF, DOCX, PPTX, etc. However, [Apache Tika](https://tika.apache.org/) is required for this task. -Read the [Getting Started with Apache Tika](https://tika.apache.org/2.6.0/gettingstarted.html) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika). +Read the [Getting Started with Apache Tika](https://tika.apache.org/) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika). See the [Tika container usage document](https://github.com/apache/tika-docker#usage) for a quickstart. As soon as Tika is installed and configured, the search service needs to be told to use it. diff --git a/services/search/pkg/content/extractor.go b/services/search/pkg/content/extractor.go index a91be6cb4e..3d4e457288 100644 --- a/services/search/pkg/content/extractor.go +++ b/services/search/pkg/content/extractor.go @@ -23,7 +23,7 @@ func getFirstValue(m map[string][]string, key string) (string, error) { return "", fmt.Errorf("unknown key: %v", key) } - if len(m) == 0 { + if len(v) == 0 { return "", fmt.Errorf("no values for: %v", key) } diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 2b475efb17..619744fed6 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -3,15 +3,13 @@ package content import ( "context" "fmt" - "math" - "strconv" + "io" + "net/http" "strings" - "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/google/go-tika/tika" - libregraph "github.com/opencloud-eu/libre-graph-api-go" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/opencloud/pkg/log" @@ -24,6 +22,7 @@ type Tika struct { *Basic Retriever tika *tika.Client + tikaURL string ContentExtractionSizeLimit uint64 CleanStopWords bool } @@ -46,6 +45,7 @@ func NewTikaExtractor(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], Basic: basic, Retriever: newCS3Retriever(gatewaySelector, logger, cfg.Extractor.CS3AllowInsecure), tika: tika.NewClient(nil, cfg.Extractor.Tika.TikaURL), + tikaURL: cfg.Extractor.Tika.TikaURL, ContentExtractionSizeLimit: cfg.ContentExtractionSizeLimit, CleanStopWords: cfg.Extractor.Tika.CleanStopWords, }, nil @@ -91,218 +91,54 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title)) } - if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil { + // tika 4 renamed the meta prefix from X-TIKA: to tk: + if content, err := getFirstValue(meta, "tk:content"); err == nil { + doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content)) + } else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil { doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content)) } - doc.Location = t.getLocation(meta) - doc.Image = t.getImage(meta) - doc.Photo = t.getPhoto(meta) - - if contentType, err := getFirstValue(meta, "Content-Type"); err == nil && strings.HasPrefix(contentType, "audio/") { - doc.Audio = t.getAudio(meta) + // keep facets from earlier entries, an embedded resource's meta + // (e.g. cover art) must not reset them + if v := t.getLocation(meta); v != nil { + doc.Location = v + } + if v := t.getImage(meta); v != nil { + doc.Image = v + } + if v := t.getPhoto(meta); v != nil { + doc.Photo = v + } + if v := t.getAudio(meta); v != nil { + doc.Audio = v } } - if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords { + if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords { doc.Content = CleanString(doc.Content, langCode) } return doc, nil } -func (t Tika) getImage(meta map[string][]string) *libregraph.Image { - var image *libregraph.Image - initImage := func() { - if image == nil { - image = libregraph.NewImage() +// detectLanguage asks tika for the language of content. Tika 4 moved the +// endpoint from /language/string to /language, so try the new path first and +// fall back for an older tika. +func (t Tika) detectLanguage(ctx context.Context, content string) string { + for _, path := range []string{"/language", "/language/string"} { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, t.tikaURL+path, strings.NewReader(content)) + if err != nil { + return "" + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return "" + } + lang, err := io.ReadAll(res.Body) + _ = res.Body.Close() + if err == nil && res.StatusCode == http.StatusOK && len(lang) > 0 { + return string(lang) } } - - if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initImage() - image.SetWidth(int32(i)) - } - } - - if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initImage() - image.SetHeight(int32(i)) - } - } - - return image -} - -func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { - var location *libregraph.GeoCoordinates - initLocation := func() { - if location == nil { - location = libregraph.NewGeoCoordinates() - } - } - - // TODO: location.Altitute: transform the following data to … feet above sea level. - // "GPS:GPS Altitude": []string{"227.4 metres"}, - // "GPS:GPS Altitude Ref": []string{"Sea level"}, - - if v, err := getFirstValue(meta, "geo:lat"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLatitude(i) - } - } - - if v, err := getFirstValue(meta, "geo:long"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLongitude(i) - } - } - - return location -} - -func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo { - var photo *libregraph.Photo - initPhoto := func() { - if photo == nil { - photo = libregraph.NewPhoto() - } - } - - if v, err := getFirstValue(meta, "tiff:Make"); err == nil { - initPhoto() - photo.SetCameraMake(v) - } - - if v, err := getFirstValue(meta, "tiff:Model"); err == nil { - initPhoto() - photo.SetCameraModel(v) - } - - if v, err := getFirstValue(meta, "exif:FNumber"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetFNumber(i) - } - } - - if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetFocalLength(i) - } - } - - if v, err := getFirstValue(meta, "Base ISO"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initPhoto() - photo.SetIso(int32(i)) - } - } - - if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initPhoto() - photo.SetOrientation(int32(i)) - } - } - - if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil { - layout := "2006-01-02T15:04:05" - if t, err := time.Parse(layout, v); err == nil { - initPhoto() - photo.SetTakenDateTime(t) - } - } - - if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetExposureNumerator(1) - photo.SetExposureDenominator(math.Round(1 / i)) - } - } - - return photo -} - -func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { - var audio *libregraph.Audio - initAudio := func() { - if audio == nil { - audio = libregraph.NewAudio() - } - } - - if v, err := getFirstValue(meta, "xmpDM:album"); err == nil { - initAudio() - audio.SetAlbum(v) - } - - if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil { - initAudio() - audio.SetAlbumArtist(v) - } - - if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil { - initAudio() - audio.SetArtist(v) - } - - // TODO: audio.Bitrate: not provided by tika - // TODO: audio.Composers: not provided by tika - // TODO: audio.Copyright: not provided by tika for audio files? - - if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetDisc(int32(i)) - } - - } - - // TODO: audio.DiscCount: not provided by tika - - if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil { - // Tika emits fractional seconds. - if f, err := strconv.ParseFloat(v, 64); err == nil { - initAudio() - audio.SetDuration(int64(math.Round(f * 1000))) - } - } - - if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil { - initAudio() - audio.SetGenre(v) - } - - // TODO: audio.HasDrm: not provided by tika - // TODO: audio.IsVariableBitrate: not provided by tika - - if v, err := getFirstValue(meta, "dc:title"); err == nil { - initAudio() - audio.SetTitle(v) - } - - if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetTrack(int32(i)) - } - } - - // TODO: audio.TrackCount: not provided by tika - - if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetYear(int32(i)) - } - } - - return audio + return "" } diff --git a/services/search/pkg/content/tika_audio.go b/services/search/pkg/content/tika_audio.go new file mode 100644 index 0000000000..6b6b088472 --- /dev/null +++ b/services/search/pkg/content/tika_audio.go @@ -0,0 +1,130 @@ +package content + +import ( + "math" + "strconv" + "strings" + "time" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { + // generic keys like dc:title show up on any document, only audio + // content types carry the audio facet + if v, err := getFirstValue(meta, "Content-Type"); err != nil || !strings.HasPrefix(v, "audio/") { + return nil + } + + var audio *libregraph.Audio + initAudio := func() { + if audio == nil { + audio = libregraph.NewAudio() + } + } + + if v, err := getFirstValue(meta, "xmpDM:album"); err == nil { + initAudio() + audio.SetAlbum(v) + } + + if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil { + initAudio() + audio.SetAlbumArtist(v) + } + + if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil { + initAudio() + audio.SetArtist(v) + } + + if v, err := getFirstValue(meta, "audio:bitrate"); err == nil { + // tika emits bits per second, graph wants kbps + if bps, err := strconv.ParseInt(v, 10, 64); err == nil && bps > 0 { + initAudio() + audio.SetBitrate(int64(math.Round(float64(bps) / 1000))) + } + } + + if v, err := getFirstValue(meta, "xmpDM:composer"); err == nil { + initAudio() + audio.SetComposers(v) + } + + if v, err := getFirstValue(meta, "xmpDM:copyright"); err == nil { + initAudio() + audio.SetCopyright(v) + } + + if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetDisc(int32(i)) + } + } + + if v, err := getFirstValue(meta, "audio:disc-count"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetDiscCount(int32(i)) + } + } + + if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil { + // Tika emits fractional seconds. + if f, err := strconv.ParseFloat(v, 64); err == nil { + initAudio() + audio.SetDuration(int64(math.Round(f * 1000))) + } + } + + if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil { + initAudio() + audio.SetGenre(v) + } + + if v, err := getFirstValue(meta, "audio:has-drm"); err == nil { + if b, err := strconv.ParseBool(v); err == nil { + initAudio() + audio.SetHasDrm(b) + } + } + + if v, err := getFirstValue(meta, "audio:is-variable-bitrate"); err == nil { + if b, err := strconv.ParseBool(v); err == nil { + initAudio() + audio.SetIsVariableBitrate(b) + } + } + + if v, err := getFirstValue(meta, "dc:title"); err == nil { + initAudio() + audio.SetTitle(v) + } + + if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetTrack(int32(i)) + } + } + + if v, err := getFirstValue(meta, "audio:track-count"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetTrackCount(int32(i)) + } + } + + if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetYear(int32(i)) + } else if d, err := time.Parse(time.DateOnly, v); err == nil { + initAudio() + audio.SetYear(int32(d.Year())) + } + } + + return audio +} diff --git a/services/search/pkg/content/tika_audio_test.go b/services/search/pkg/content/tika_audio_test.go new file mode 100644 index 0000000000..02f85719f1 --- /dev/null +++ b/services/search/pkg/content/tika_audio_test.go @@ -0,0 +1,80 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getAudio", func() { + It("maps the audio metadata to the audio facet", func() { + meta := map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "xmpDM:genre": {"Some Genre"}, + "xmpDM:album": {"Some Album"}, + "xmpDM:trackNumber": {"7"}, + "xmpDM:discNumber": {"4"}, + "xmpDM:releaseDate": {"2004"}, + "xmpDM:artist": {"Some Artist"}, + "xmpDM:albumArtist": {"Some AlbumArtist"}, + "dc:title": {"Some Title"}, + "xmpDM:duration": {"225.5"}, + "xmpDM:composer": {"Some Composers"}, + "xmpDM:copyright": {"Some Copyright"}, + "audio:bitrate": {"192000"}, + "audio:is-variable-bitrate": {"true"}, + "audio:has-drm": {"false"}, + "audio:track-count": {"9"}, + "audio:disc-count": {"5"}, + } + + audio := Tika{}.getAudio(meta) + Expect(audio).ToNot(BeNil()) + + Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) + Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) + Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) + Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright"))) + Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4))) + Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) + Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500))) + Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) + Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) + Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) + Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) + Expect(audio.Track).To(Equal(libregraph.PtrInt32(7))) + Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9))) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + + It("returns nil when no audio metadata is present", func() { + Expect(Tika{}.getAudio(map[string][]string{})).To(BeNil()) + }) + + It("returns nil for non-audio content types", func() { + Expect(Tika{}.getAudio(map[string][]string{ + "Content-Type": {"application/pdf"}, + "dc:title": {"quarterly report"}, + })).To(BeNil()) + }) + + It("takes the year from a full release date", func() { + audio := Tika{}.getAudio(map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "xmpDM:releaseDate": {"2004-06-01"}, + }) + Expect(audio).ToNot(BeNil()) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + + It("rounds the bitrate to the nearest kbps", func() { + audio := Tika{}.getAudio(map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "audio:bitrate": {"191999"}, + }) + Expect(audio).ToNot(BeNil()) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + }) +}) diff --git a/services/search/pkg/content/tika_image.go b/services/search/pkg/content/tika_image.go new file mode 100644 index 0000000000..b5ce2a4037 --- /dev/null +++ b/services/search/pkg/content/tika_image.go @@ -0,0 +1,32 @@ +package content + +import ( + "strconv" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getImage(meta map[string][]string) *libregraph.Image { + var image *libregraph.Image + initImage := func() { + if image == nil { + image = libregraph.NewImage() + } + } + + if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initImage() + image.SetWidth(int32(i)) + } + } + + if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initImage() + image.SetHeight(int32(i)) + } + } + + return image +} diff --git a/services/search/pkg/content/tika_image_test.go b/services/search/pkg/content/tika_image_test.go new file mode 100644 index 0000000000..91456ea97c --- /dev/null +++ b/services/search/pkg/content/tika_image_test.go @@ -0,0 +1,23 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getImage", func() { + It("maps the image dimensions to the image facet", func() { + image := Tika{}.getImage(map[string][]string{ + "tiff:ImageWidth": {"100"}, + "tiff:ImageLength": {"200"}, + }) + Expect(image).ToNot(BeNil()) + Expect(image.Width).To(Equal(libregraph.PtrInt32(100))) + Expect(image.Height).To(Equal(libregraph.PtrInt32(200))) + }) + + It("returns nil when no image metadata is present", func() { + Expect(Tika{}.getImage(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_location.go b/services/search/pkg/content/tika_location.go new file mode 100644 index 0000000000..586a4731c7 --- /dev/null +++ b/services/search/pkg/content/tika_location.go @@ -0,0 +1,52 @@ +package content + +import ( + "fmt" + "math" + "strconv" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +// graph geoCoordinates.altitude is in feet, exif GPS altitude (geo:alt) in metres. +const metresToFeet = 3.280839895 + +func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { + // the facet needs a sane coordinate pair, an altitude alone is useless + lat, latErr := parseCoordinate(meta, "geo:lat", 90) + long, longErr := parseCoordinate(meta, "geo:long", 180) + if latErr != nil || longErr != nil { + return nil + } + + location := libregraph.NewGeoCoordinates() + location.SetLatitude(lat) + location.SetLongitude(long) + + // tika emits metres (negative below sea level), graph wants feet + if v, err := getFirstValue(meta, "geo:alt"); err == nil { + if metres, err := strconv.ParseFloat(v, 64); err == nil { + location.SetAltitude(metres * metresToFeet) + } + } + + return location +} + +func parseCoordinate(meta map[string][]string, key string, limit float64) (float64, error) { + v, err := getFirstValue(meta, key) + if err != nil { + return 0, err + } + + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0, err + } + // ParseFloat accepts "NaN", which json cannot marshal + if math.IsNaN(f) || math.Abs(f) > limit { + return 0, fmt.Errorf("%s out of range: %v", key, f) + } + + return f, nil +} diff --git a/services/search/pkg/content/tika_location_test.go b/services/search/pkg/content/tika_location_test.go new file mode 100644 index 0000000000..413dae0c73 --- /dev/null +++ b/services/search/pkg/content/tika_location_test.go @@ -0,0 +1,48 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getLocation", func() { + It("maps lat/long and converts altitude from metres to feet", func() { + metres := 227.4 + location := Tika{}.getLocation(map[string][]string{ + "geo:lat": {"49.48675890884328"}, + "geo:long": {"11.103870357204285"}, + "geo:alt": {"227.4"}, + }) + Expect(location).ToNot(BeNil()) + Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) + Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet))) + }) + + It("keeps below-sea-level altitude negative", func() { + metres := -227.4 + location := Tika{}.getLocation(map[string][]string{ + "geo:lat": {"31.5"}, + "geo:long": {"35.47"}, + "geo:alt": {"-227.4"}, + }) + Expect(location).ToNot(BeNil()) + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet))) + }) + + It("returns nil for an incomplete coordinate pair", func() { + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:alt": {"227.4"}})).To(BeNil()) + }) + + It("returns nil for out-of-range coordinates", func() { + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"91"}, "geo:long": {"11.1"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}, "geo:long": {"-180.5"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"NaN"}, "geo:long": {"11.1"}})).To(BeNil()) + }) + + It("returns nil when no location metadata is present", func() { + Expect(Tika{}.getLocation(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_photo.go b/services/search/pkg/content/tika_photo.go new file mode 100644 index 0000000000..2ee961a300 --- /dev/null +++ b/services/search/pkg/content/tika_photo.go @@ -0,0 +1,74 @@ +package content + +import ( + "math" + "strconv" + "time" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo { + var photo *libregraph.Photo + initPhoto := func() { + if photo == nil { + photo = libregraph.NewPhoto() + } + } + + if v, err := getFirstValue(meta, "tiff:Make"); err == nil { + initPhoto() + photo.SetCameraMake(v) + } + + if v, err := getFirstValue(meta, "tiff:Model"); err == nil { + initPhoto() + photo.SetCameraModel(v) + } + + if v, err := getFirstValue(meta, "exif:FNumber"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initPhoto() + photo.SetFNumber(i) + } + } + + if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initPhoto() + photo.SetFocalLength(i) + } + } + + if v, err := getFirstValue(meta, "Base ISO"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initPhoto() + photo.SetIso(int32(i)) + } + } + + if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initPhoto() + photo.SetOrientation(int32(i)) + } + } + + if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil { + layout := "2006-01-02T15:04:05" + if t, err := time.Parse(layout, v); err == nil { + initPhoto() + photo.SetTakenDateTime(t) + } + } + + if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil && i > 0 { + initPhoto() + photo.SetExposureNumerator(1) + photo.SetExposureDenominator(math.Round(1 / i)) + } + } + + return photo +} diff --git a/services/search/pkg/content/tika_photo_test.go b/services/search/pkg/content/tika_photo_test.go new file mode 100644 index 0000000000..9995d59587 --- /dev/null +++ b/services/search/pkg/content/tika_photo_test.go @@ -0,0 +1,38 @@ +package content + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getPhoto", func() { + It("maps the exif metadata to the photo facet", func() { + photo := Tika{}.getPhoto(map[string][]string{ + "tiff:Make": {"Canon"}, + "tiff:Model": {"Canon EOS 5D"}, + "exif:ExposureTime": {"0.001"}, + "exif:FNumber": {"1.8"}, + "exif:FocalLength": {"50"}, + "Base ISO": {"100"}, + "tiff:Orientation": {"1"}, + "exif:DateTimeOriginal": {"2018-01-01T12:34:56"}, + }) + Expect(photo).ToNot(BeNil()) + Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon"))) + Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D"))) + Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1))) + Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000))) + Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8))) + Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50))) + Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100))) + Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1))) + Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC)))) + }) + + It("returns nil when no photo metadata is present", func() { + Expect(Tika{}.getPhoto(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index 286ffa8b9c..772ed185b3 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -7,7 +7,6 @@ import ( "net/http" "net/http/httptest" "strings" - "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" . "github.com/onsi/ginkgo/v2" @@ -24,17 +23,19 @@ import ( var _ = Describe("Tika", func() { Describe("extract", func() { var ( - body string - fullResponse string - language string - version string - srv *httptest.Server - tika *content.Tika + body string + fullResponse string + language string + tika4Language bool + version string + srv *httptest.Server + tika *content.Tika ) BeforeEach(func() { body = "" language = "" + tika4Language = false version = "" fullResponse = "" srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -42,6 +43,13 @@ var _ = Describe("Tika", func() { switch req.URL.Path { case "/version": out = version + case "/language": + if tika4Language { + out = language + } else { + w.WriteHeader(http.StatusNotFound) + return + } case "/language/string": out = language case "/rmeta/text": @@ -102,6 +110,17 @@ var _ = Describe("Tika", func() { Expect(doc.Title).To(Equal("quarterly report")) }) + It("adds the content of a tika 4", func() { + fullResponse = `[{"tk:content": "some data"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("some data")) + }) + It("adds the title of an older tika", func() { fullResponse = `[{"title": "quarterly report"}]` @@ -113,129 +132,17 @@ var _ = Describe("Tika", func() { Expect(doc.Title).To(Equal("quarterly report")) }) - It("adds audio content", func() { - fullResponse = `[ - { - "xmpDM:genre": "Some Genre", - "xmpDM:album": "Some Album", - "xmpDM:trackNumber": "7", - "xmpDM:discNumber": "4", - "xmpDM:releaseDate": "2004", - "xmpDM:artist": "Some Artist", - "xmpDM:albumArtist": "Some AlbumArtist", - "xmpDM:audioCompressor": "MP3", - "xmpDM:audioChannelType": "Stereo", - "version": "MPEG 3 Layer III Version 1", - "xmpDM:logComment": "some comment", - "xmpDM:audioSampleRate": "44100", - "channels": "2", - "dc:title": "Some Title", - "xmpDM:duration": "225.5", - "Content-Type": "audio/mpeg", - "samplerate": "44100" - } - ]` + It("removes stop words with a tika 4 language endpoint", func() { + body = "body to test stop words!!! against almost everyone" + language = "en" + tika4Language = true + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ Type: provider.ResourceType_RESOURCE_TYPE_FILE, Size: 1, }) Expect(err).ToNot(HaveOccurred()) - - audio := doc.Audio - Expect(audio).ToNot(BeNil()) - - Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) - Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) - Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) - // Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) - // Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) - // Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright"))) - Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4))) - // Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) - Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500))) - Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) - // Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) - // Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) - Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) - Expect(audio.Track).To(Equal(libregraph.PtrInt32(7))) - // Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9))) - Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) - - }) - - It("adds location content", func() { - fullResponse = `[ - { - "geo:lat": "49.48675890884328", - "geo:long": "11.103870357204285" - } - ]` - doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ - Type: provider.ResourceType_RESOURCE_TYPE_FILE, - Size: 1, - }) - Expect(err).ToNot(HaveOccurred()) - - location := doc.Location - Expect(location).ToNot(BeNil()) - - // TODO: Altitude is not supported right now - Expect(location.Altitude).To(BeNil()) - Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) - Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) - }) - - It("adds image content", func() { - fullResponse = `[ - { - "tiff:ImageWidth": "100", - "tiff:ImageLength": "100" - } - ]` - doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ - Type: provider.ResourceType_RESOURCE_TYPE_FILE, - Size: 1, - }) - Expect(err).ToNot(HaveOccurred()) - - image := doc.Image - Expect(image).ToNot(BeNil()) - - Expect(image.Width).To(Equal(libregraph.PtrInt32(100))) - Expect(image.Height).To(Equal(libregraph.PtrInt32(100))) - }) - - It("adds photo content", func() { - fullResponse = `[ - { - "tiff:Make": "Canon", - "tiff:Model": "Canon EOS 5D", - "exif:ExposureTime": "0.001", - "exif:FNumber": "1.8", - "exif:FocalLength": "50", - "Base ISO": "100", - "tiff:Orientation": "1", - "exif:DateTimeOriginal": "2018-01-01T12:34:56" - } - ]` - doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ - Type: provider.ResourceType_RESOURCE_TYPE_FILE, - Size: 1, - }) - Expect(err).ToNot(HaveOccurred()) - - photo := doc.Photo - Expect(photo).ToNot(BeNil()) - - Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon"))) - Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D"))) - Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1))) - Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000))) - Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8))) - Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50))) - Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100))) - Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1))) - Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC)))) + Expect(doc.Content).To(Equal("body test stop words!!!")) }) It("removes stop words", func() { @@ -250,6 +157,53 @@ var _ = Describe("Tika", func() { Expect(doc.Content).To(Equal("body test stop words!!!")) }) + It("keeps the audio facet when an embedded resource follows", func() { + fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Audio).ToNot(BeNil()) + Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker"))) + Expect(doc.Image).ToNot(BeNil()) + }) + + It("adds no audio facet to non-audio documents", func() { + fullResponse = `[{"Content-Type": "application/pdf", "dc:title": "quarterly report"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Audio).To(BeNil()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + + It("prefers the tika 4 content key over the legacy one", func() { + fullResponse = `[{"tk:content": "new", "X-TIKA:content": "old"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("new")) + }) + + It("joins the content of all meta entries", func() { + fullResponse = `[{"tk:content": "one"}, {"X-TIKA:content": "two"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("one two")) + }) + It("keeps stop words", func() { body = "body to test stop words!!! against almost everyone" language = "en" diff --git a/tests/acceptance/docker/src/tika.yml b/tests/acceptance/docker/src/tika.yml index 4c9768a090..fe8eff86e8 100644 --- a/tests/acceptance/docker/src/tika.yml +++ b/tests/acceptance/docker/src/tika.yml @@ -5,4 +5,4 @@ services: - tika command: tika:9998 tika: - image: apache/tika:3.2.3.0-full + image: apache/tika:4.0.0-full