Compare commits

...

22 Commits

Author SHA1 Message Date
Viktor Scharf
53d0bb467b fix issue-2146 2026-01-15 16:57:26 +01:00
dependabot[bot]
dccc3a0f21 build(deps): bump github.com/sirupsen/logrus
Bumps [github.com/sirupsen/logrus](https://github.com/sirupsen/logrus) from 1.9.4-0.20230606125235-dd1b4c2e81af to 1.9.4.
- [Release notes](https://github.com/sirupsen/logrus/releases)
- [Changelog](https://github.com/sirupsen/logrus/blob/master/CHANGELOG.md)
- [Commits](https://github.com/sirupsen/logrus/commits/v1.9.4)

---
updated-dependencies:
- dependency-name: github.com/sirupsen/logrus
  dependency-version: 1.9.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 16:48:11 +01:00
dependabot[bot]
22a7eaa005 build(deps): bump github.com/go-chi/chi/v5 from 5.2.3 to 5.2.4
Bumps [github.com/go-chi/chi/v5](https://github.com/go-chi/chi) from 5.2.3 to 5.2.4.
- [Release notes](https://github.com/go-chi/chi/releases)
- [Changelog](https://github.com/go-chi/chi/blob/master/CHANGELOG.md)
- [Commits](https://github.com/go-chi/chi/compare/v5.2.3...v5.2.4)

---
updated-dependencies:
- dependency-name: github.com/go-chi/chi/v5
  dependency-version: 5.2.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 15:17:16 +01:00
Michael Flemming
2f7542c36e Merge pull request #2159 from opencloud-eu/stablecitest-12.3.4 2026-01-15 13:36:15 +01:00
Viktor Scharf
3eac173644 fix flaky #2145 2026-01-15 13:23:44 +01:00
dependabot[bot]
42fd54dd35 build(deps): bump go.opentelemetry.io/contrib/zpages
Bumps [go.opentelemetry.io/contrib/zpages](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.63.0 to 0.64.0.
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/zpages/v0.63.0...zpages/v0.64.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/contrib/zpages
  dependency-version: 0.64.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 12:51:28 +01:00
dependabot[bot]
21207eba40 build(deps): bump github.com/blevesearch/bleve/v2 from 2.5.5 to 2.5.7
Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.5.5 to 2.5.7.
- [Release notes](https://github.com/blevesearch/bleve/releases)
- [Commits](https://github.com/blevesearch/bleve/compare/v2.5.5...v2.5.7)

---
updated-dependencies:
- dependency-name: github.com/blevesearch/bleve/v2
  dependency-version: 2.5.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 12:39:32 +01:00
Viktor Scharf
e33ff722f7 run wopi-validator tests localy (#2151) 2026-01-15 12:22:04 +01:00
dependabot[bot]
4b6e44d15c build(deps): bump go.opentelemetry.io/otel/exporters/stdout/stdouttrace
Bumps [go.opentelemetry.io/otel/exporters/stdout/stdouttrace](https://github.com/open-telemetry/opentelemetry-go) from 1.38.0 to 1.39.0.
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.38.0...v1.39.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/otel/exporters/stdout/stdouttrace
  dependency-version: 1.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 11:18:45 +01:00
dependabot[bot]
82033a0c2f build(deps): bump golang.org/x/image from 0.34.0 to 0.35.0
Bumps [golang.org/x/image](https://github.com/golang/image) from 0.34.0 to 0.35.0.
- [Commits](https://github.com/golang/image/compare/v0.34.0...v0.35.0)

---
updated-dependencies:
- dependency-name: golang.org/x/image
  dependency-version: 0.35.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 10:47:21 +01:00
dependabot[bot]
7759adb4a9 build(deps): bump github.com/nats-io/nats.go from 1.47.0 to 1.48.0
Bumps [github.com/nats-io/nats.go](https://github.com/nats-io/nats.go) from 1.47.0 to 1.48.0.
- [Release notes](https://github.com/nats-io/nats.go/releases)
- [Commits](https://github.com/nats-io/nats.go/compare/v1.47.0...v1.48.0)

---
updated-dependencies:
- dependency-name: github.com/nats-io/nats.go
  dependency-version: 1.48.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 09:45:56 +01:00
dependabot[bot]
0fc4af406b build(deps): bump github.com/onsi/ginkgo/v2 from 2.27.2 to 2.27.5
Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.27.2 to 2.27.5.
- [Release notes](https://github.com/onsi/ginkgo/releases)
- [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md)
- [Commits](https://github.com/onsi/ginkgo/compare/v2.27.2...v2.27.5)

---
updated-dependencies:
- dependency-name: github.com/onsi/ginkgo/v2
  dependency-version: 2.27.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-15 09:33:49 +01:00
Viktor Scharf
92884a4cbd fix ci format (#2149) 2026-01-14 20:21:08 +01:00
dependabot[bot]
063217c3e6 build(deps): bump github.com/olekukonko/tablewriter from 1.1.1 to 1.1.2
Bumps [github.com/olekukonko/tablewriter](https://github.com/olekukonko/tablewriter) from 1.1.1 to 1.1.2.
- [Commits](https://github.com/olekukonko/tablewriter/compare/v1.1.1...v1.1.2)

---
updated-dependencies:
- dependency-name: github.com/olekukonko/tablewriter
  dependency-version: 1.1.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-14 17:03:18 +01:00
dependabot[bot]
e9bd5c4058 build(deps): bump github.com/spf13/cobra from 1.10.1 to 1.10.2
Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.10.1 to 1.10.2.
- [Release notes](https://github.com/spf13/cobra/releases)
- [Commits](https://github.com/spf13/cobra/compare/v1.10.1...v1.10.2)

---
updated-dependencies:
- dependency-name: github.com/spf13/cobra
  dependency-version: 1.10.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-14 16:46:02 +01:00
Michael Flemming
df529acdce Merge pull request #2129 from opencloud-eu/add_docs_pr_gen_pipeline
add pipeline to autogenerate a docs pr for env var changes
2026-01-14 16:40:32 +01:00
Michael Barz
3654897f60 fix: markdown links formatting (#2143) 2026-01-14 16:19:32 +01:00
dependabot[bot]
84ff31c7b6 build(deps): bump golang.org/x/net from 0.48.0 to 0.49.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.48.0 to 0.49.0.
- [Commits](https://github.com/golang/net/compare/v0.48.0...v0.49.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-version: 0.49.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-14 15:40:20 +01:00
dependabot[bot]
123b641fcb build(deps): bump github.com/onsi/gomega from 1.38.2 to 1.39.0
Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.38.2 to 1.39.0.
- [Release notes](https://github.com/onsi/gomega/releases)
- [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md)
- [Commits](https://github.com/onsi/gomega/compare/v1.38.2...v1.39.0)

---
updated-dependencies:
- dependency-name: github.com/onsi/gomega
  dependency-version: 1.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-14 14:42:37 +01:00
dependabot[bot]
98d876b120 build(deps): bump golang.org/x/crypto from 0.46.0 to 0.47.0
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.46.0 to 0.47.0.
- [Commits](https://github.com/golang/crypto/compare/v0.46.0...v0.47.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.47.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-14 14:42:05 +01:00
opencloudeu
9264617a28 [tx] updated from transifex 2026-01-14 00:09:55 +00:00
dependabot[bot]
beab7ce18c build(deps): bump go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp
Bumps [go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.63.0 to 0.64.0.
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/zpages/v0.63.0...zpages/v0.64.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp
  dependency-version: 0.64.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-13 11:41:08 +01:00
151 changed files with 6677 additions and 4988 deletions

View File

@@ -24,6 +24,7 @@ OC_CI_NODEJS_ALPINE = "quay.io/opencloudeu/nodejs-alpine-ci:24"
OC_CI_PHP = "quay.io/opencloudeu/php-alpine-ci:%s"
OC_CI_WAIT_FOR = "quay.io/opencloudeu/wait-for-ci:latest"
OC_CS3_API_VALIDATOR = "opencloudeu/cs3api-validator:latest"
OC_CI_WOPI_VALIDATOR = "quay.io/opencloudeu/wopi-validator-ci:latest"
OC_LITMUS = "owncloudci/litmus:latest"
ONLYOFFICE_DOCUMENT_SERVER = "onlyoffice/documentserver:7.5.1"
PLUGINS_DOCKER_BUILDX = "woodpeckerci/plugin-docker-buildx:latest"
@@ -1137,7 +1138,7 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty =
for testgroup in testgroups:
validatorTests.append({
"name": "wopiValidatorTests-%s" % testgroup,
"image": "owncloudci/wopi-validator",
"image": OC_CI_WOPI_VALIDATOR,
"commands": [
"export WOPI_TOKEN=$(cat accesstoken)",
"echo $WOPI_TOKEN",
@@ -1153,7 +1154,7 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty =
for builtinOnlyGroup in builtinOnlyTestGroups:
validatorTests.append({
"name": "wopiValidatorTests-%s" % builtinOnlyGroup,
"image": "owncloudci/wopi-validator",
"image": OC_CI_WOPI_VALIDATOR,
"commands": [
"export WOPI_TOKEN=$(cat accesstoken)",
"echo $WOPI_TOKEN",

46
go.mod
View File

@@ -11,7 +11,7 @@ require (
github.com/Nerzal/gocloak/v13 v13.9.0
github.com/bbalet/stopwords v1.0.0
github.com/beevik/etree v1.6.0
github.com/blevesearch/bleve/v2 v2.5.5
github.com/blevesearch/bleve/v2 v2.5.7
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/coreos/go-oidc/v3 v3.17.0
github.com/cs3org/go-cs3apis v0.0.0-20250908152307-4ca807afe54e
@@ -20,7 +20,7 @@ require (
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
github.com/gabriel-vasile/mimetype v1.4.12
github.com/ggwhite/go-masker v1.1.0
github.com/go-chi/chi/v5 v5.2.3
github.com/go-chi/chi/v5 v5.2.4
github.com/go-chi/render v1.0.3
github.com/go-jose/go-jose/v3 v3.0.4
github.com/go-ldap/ldap/v3 v3.4.12
@@ -55,12 +55,12 @@ require (
github.com/mna/pigeon v1.3.0
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
github.com/nats-io/nats-server/v2 v2.12.3
github.com/nats-io/nats.go v1.47.0
github.com/nats-io/nats.go v1.48.0
github.com/oklog/run v1.2.0
github.com/olekukonko/tablewriter v1.1.1
github.com/olekukonko/tablewriter v1.1.2
github.com/onsi/ginkgo v1.16.5
github.com/onsi/ginkgo/v2 v2.27.2
github.com/onsi/gomega v1.38.2
github.com/onsi/ginkgo/v2 v2.27.5
github.com/onsi/gomega v1.39.0
github.com/open-policy-agent/opa v1.11.1
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76
@@ -75,9 +75,9 @@ require (
github.com/rogpeppe/go-internal v1.14.1
github.com/rs/cors v1.11.1
github.com/rs/zerolog v1.34.0
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af
github.com/sirupsen/logrus v1.9.4
github.com/spf13/afero v1.15.0
github.com/spf13/cobra v1.10.1
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
@@ -95,21 +95,21 @@ require (
go-micro.dev/v4 v4.11.0
go.etcd.io/bbolt v1.4.3
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0
go.opentelemetry.io/contrib/zpages v0.63.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0
go.opentelemetry.io/contrib/zpages v0.64.0
go.opentelemetry.io/otel v1.39.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0
go.opentelemetry.io/otel/sdk v1.39.0
go.opentelemetry.io/otel/trace v1.39.0
golang.org/x/crypto v0.46.0
golang.org/x/crypto v0.47.0
golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac
golang.org/x/image v0.34.0
golang.org/x/net v0.48.0
golang.org/x/image v0.35.0
golang.org/x/net v0.49.0
golang.org/x/oauth2 v0.34.0
golang.org/x/sync v0.19.0
golang.org/x/term v0.38.0
golang.org/x/text v0.32.0
golang.org/x/term v0.39.0
golang.org/x/text v0.33.0
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217
google.golang.org/grpc v1.78.0
google.golang.org/protobuf v1.36.11
@@ -157,7 +157,7 @@ require (
github.com/blevesearch/zapx/v13 v13.4.2 // indirect
github.com/blevesearch/zapx/v14 v14.4.2 // indirect
github.com/blevesearch/zapx/v15 v15.4.2 // indirect
github.com/blevesearch/zapx/v16 v16.2.7 // indirect
github.com/blevesearch/zapx/v16 v16.2.8 // indirect
github.com/bluele/gcache v0.0.2 // indirect
github.com/bombsimon/logrusr/v3 v3.1.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -165,9 +165,9 @@ require (
github.com/ceph/go-ceph v0.37.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73 // indirect
github.com/clipperhouse/displaywidth v0.3.1 // indirect
github.com/clipperhouse/displaywidth v0.6.0 // indirect
github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
github.com/cloudflare/circl v1.6.1 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
@@ -312,7 +312,7 @@ require (
github.com/nxadm/tail v1.4.8 // indirect
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
github.com/olekukonko/errors v1.1.0 // indirect
github.com/olekukonko/ll v0.1.2 // indirect
github.com/olekukonko/ll v0.1.3 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
@@ -390,10 +390,10 @@ require (
go.uber.org/zap v1.27.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.30.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/mod v0.31.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.39.0 // indirect
golang.org/x/tools v0.40.0 // indirect
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect

92
go.sum
View File

@@ -151,8 +151,8 @@ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6
github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4=
github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blevesearch/bleve/v2 v2.5.5 h1:lzC89QUCco+y1qBnJxGqm4AbtsdsnlUvq0kXok8n3C8=
github.com/blevesearch/bleve/v2 v2.5.5/go.mod h1:t5WoESS5TDteTdnjhhvpA1BpLYErOBX2IQViTMLK7wo=
github.com/blevesearch/bleve/v2 v2.5.7 h1:2d9YrL5zrX5EBBW++GOaEKjE+NPWeZGaX77IM26m1Z8=
github.com/blevesearch/bleve/v2 v2.5.7/go.mod h1:yj0NlS7ocGC4VOSAedqDDMktdh2935v2CSWOCDMHdSA=
github.com/blevesearch/bleve_index_api v1.2.11 h1:bXQ54kVuwP8hdrXUSOnvTQfgK0KI1+f9A0ITJT8tX1s=
github.com/blevesearch/bleve_index_api v1.2.11/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
github.com/blevesearch/geo v0.2.4 h1:ECIGQhw+QALCZaDcogRTNSJYQXRtC8/m8IKiA706cqk=
@@ -185,8 +185,8 @@ github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT
github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8=
github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k=
github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
github.com/blevesearch/zapx/v16 v16.2.7 h1:xcgFRa7f/tQXOwApVq7JWgPYSlzyUMmkuYa54tMDuR0=
github.com/blevesearch/zapx/v16 v16.2.7/go.mod h1:murSoCJPCk25MqURrcJaBQ1RekuqSCSfMjXH4rHyA14=
github.com/blevesearch/zapx/v16 v16.2.8 h1:SlnzF0YGtSlrsOE3oE7EgEX6BIepGpeqxs1IjMbHLQI=
github.com/blevesearch/zapx/v16 v16.2.8/go.mod h1:murSoCJPCk25MqURrcJaBQ1RekuqSCSfMjXH4rHyA14=
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
@@ -223,12 +223,12 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/clipperhouse/displaywidth v0.3.1 h1:k07iN9gD32177o1y4O1jQMzbLdCrsGJh+blirVYybsk=
github.com/clipperhouse/displaywidth v0.3.1/go.mod h1:tgLJKKyaDOCadywag3agw4snxS5kYEuYR6Y9+qWDDYM=
github.com/clipperhouse/displaywidth v0.6.0 h1:k32vueaksef9WIKCNcoqRNyKbyvkvkysNYnAWz2fN4s=
github.com/clipperhouse/displaywidth v0.6.0/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o=
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
@@ -381,8 +381,8 @@ github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkPro
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s=
@@ -916,8 +916,8 @@ github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
github.com/nats-io/nats-server/v2 v2.12.3 h1:KRv+1n7lddMVgkJPQer+pt36TcO0ENxjilBmeWdjcHs=
github.com/nats-io/nats-server/v2 v2.12.3/go.mod h1:MQXjG9WjyXKz9koWzUc3jYUMKD8x3CLmTNy91IQQz3Y=
github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM=
github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -940,23 +940,23 @@ github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM=
github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
github.com/olekukonko/ll v0.1.2 h1:lkg/k/9mlsy0SxO5aC+WEpbdT5K83ddnNhAepz7TQc0=
github.com/olekukonko/ll v0.1.2/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew=
github.com/olekukonko/ll v0.1.3 h1:sV2jrhQGq5B3W0nENUISCR6azIPf7UBUpVq0x/y70Fg=
github.com/olekukonko/ll v0.1.3/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/olekukonko/tablewriter v1.1.1 h1:b3reP6GCfrHwmKkYwNRFh2rxidGHcT6cgxj/sHiDDx0=
github.com/olekukonko/tablewriter v1.1.1/go.mod h1:De/bIcTF+gpBDB3Alv3fEsZA+9unTsSzAg/ZGADCtn4=
github.com/olekukonko/tablewriter v1.1.2 h1:L2kI1Y5tZBct/O/TyZK1zIE9GlBj/TVs+AY5tZDCDSc=
github.com/olekukonko/tablewriter v1.1.2/go.mod h1:z7SYPugVqGVavWoA2sGsFIoOVNmEHxUAAMrhXONtfkg=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
github.com/onsi/ginkgo/v2 v2.27.5 h1:ZeVgZMx2PDMdJm/+w5fE/OyG6ILo1Y3e+QX4zSR0zTE=
github.com/onsi/ginkgo/v2 v2.27.5/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q=
github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4=
github.com/open-policy-agent/opa v1.11.1 h1:4bMlG6DjRZTRAswRyF+KUCgxHu1Gsk0h9EbZ4W9REvM=
github.com/open-policy-agent/opa v1.11.1/go.mod h1:QimuJO4T3KYxWzrmAymqlFvsIanCjKrGjmmC8GgAdgE=
github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a h1:Sakl76blJAaM6NxylVkgSzktjo2dS504iDotEFJsh3M=
@@ -1138,8 +1138,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0=
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
@@ -1164,8 +1164,8 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -1311,10 +1311,10 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0 h1:RN3ifU8y4prNWeEnQp2kRRHz8UwonAEYZl8tUzHEXAk=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0/go.mod h1:habDz3tEWiFANTo6oUE99EmaFUrCNYAAg3wiVmusm70=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
go.opentelemetry.io/contrib/zpages v0.63.0 h1:TppOKuZGbqXMgsfjqq3i09N5Vbo1JLtLImUqiTPGnX4=
go.opentelemetry.io/contrib/zpages v0.63.0/go.mod h1:5F8uugz75ay/MMhRRhxAXY33FuaI8dl7jTxefrIy5qk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ=
go.opentelemetry.io/contrib/zpages v0.64.0 h1:iMybqKVR8AHHxFX4DuEWJ9dY75+9E7+IPwUK3Ll7NxM=
go.opentelemetry.io/contrib/zpages v0.64.0/go.mod h1:DnkiyoQ7Yx/NmmKn10b6M2YBXreUqq0qhFa/kYgSZME=
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0=
@@ -1323,8 +1323,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 h1:in9O8
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0/go.mod h1:Rp0EXBm5tfnv0WL+ARyO/PHBEaEAT8UUHQ6AGJcSq6c=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0 h1:8UPA4IbVZxpsD76ihGOQiFml99GPAEZLohDXvqHdi6U=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0/go.mod h1:MZ1T/+51uIVKlRzGw1Fo46KEWThjlCBZKl2LzY5nv4g=
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
@@ -1375,8 +1375,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1392,8 +1392,8 @@ golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScy
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
golang.org/x/image v0.35.0 h1:LKjiHdgMtO8z7Fh18nGY6KDcoEtVfsgLDPeLyguqb7I=
golang.org/x/image v0.35.0/go.mod h1:MwPLTVgvxSASsxdLzKrl8BRFuyqMyGhLwmC+TO1Sybk=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1418,8 +1418,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1473,8 +1473,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1586,8 +1586,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1599,8 +1599,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1615,8 +1615,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1679,8 +1679,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@@ -5,6 +5,7 @@ import (
"bytes"
"fmt"
"io"
"regexp"
"strings"
)
@@ -77,7 +78,7 @@ func (md MD) WriteToc(w io.Writer) (int64, error) {
// main title not in toc
continue
}
link := fmt.Sprintf("#%s", strings.ToLower(strings.Replace(h.Header, " ", "-", -1)))
link := fmt.Sprintf("#%s", toAnchor(h.Header))
s := fmt.Sprintf("%s* [%s](%s)\n", strings.Repeat(" ", h.Level-2), h.Header, link)
n, err := w.Write([]byte(s))
if err != nil {
@@ -137,3 +138,12 @@ func headingFromString(s string) Heading {
Header: strings.TrimPrefix(con, " "),
}
}
func toAnchor(header string) string {
// Remove everything except letters, numbers, and spaces
reg := regexp.MustCompile(`[^a-zA-Z0-9 ]+`)
anchor := reg.ReplaceAllString(header, "")
// Replace spaces with hyphens and convert to lowercase
anchor = strings.ReplaceAll(anchor, " ", "-")
return strings.ToLower(anchor)
}

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-12-25 00:05+0000\n"
"POT-Creation-Date: 2026-01-14 00:09+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: ii kaka, 2025\n"
"Language-Team: Japanese (https://app.transifex.com/opencloud-eu/teams/204053/ja/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-12-25 00:05+0000\n"
"POT-Creation-Date: 2026-01-14 00:09+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: ii kaka, 2025\n"
"Language-Team: Japanese (https://app.transifex.com/opencloud-eu/teams/204053/ja/)\n"

View File

@@ -585,3 +585,34 @@ By default, the system uses `posix` storage. However, you can override this by s
```bash
STORAGE_DRIVER=posix ./tests/acceptance/run_api_tests.sh
```
## Running WOPI Validator Tests
### Available Test Groups
```text
BaseWopiViewing
CheckFileInfoSchema
EditFlows
Locks
AccessTokens
GetLock
ExtendedLockLength
FileVersion
Features
PutRelativeFile
RenameFileIfCreateChildFileIsNotSupported
```
### Run Test
```bash
TEST_GROUP=BaseWopiViewing docker compose -f tests/acceptance/docker/src/wopi-validator-test.yml up -d
```
### for macOS use arm image
```bash
WOPI_VALIDATOR_IMAGE=scharfvi/wopi-validator \
TEST_GROUP=BaseWopiViewing \
docker compose -f tests/acceptance/docker/src/wopi-validator-test.yml up -d
```

View File

@@ -100,7 +100,7 @@ class HttpRequestHelper {
$parsedUrl = parse_url($url);
$baseUrl = $parsedUrl['scheme'] . '://' . $parsedUrl['host'];
$baseUrl .= isset($parsedUrl['port']) ? ':' . $parsedUrl['port'] : '';
$testUrl = $baseUrl . "/graph/v1.0/use/$user";
$testUrl = $baseUrl . "/graph/v1.0/user/$user";
if (OcHelper::isTestingOnReva()) {
$url = $baseUrl . "/ocs/v2.php/cloud/users/$user";
}

View File

@@ -1325,6 +1325,20 @@ trait WebDav {
if ($statusCode === 404 || $statusCode === 405) {
return;
}
// After MOVE the source path might still be visible for a short time
// We wait 1 second and retry once to avoid flaky failures.
if ($statusCode === 207) {
sleep(1);
$response = $this->listFolder(
$user,
$path,
'0',
null,
null,
$type
);
$statusCode = $response->getStatusCode();
}
if ($statusCode === 207) {
$responseXmlObject = HttpRequestHelper::getResponseXml(
$response,

View File

@@ -0,0 +1,45 @@
#!/bin/sh
set -xe
if [ -z "$TEST_GROUP" ]; then
echo "TEST_GROUP not set"
exit 1
fi
echo "Waiting for collaboration WOPI endpoint..."
until curl -s http://collaboration:9304 >/dev/null; do
echo "Waiting for collaboration WOPI endpoint..."
sleep 2
done
echo "Collaboration is up"
if [ -z "$OC_URL" ]; then
OC_URL="https://opencloud-server:9200"
fi
curl -vk -X DELETE "$OC_URL/remote.php/webdav/test.wopitest" -u admin:admin
curl -vk -X PUT "$OC_URL/remote.php/webdav/test.wopitest" -u admin:admin -D headers.txt
cat headers.txt
FILE_ID="$(cat headers.txt | sed -n -e 's/^.*oc-fileid: //Ip')"
export FILE_ID
URL="$OC_URL/app/open?app_name=FakeOffice&file_id=$FILE_ID"
URL="$(echo "$URL" | tr -d '[:cntrl:]')"
export URL
curl -vk -X POST "$URL" -u admin:admin > open.json
cat open.json
cat open.json | jq .form_parameters.access_token | tr -d '"' > accesstoken
cat open.json | jq .form_parameters.access_token_ttl | tr -d '"' > accesstokenttl
WOPI_FILE_ID="$(cat open.json | jq .app_url | sed -n -e 's/^.*files%2F//p' | tr -d '"')"
echo "http://collaboration:9300/wopi/files/$WOPI_FILE_ID" > wopisrc
WOPI_TOKEN=$(cat accesstoken)
export WOPI_TOKEN
WOPI_TTL=$(cat accesstokenttl)
export WOPI_TTL
WOPI_SRC=$(cat wopisrc)
export WOPI_SRC
/app/Microsoft.Office.WopiValidator -s -t "$WOPI_TOKEN" -w "$WOPI_SRC" -l "$WOPI_TTL" --testgroup $TEST_GROUP

View File

@@ -0,0 +1,86 @@
services:
fakeoffice:
image: owncloudci/alpine:latest
entrypoint: /bin/sh
command:
[
"-c",
"while true; do echo -e \"HTTP/1.1 200 OK\n\n$(cat /hosting-discovery.xml)\" | nc -l -k -p 8080; done",
]
ports:
- 8080:8080
extra_hosts:
- opencloud.local:${DOCKER_HOST:-host-gateway}
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080"]
volumes:
- ./../../../config/woodpecker/hosting-discovery.xml:/hosting-discovery.xml
opencloud:
image: opencloudeu/opencloud:dev
container_name: opencloud-server
ports:
- 9200:9200
entrypoint: /bin/sh
command: ["-c", "opencloud init || true; sleep 10; opencloud server"]
environment:
OC_URL: https://opencloud-server:9200
OC_CONFIG_DIR: /etc/opencloud
STORAGE_USERS_DRIVER: posix
PROXY_ENABLE_BASIC_AUTH: true
OC_LOG_LEVEL: error
OC_LOG_COLOR: false
OC_INSECURE: true
IDM_ADMIN_PASSWORD: admin
GATEWAY_GRPC_ADDR: 0.0.0.0:9142
NATS_NATS_HOST: 0.0.0.0
NATS_NATS_PORT: 9233
volumes:
- config:/etc/opencloud
depends_on:
fakeoffice:
condition: service_healthy
collaboration:
image: opencloudeu/opencloud:dev
restart: unless-stopped
ports:
- 9300:9300
entrypoint:
- /bin/sh
command: ["-c", "opencloud collaboration server"]
environment:
OC_CONFIG_DIR: /etc/opencloud
MICRO_REGISTRY: nats-js-kv
MICRO_REGISTRY_ADDRESS: opencloud:9233
COLLABORATION_LOG_LEVEL: info
COLLABORATION_GRPC_ADDR: 0.0.0.0:9301
COLLABORATION_HTTP_ADDR: 0.0.0.0:9300
COLLABORATION_DEBUG_ADDR: 0.0.0.0:9304
COLLABORATION_APP_PROOF_DISABLE: true
COLLABORATION_APP_INSECURE: true
COLLABORATION_CS3API_DATAGATEWAY_INSECURE: true
COLLABORATION_WOPI_SECRET: some-wopi-secret
COLLABORATION_SERVICE_NAME: collaboration-fakeoffice
COLLABORATION_APP_NAME: FakeOffice
COLLABORATION_APP_PRODUCT: Microsoft
COLLABORATION_APP_ADDR: http://fakeoffice:8080
COLLABORATION_WOPI_SRC: http://collaboration:9300
volumes:
- config:/etc/opencloud
depends_on:
- opencloud
wopi-validator:
image: ${WOPI_VALIDATOR_IMAGE:-opencloudeu/wopi-validator-ci}
volumes:
- ./run-wopi-validator.sh:/app/run-wopi-validator.sh
environment:
TEST_GROUP: ${TEST_GROUP:-PutRelativeFile}
entrypoint: /app/run-wopi-validator.sh
depends_on:
- collaboration
restart: "on-failure"
volumes:
config:

View File

@@ -1,25 +0,0 @@
sudo: false
language: go
go:
- "1.21.x"
- "1.22.x"
- "1.23.x"
script:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- go get github.com/kisielk/errcheck
- go get -u github.com/FiloSottile/gvt
- gvt restore
- go test -race -v $(go list ./... | grep -v vendor/)
- go vet $(go list ./... | grep -v vendor/)
- go test ./test -v -indexType scorch
- errcheck -ignorepkg fmt $(go list ./... | grep -v vendor/);
- scripts/project-code-coverage.sh
- scripts/build_children.sh
notifications:
email:
- fts-team@couchbase.com

View File

@@ -1,7 +1,7 @@
# ![bleve](docs/bleve.png) bleve
[![Tests](https://github.com/blevesearch/bleve/actions/workflows/tests.yml/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve/actions/workflows/tests.yml?query=event%3Apush+branch%3Amaster)
[![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg?branch=master)](https://coveralls.io/github/blevesearch/bleve?branch=master)
[![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg)](https://coveralls.io/github/blevesearch/bleve)
[![Go Reference](https://pkg.go.dev/badge/github.com/blevesearch/bleve/v2.svg)](https://pkg.go.dev/github.com/blevesearch/bleve/v2)
[![Join the chat](https://badges.gitter.im/join_chat.svg)](https://app.gitter.im/#/room/#blevesearch_bleve:gitter.im)
[![Go Report Card](https://goreportcard.com/badge/github.com/blevesearch/bleve/v2)](https://goreportcard.com/report/github.com/blevesearch/bleve/v2)

View File

@@ -180,7 +180,7 @@ func NewGeoShapeFieldFromShapeWithIndexingOptions(name string, arrayPositions []
// docvalues are always enabled for geoshape fields, even if the
// indexing options are set to not include docvalues.
options = options | index.DocValues
options |= index.DocValues
return &GeoShapeField{
shape: shape,
@@ -232,7 +232,7 @@ func NewGeometryCollectionFieldFromShapesWithIndexingOptions(name string,
// docvalues are always enabled for geoshape fields, even if the
// indexing options are set to not include docvalues.
options = options | index.DocValues
options |= index.DocValues
return &GeoShapeField{
shape: shape,

View File

@@ -109,6 +109,10 @@ func NewVectorField(name string, arrayPositions []uint64,
func NewVectorFieldWithIndexingOptions(name string, arrayPositions []uint64,
vector []float32, dims int, similarity, vectorIndexOptimizedFor string,
options index.FieldIndexingOptions) *VectorField {
// ensure the options are set to not store/index term vectors/doc values
options &^= index.StoreField | index.IncludeTermVectors | index.DocValues
// skip freq/norms for vector field
options |= index.SkipFreqNorm
return &VectorField{
name: name,

View File

@@ -17,113 +17,125 @@ package fusion
import (
"fmt"
"sort"
"github.com/blevesearch/bleve/v2/search"
)
// formatRRFMessage builds the explanation string for a single component of the
// Reciprocal Rank Fusion calculation.
func formatRRFMessage(weight float64, rank int, rankConstant int) string {
return fmt.Sprintf("rrf score (weight=%.3f, rank=%d, rank_constant=%d), normalized score of", weight, rank, rankConstant)
}
// ReciprocalRankFusion performs a reciprocal rank fusion on the search results.
func ReciprocalRankFusion(hits search.DocumentMatchCollection, weights []float64, rankConstant int, windowSize int, numKNNQueries int, explain bool) FusionResult {
if len(hits) == 0 {
return FusionResult{
Hits: hits,
// ReciprocalRankFusion applies Reciprocal Rank Fusion across the primary FTS
// results and each KNN sub-query. Ranks are limited to `windowSize` per source,
// weighted, and combined into a single fused score, with optional explanation
// details.
func ReciprocalRankFusion(hits search.DocumentMatchCollection, weights []float64, rankConstant int, windowSize int, numKNNQueries int, explain bool) *FusionResult {
nHits := len(hits)
if nHits == 0 || windowSize == 0 {
return &FusionResult{
Hits: search.DocumentMatchCollection{},
Total: 0,
MaxScore: 0.0,
}
}
// Create a map of document ID to a slice of ranks.
// The first element of the slice is the rank from the FTS search,
// and the subsequent elements are the ranks from the KNN searches.
docRanks := make(map[string][]int)
limit := min(nHits, windowSize)
// Pre-assign rank lists to each candidate document
for _, hit := range hits {
docRanks[hit.ID] = make([]int, numKNNQueries+1)
// precompute rank+scores to prevent additional division ops later
rankReciprocals := make([]float64, limit)
for i := range rankReciprocals {
rankReciprocals[i] = 1.0 / float64(rankConstant+i+1)
}
// Only a max of `window_size` elements need to be counted for. Stop
// calculating rank once this threshold is hit.
sort.Slice(hits, func(a, b int) bool {
return scoreSortFunc()(hits[a], hits[b]) < 0
})
// Only consider top windowSize docs for rescoring
for i := range min(windowSize, len(hits)) {
if hits[i].Score != 0.0 {
// Skip if Score is 0, since that means the document was not
// found as part of FTS, and only in KNN.
docRanks[hits[i].ID][0] = i + 1
// init explanations if required
var fusionExpl map[*search.DocumentMatch][]*search.Explanation
if explain {
fusionExpl = make(map[*search.DocumentMatch][]*search.Explanation, nHits)
}
// The code here mainly deals with obtaining rank/score for fts hits.
// First sort hits by score
sortDocMatchesByScore(hits)
// Calculate fts rank+scores
ftsWeight := weights[0]
for i := 0; i < nHits; i++ {
if i < windowSize {
hit := hits[i]
// No fts scores from this hit onwards, break loop
if hit.Score == 0.0 {
break
}
contrib := ftsWeight * rankReciprocals[i]
hit.Score = contrib
if explain {
expl := getFusionExplAt(
hit,
0,
contrib,
formatRRFMessage(ftsWeight, i+1, rankConstant),
)
fusionExpl[hit] = append(fusionExpl[hit], expl)
}
} else {
// These FTS hits are not counted in the results, so set to 0
hits[i].Score = 0.0
}
}
// Allocate knnDocs and reuse it within the loop
knnDocs := make([]*search.DocumentMatch, 0, len(hits))
// Code from here is to calculate knn ranks and scores
// iterate over each knn query and calculate knn rank+scores
for queryIdx := 0; queryIdx < numKNNQueries; queryIdx++ {
knnWeight := weights[queryIdx+1]
// Sorts hits in decreasing order of hit.ScoreBreakdown[i]
sortDocMatchesByBreakdown(hits, queryIdx)
// For each KNN query, rank the documents based on their KNN score.
for i := range numKNNQueries {
knnDocs = knnDocs[:0]
for i := 0; i < nHits; i++ {
// break if score breakdown doesn't exist (sort function puts these hits at the end)
// or if we go past the windowSize
_, scoreBreakdownExists := scoreBreakdownForQuery(hits[i], queryIdx)
if i >= windowSize || !scoreBreakdownExists {
break
}
for _, hit := range hits {
if _, ok := hit.ScoreBreakdown[i]; ok {
knnDocs = append(knnDocs, hit)
hit := hits[i]
contrib := knnWeight * rankReciprocals[i]
hit.Score += contrib
if explain {
expl := getFusionExplAt(
hit,
queryIdx+1,
contrib,
formatRRFMessage(knnWeight, i+1, rankConstant),
)
fusionExpl[hit] = append(fusionExpl[hit], expl)
}
}
// Sort the documents based on their score for this KNN query.
sort.Slice(knnDocs, func(a, b int) bool {
return scoreBreakdownSortFunc(i)(knnDocs[a], knnDocs[b]) < 0
})
// Update the ranks of the documents in the docRanks map.
// Only consider top windowSize docs for rescoring.
for j := range min(windowSize, len(knnDocs)) {
docRanks[knnDocs[j].ID][i+1] = j + 1
}
}
// Calculate the RRF score for each document.
var maxScore float64
for _, hit := range hits {
var rrfScore float64
var explChildren []*search.Explanation
if explain {
explChildren = make([]*search.Explanation, 0, numKNNQueries+1)
finalizeFusionExpl(hit, fusionExpl[hit])
}
for i, rank := range docRanks[hit.ID] {
if rank > 0 {
partialRrfScore := weights[i] * 1.0 / float64(rankConstant+rank)
if explain {
expl := getFusionExplAt(
hit,
i,
partialRrfScore,
formatRRFMessage(weights[i], rank, rankConstant),
)
explChildren = append(explChildren, expl)
}
rrfScore += partialRrfScore
}
}
hit.Score = rrfScore
hit.ScoreBreakdown = nil
if rrfScore > maxScore {
maxScore = rrfScore
}
if explain {
finalizeFusionExpl(hit, explChildren)
if hit.Score > maxScore {
maxScore = hit.Score
}
}
sort.Sort(hits)
if len(hits) > windowSize {
sortDocMatchesByScore(hits)
if nHits > windowSize {
hits = hits[:windowSize]
}
return FusionResult{
return &FusionResult{
Hits: hits,
Total: uint64(len(hits)),
MaxScore: maxScore,

View File

@@ -16,145 +16,147 @@ package fusion
import (
"fmt"
"sort"
"github.com/blevesearch/bleve/v2/search"
)
// formatRSFMessage builds the explanation string associated with a single
// component of the Relative Score Fusion calculation.
func formatRSFMessage(weight float64, normalizedScore float64, minScore float64, maxScore float64) string {
return fmt.Sprintf("rsf score (weight=%.3f, normalized=%.6f, min=%.6f, max=%.6f), normalized score of",
weight, normalizedScore, minScore, maxScore)
}
// RelativeScoreFusion normalizes scores based on min/max values for FTS and each KNN query, then applies weights.
func RelativeScoreFusion(hits search.DocumentMatchCollection, weights []float64, windowSize int, numKNNQueries int, explain bool) FusionResult {
if len(hits) == 0 {
return FusionResult{
Hits: hits,
// RelativeScoreFusion normalizes the best-scoring documents from the primary
// FTS query and each KNN query, scales those normalized values by the supplied
// weights, and combines them into a single fused score. Only the top
// `windowSize` documents per source are considered, and explanations are
// materialized lazily when requested.
func RelativeScoreFusion(hits search.DocumentMatchCollection, weights []float64, windowSize int, numKNNQueries int, explain bool) *FusionResult {
nHits := len(hits)
if nHits == 0 || windowSize == 0 {
return &FusionResult{
Hits: search.DocumentMatchCollection{},
Total: 0,
MaxScore: 0.0,
}
}
rsfScores := make(map[string]float64)
// contains the docs under consideration for scoring.
// Reused for fts and knn hits
scoringDocs := make([]*search.DocumentMatch, 0, len(hits))
var explMap map[string][]*search.Explanation
// init explanations if required
var fusionExpl map[*search.DocumentMatch][]*search.Explanation
if explain {
explMap = make(map[string][]*search.Explanation)
fusionExpl = make(map[*search.DocumentMatch][]*search.Explanation, nHits)
}
// remove non-fts hits
// Code here for calculating fts results
// Sort by fts scores
sortDocMatchesByScore(hits)
// ftsLimit holds the total number of fts hits to consider for rsf
ftsLimit := 0
for _, hit := range hits {
if hit.Score != 0.0 {
scoringDocs = append(scoringDocs, hit)
if hit.Score == 0.0 {
break
}
ftsLimit++
}
// sort hits by fts score
sort.Slice(scoringDocs, func(a, b int) bool {
return scoreSortFunc()(scoringDocs[a], scoringDocs[b]) < 0
})
// Reslice to correct size
if len(scoringDocs) > windowSize {
scoringDocs = scoringDocs[:windowSize]
}
ftsLimit = min(ftsLimit, windowSize)
var min, max float64
if len(scoringDocs) > 0 {
min, max = scoringDocs[len(scoringDocs)-1].Score, scoringDocs[0].Score
}
// calculate fts scores
if ftsLimit > 0 {
max := hits[0].Score
min := hits[ftsLimit-1].Score
denom := max - min
weight := weights[0]
for _, hit := range scoringDocs {
var tempRsfScore float64
if max > min {
tempRsfScore = (hit.Score - min) / (max - min)
} else {
tempRsfScore = 1.0
}
if explain {
// create and replace new explanation
expl := getFusionExplAt(
hit,
0,
tempRsfScore,
formatRSFMessage(weights[0], tempRsfScore, min, max),
)
explMap[hit.ID] = append(explMap[hit.ID], expl)
}
rsfScores[hit.ID] = weights[0] * tempRsfScore
}
for i := range numKNNQueries {
scoringDocs = scoringDocs[:0]
for _, hit := range hits {
if _, exists := hit.ScoreBreakdown[i]; exists {
scoringDocs = append(scoringDocs, hit)
for i := 0; i < ftsLimit; i++ {
hit := hits[i]
norm := 1.0
if denom > 0 {
norm = (hit.Score - min) / denom
}
}
sort.Slice(scoringDocs, func(a, b int) bool {
return scoreBreakdownSortFunc(i)(scoringDocs[a], scoringDocs[b]) < 0
})
if len(scoringDocs) > windowSize {
scoringDocs = scoringDocs[:windowSize]
}
if len(scoringDocs) > 0 {
min, max = scoringDocs[len(scoringDocs)-1].ScoreBreakdown[i], scoringDocs[0].ScoreBreakdown[i]
} else {
min, max = 0.0, 0.0
}
for _, hit := range scoringDocs {
var tempRsfScore float64
if max > min {
tempRsfScore = (hit.ScoreBreakdown[i] - min) / (max - min)
} else {
tempRsfScore = 1.0
}
contrib := weight * norm
if explain {
expl := getFusionExplAt(
hit,
i+1,
tempRsfScore,
formatRSFMessage(weights[i+1], tempRsfScore, min, max),
0,
norm,
formatRSFMessage(weight, norm, min, max),
)
explMap[hit.ID] = append(explMap[hit.ID], expl)
fusionExpl[hit] = append(fusionExpl[hit], expl)
}
rsfScores[hit.ID] += weights[i+1] * tempRsfScore
hit.Score = contrib
}
for i := ftsLimit; i < nHits; i++ {
// These FTS hits are not counted in the results, so set to 0
hits[i].Score = 0.0
}
}
var maxScore float64
for _, hit := range hits {
if rsfScore, exists := rsfScores[hit.ID]; exists {
hit.Score = rsfScore
if rsfScore > maxScore {
maxScore = rsfScore
// Code from here is for calculating knn scores
for queryIdx := 0; queryIdx < numKNNQueries; queryIdx++ {
sortDocMatchesByBreakdown(hits, queryIdx)
// knnLimit holds the total number of knn hits retrieved for a specific knn query
knnLimit := 0
for _, hit := range hits {
if _, ok := scoreBreakdownForQuery(hit, queryIdx); !ok {
break
}
if explain {
finalizeFusionExpl(hit, explMap[hit.ID])
}
} else {
hit.Score = 0.0
knnLimit++
}
knnLimit = min(knnLimit, windowSize)
// if limit is 0, skip calculating
if knnLimit == 0 {
continue
}
max, _ := scoreBreakdownForQuery(hits[0], queryIdx)
min, _ := scoreBreakdownForQuery(hits[knnLimit-1], queryIdx)
denom := max - min
weight := weights[queryIdx+1]
for i := 0; i < knnLimit; i++ {
hit := hits[i]
score, _ := scoreBreakdownForQuery(hit, queryIdx)
norm := 1.0
if denom > 0 {
norm = (score - min) / denom
}
contrib := weight * norm
if explain {
expl := getFusionExplAt(
hit,
queryIdx+1,
norm,
formatRSFMessage(weight, norm, min, max),
)
fusionExpl[hit] = append(fusionExpl[hit], expl)
}
hit.Score += contrib
}
}
// Finalize scores
var maxScore float64
for _, hit := range hits {
if explain {
finalizeFusionExpl(hit, fusionExpl[hit])
}
if hit.Score > maxScore {
maxScore = hit.Score
}
hit.ScoreBreakdown = nil
}
sort.Sort(hits)
sortDocMatchesByScore(hits)
if len(hits) > windowSize {
if nHits > windowSize {
hits = hits[:windowSize]
}
return FusionResult{
return &FusionResult{
Hits: hits,
Total: uint64(len(hits)),
MaxScore: maxScore,

View File

@@ -16,70 +16,82 @@
package fusion
import (
"sort"
"github.com/blevesearch/bleve/v2/search"
)
// scoreBreakdownSortFunc returns a comparison function for sorting DocumentMatch objects
// by their ScoreBreakdown at the specified index in descending order.
// In case of ties, documents with lower HitNumber (earlier hits) are preferred.
// If either document is missing the ScoreBreakdown for the specified index,
// it's treated as having a score of 0.0.
func scoreBreakdownSortFunc(idx int) func(i, j *search.DocumentMatch) int {
return func(i, j *search.DocumentMatch) int {
// Safely extract scores, defaulting to 0.0 if missing
iScore := 0.0
jScore := 0.0
if i.ScoreBreakdown != nil {
if score, ok := i.ScoreBreakdown[idx]; ok {
iScore = score
}
}
if j.ScoreBreakdown != nil {
if score, ok := j.ScoreBreakdown[idx]; ok {
jScore = score
}
}
// Sort by score in descending order (higher scores first)
if iScore > jScore {
return -1
} else if iScore < jScore {
return 1
}
// Break ties by HitNumber in ascending order (lower HitNumber wins)
if i.HitNumber < j.HitNumber {
return -1
} else if i.HitNumber > j.HitNumber {
return 1
}
return 0 // Equal scores and HitNumbers
// sortDocMatchesByScore orders the provided collection in-place by the primary
// score in descending order, breaking ties with the original `HitNumber` to
// ensure deterministic output.
func sortDocMatchesByScore(hits search.DocumentMatchCollection) {
if len(hits) < 2 {
return
}
sort.Slice(hits, func(a, b int) bool {
i := hits[a]
j := hits[b]
if i.Score == j.Score {
return i.HitNumber < j.HitNumber
}
return i.Score > j.Score
})
}
func scoreSortFunc() func(i, j *search.DocumentMatch) int {
return func(i, j *search.DocumentMatch) int {
// Sort by score in descending order
if i.Score > j.Score {
return -1
} else if i.Score < j.Score {
return 1
}
// Break ties by HitNumber
if i.HitNumber < j.HitNumber {
return -1
} else if i.HitNumber > j.HitNumber {
return 1
}
return 0
// scoreBreakdownForQuery fetches the score for a specific KNN query index from
// the provided hit. The boolean return indicates whether the score is present.
func scoreBreakdownForQuery(hit *search.DocumentMatch, idx int) (float64, bool) {
if hit == nil || hit.ScoreBreakdown == nil {
return 0, false
}
score, ok := hit.ScoreBreakdown[idx]
return score, ok
}
// sortDocMatchesByBreakdown orders the hits in-place using the KNN score for
// the supplied query index (descending), breaking ties with `HitNumber` and
// placing hits without a score at the end.
func sortDocMatchesByBreakdown(hits search.DocumentMatchCollection, queryIdx int) {
if len(hits) < 2 {
return
}
sort.SliceStable(hits, func(a, b int) bool {
left := hits[a]
right := hits[b]
var leftScore float64
leftOK := false
if left != nil && left.ScoreBreakdown != nil {
leftScore, leftOK = left.ScoreBreakdown[queryIdx]
}
var rightScore float64
rightOK := false
if right != nil && right.ScoreBreakdown != nil {
rightScore, rightOK = right.ScoreBreakdown[queryIdx]
}
if leftOK && rightOK {
if leftScore == rightScore {
return left.HitNumber < right.HitNumber
}
return leftScore > rightScore
}
if leftOK != rightOK {
return leftOK
}
return left.HitNumber < right.HitNumber
})
}
// getFusionExplAt copies the existing explanation child at the requested index
// and wraps it in a new node describing how the fusion algorithm adjusted the
// score.
func getFusionExplAt(hit *search.DocumentMatch, i int, value float64, message string) *search.Explanation {
return &search.Explanation{
Value: value,
@@ -88,6 +100,9 @@ func getFusionExplAt(hit *search.DocumentMatch, i int, value float64, message st
}
}
// finalizeFusionExpl installs the collection of fusion explanation children and
// updates the root message so the caller sees the fused score as the sum of its
// parts.
func finalizeFusionExpl(hit *search.DocumentMatch, explChildren []*search.Explanation) {
hit.Expl.Children = explChildren

View File

@@ -35,43 +35,45 @@ type Event struct {
// EventKind represents an event code for OnEvent() callbacks.
type EventKind int
// EventKindCloseStart is fired when a Scorch.Close() has begun.
var EventKindCloseStart = EventKind(1)
const (
// EventKindCloseStart is fired when a Scorch.Close() has begun.
EventKindCloseStart EventKind = iota
// EventKindClose is fired when a scorch index has been fully closed.
var EventKindClose = EventKind(2)
// EventKindClose is fired when a scorch index has been fully closed.
EventKindClose
// EventKindMergerProgress is fired when the merger has completed a
// round of merge processing.
var EventKindMergerProgress = EventKind(3)
// EventKindMergerProgress is fired when the merger has completed a
// round of merge processing.
EventKindMergerProgress
// EventKindPersisterProgress is fired when the persister has completed
// a round of persistence processing.
var EventKindPersisterProgress = EventKind(4)
// EventKindPersisterProgress is fired when the persister has completed
// a round of persistence processing.
EventKindPersisterProgress
// EventKindBatchIntroductionStart is fired when Batch() is invoked which
// introduces a new segment.
var EventKindBatchIntroductionStart = EventKind(5)
// EventKindBatchIntroductionStart is fired when Batch() is invoked which
// introduces a new segment.
EventKindBatchIntroductionStart
// EventKindBatchIntroduction is fired when Batch() completes.
var EventKindBatchIntroduction = EventKind(6)
// EventKindBatchIntroduction is fired when Batch() completes.
EventKindBatchIntroduction
// EventKindMergeTaskIntroductionStart is fired when the merger is about to
// start the introduction of merged segment from a single merge task.
var EventKindMergeTaskIntroductionStart = EventKind(7)
// EventKindMergeTaskIntroductionStart is fired when the merger is about to
// start the introduction of merged segment from a single merge task.
EventKindMergeTaskIntroductionStart
// EventKindMergeTaskIntroduction is fired when the merger has completed
// the introduction of merged segment from a single merge task.
var EventKindMergeTaskIntroduction = EventKind(8)
// EventKindMergeTaskIntroduction is fired when the merger has completed
// the introduction of merged segment from a single merge task.
EventKindMergeTaskIntroduction
// EventKindPreMergeCheck is fired before the merge begins to check if
// the caller should proceed with the merge.
var EventKindPreMergeCheck = EventKind(9)
// EventKindPreMergeCheck is fired before the merge begins to check if
// the caller should proceed with the merge.
EventKindPreMergeCheck
// EventKindIndexStart is fired when Index() is invoked which
// creates a new Document object from an interface using the index mapping.
var EventKindIndexStart = EventKind(10)
// EventKindIndexStart is fired when Index() is invoked which
// creates a new Document object from an interface using the index mapping.
EventKindIndexStart
// EventKindPurgerCheck is fired before the purge code is invoked and decides
// whether to execute or not. For unit test purposes
var EventKindPurgerCheck = EventKind(11)
// EventKindPurgerCheck is fired before the purge code is invoked and decides
// whether to execute or not. For unit test purposes
EventKindPurgerCheck
)

View File

@@ -24,6 +24,8 @@ import (
segment "github.com/blevesearch/scorch_segment_api/v2"
)
const introducer = "introducer"
type segmentIntroduction struct {
id uint64
data segment.Segment
@@ -50,10 +52,11 @@ type epochWatcher struct {
func (s *Scorch) introducerLoop() {
defer func() {
if r := recover(); r != nil {
s.fireAsyncError(&AsyncPanicError{
Source: "introducer",
Path: s.path,
})
s.fireAsyncError(NewScorchError(
introducer,
fmt.Sprintf("panic: %v, path: %s", r, s.path),
ErrAsyncPanic,
))
}
s.asyncTasks.Done()

View File

@@ -29,13 +29,16 @@ import (
segment "github.com/blevesearch/scorch_segment_api/v2"
)
const merger = "merger"
func (s *Scorch) mergerLoop() {
defer func() {
if r := recover(); r != nil {
s.fireAsyncError(&AsyncPanicError{
Source: "merger",
Path: s.path,
})
s.fireAsyncError(NewScorchError(
merger,
fmt.Sprintf("panic: %v, path: %s", r, s.path),
ErrAsyncPanic,
))
}
s.asyncTasks.Done()
@@ -45,7 +48,11 @@ func (s *Scorch) mergerLoop() {
var ctrlMsg *mergerCtrl
mergePlannerOptions, err := s.parseMergePlannerOptions()
if err != nil {
s.fireAsyncError(fmt.Errorf("mergePlannerOption json parsing err: %v", err))
s.fireAsyncError(NewScorchError(
merger,
fmt.Sprintf("mergerPlannerOptions json parsing err: %v", err),
ErrOptionsParse,
))
return
}
ctrlMsgDflt := &mergerCtrl{ctx: context.Background(),
@@ -110,7 +117,12 @@ OUTER:
ctrlMsg = nil
break OUTER
}
s.fireAsyncError(fmt.Errorf("merging err: %v", err))
s.fireAsyncError(NewScorchError(
merger,
fmt.Sprintf("merging err: %v", err),
ErrPersist,
))
_ = ourSnapshot.DecRef()
atomic.AddUint64(&s.stats.TotFileMergeLoopErr, 1)
continue OUTER

View File

@@ -38,6 +38,8 @@ import (
bolt "go.etcd.io/bbolt"
)
const persister = "persister"
// DefaultPersisterNapTimeMSec is kept to zero as this helps in direct
// persistence of segments with the default safe batch option.
// If the default safe batch option results in high number of
@@ -95,10 +97,11 @@ type notificationChan chan struct{}
func (s *Scorch) persisterLoop() {
defer func() {
if r := recover(); r != nil {
s.fireAsyncError(&AsyncPanicError{
Source: "persister",
Path: s.path,
})
s.fireAsyncError(NewScorchError(
persister,
fmt.Sprintf("panic: %v, path: %s", r, s.path),
ErrAsyncPanic,
))
}
s.asyncTasks.Done()
@@ -112,7 +115,11 @@ func (s *Scorch) persisterLoop() {
po, err := s.parsePersisterOptions()
if err != nil {
s.fireAsyncError(fmt.Errorf("persisterOptions json parsing err: %v", err))
s.fireAsyncError(NewScorchError(
persister,
fmt.Sprintf("persisterOptions json parsing err: %v", err),
ErrOptionsParse,
))
return
}
@@ -173,7 +180,11 @@ OUTER:
// the retry attempt
unpersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...)
s.fireAsyncError(fmt.Errorf("got err persisting snapshot: %v", err))
s.fireAsyncError(NewScorchError(
persister,
fmt.Sprintf("got err persisting snapshot: %v", err),
ErrPersist,
))
_ = ourSnapshot.DecRef()
atomic.AddUint64(&s.stats.TotPersistLoopErr, 1)
continue OUTER
@@ -1060,13 +1071,21 @@ func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, erro
func (s *Scorch) removeOldData() {
removed, err := s.removeOldBoltSnapshots()
if err != nil {
s.fireAsyncError(fmt.Errorf("got err removing old bolt snapshots: %v", err))
s.fireAsyncError(NewScorchError(
persister,
fmt.Sprintf("got err removing old bolt snapshots: %v", err),
ErrCleanup,
))
}
atomic.AddUint64(&s.stats.TotSnapshotsRemovedFromMetaStore, uint64(removed))
err = s.removeOldZapFiles()
if err != nil {
s.fireAsyncError(fmt.Errorf("got err removing old zap files: %v", err))
s.fireAsyncError(NewScorchError(
persister,
fmt.Sprintf("got err removing old zap files: %v", err),
ErrCleanup,
))
}
}

View File

@@ -88,14 +88,45 @@ type Scorch struct {
spatialPlugin index.SpatialAnalyzerPlugin
}
// AsyncPanicError is passed to scorch asyncErrorHandler when panic occurs in scorch background process
type AsyncPanicError struct {
Source string
Path string
type ScorchErrorType string
func (t ScorchErrorType) Error() string {
return string(t)
}
func (e *AsyncPanicError) Error() string {
return fmt.Sprintf("%s panic when processing %s", e.Source, e.Path)
// ErrType values for ScorchError
const (
ErrAsyncPanic = ScorchErrorType("async panic error")
ErrPersist = ScorchErrorType("persist error")
ErrCleanup = ScorchErrorType("cleanup error")
ErrOptionsParse = ScorchErrorType("options parse error")
)
// ScorchError is passed to onAsyncError when errors are
// fired from scorch background processes
type ScorchError struct {
Source string
ErrMsg string
ErrType ScorchErrorType
}
func (e *ScorchError) Error() string {
return fmt.Sprintf("source: %s, %v: %s", e.Source, e.ErrType, e.ErrMsg)
}
// Lets the onAsyncError function verify what type of
// error is fired using errors.Is(...). This lets the function
// handle errors differently.
func (e *ScorchError) Unwrap() error {
return e.ErrType
}
func NewScorchError(source, errMsg string, errType ScorchErrorType) error {
return &ScorchError{
Source: source,
ErrMsg: errMsg,
ErrType: errType,
}
}
type internalStats struct {

View File

@@ -23,7 +23,6 @@ import (
"path/filepath"
"reflect"
"sort"
"strings"
"sync"
"sync/atomic"
@@ -147,7 +146,7 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
makeItr func(i segment.TermDictionary) segment.DictionaryIterator,
randomLookup bool,
) (*IndexSnapshotFieldDict, error) {
results := make(chan *asynchSegmentResult)
results := make(chan *asynchSegmentResult, len(is.segment))
var totalBytesRead uint64
var fieldCardinality int64
for _, s := range is.segment {
@@ -281,10 +280,13 @@ func (is *IndexSnapshot) FieldDictRange(field string, startTerm []byte,
// to use as the end key in a traditional (inclusive, exclusive]
// start/end range
func calculateExclusiveEndFromPrefix(in []byte) []byte {
if len(in) == 0 {
return nil
}
rv := make([]byte, len(in))
copy(rv, in)
for i := len(rv) - 1; i >= 0; i-- {
rv[i] = rv[i] + 1
rv[i]++
if rv[i] != 0 {
return rv // didn't overflow, so stop
}
@@ -391,7 +393,7 @@ func (is *IndexSnapshot) FieldDictContains(field string) (index.FieldDictContain
}
func (is *IndexSnapshot) DocIDReaderAll() (index.DocIDReader, error) {
results := make(chan *asynchSegmentResult)
results := make(chan *asynchSegmentResult, len(is.segment))
for index, segment := range is.segment {
go func(index int, segment *SegmentSnapshot) {
results <- &asynchSegmentResult{
@@ -405,7 +407,7 @@ func (is *IndexSnapshot) DocIDReaderAll() (index.DocIDReader, error) {
}
func (is *IndexSnapshot) DocIDReaderOnly(ids []string) (index.DocIDReader, error) {
results := make(chan *asynchSegmentResult)
results := make(chan *asynchSegmentResult, len(is.segment))
for index, segment := range is.segment {
go func(index int, segment *SegmentSnapshot) {
docs, err := segment.DocNumbers(ids)
@@ -451,7 +453,7 @@ func (is *IndexSnapshot) newDocIDReader(results chan *asynchSegmentResult) (inde
func (is *IndexSnapshot) Fields() ([]string, error) {
// FIXME not making this concurrent for now as it's not used in hot path
// of any searches at the moment (just a debug aid)
fieldsMap := map[string]struct{}{}
fieldsMap := make(map[string]struct{})
for _, segment := range is.segment {
fields := segment.Fields()
for _, field := range fields {
@@ -765,7 +767,7 @@ func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReade
is.m2.Lock()
if is.fieldTFRs == nil {
is.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{}
is.fieldTFRs = make(map[string][]*IndexSnapshotTermFieldReader)
}
if len(is.fieldTFRs[tfr.field]) < is.getFieldTFRCacheThreshold() {
tfr.bytesRead = 0
@@ -813,7 +815,7 @@ func (is *IndexSnapshot) documentVisitFieldTermsOnSegment(
// Filter out fields that have been completely deleted or had their
// docvalues data deleted from both visitable fields and required fields
filterUpdatedFields := func(fields []string) []string {
filteredFields := make([]string, 0)
filteredFields := make([]string, 0, len(fields))
for _, field := range fields {
if info, ok := is.updatedFields[field]; ok &&
(info.DocValues || info.Deleted) {
@@ -978,15 +980,17 @@ func subtractStrings(a, b []string) []string {
return a
}
// Create a map for O(1) lookups
bMap := make(map[string]struct{}, len(b))
for _, bs := range b {
bMap[bs] = struct{}{}
}
rv := make([]string, 0, len(a))
OUTER:
for _, as := range a {
for _, bs := range b {
if as == bs {
continue OUTER
}
if _, exists := bMap[as]; !exists {
rv = append(rv, as)
}
rv = append(rv, as)
}
return rv
}
@@ -1279,7 +1283,7 @@ func (is *IndexSnapshot) TermFrequencies(field string, limit int, descending boo
sort.Slice(termFreqs, func(i, j int) bool {
if termFreqs[i].Frequency == termFreqs[j].Frequency {
// If frequencies are equal, sort by term lexicographically
return strings.Compare(termFreqs[i].Term, termFreqs[j].Term) < 0
return termFreqs[i].Term < termFreqs[j].Term
}
if descending {
return termFreqs[i].Frequency > termFreqs[j].Frequency

View File

@@ -37,14 +37,10 @@ func (is *IndexSnapshot) VectorReader(ctx context.Context, vector []float32,
snapshot: is,
searchParams: searchParams,
eligibleSelector: eligibleSelector,
postings: make([]segment_api.VecPostingsList, len(is.segment)),
iterators: make([]segment_api.VecPostingsIterator, len(is.segment)),
}
if rv.postings == nil {
rv.postings = make([]segment_api.VecPostingsList, len(is.segment))
}
if rv.iterators == nil {
rv.iterators = make([]segment_api.VecPostingsIterator, len(is.segment))
}
// initialize postings and iterators within the OptimizeVR's Finish()
return rv, nil
}

View File

@@ -18,7 +18,6 @@ import (
"context"
"fmt"
"sort"
"strings"
"sync"
"time"
@@ -905,7 +904,7 @@ func preSearchDataSearch(ctx context.Context, req *SearchRequest, flags *preSear
// which would happen in the case of an alias tree and depending on the level of the tree, the preSearchData
// needs to be redistributed to the indexes at that level
func redistributePreSearchData(req *SearchRequest, indexes []Index) (map[string]map[string]interface{}, error) {
rv := make(map[string]map[string]interface{})
rv := make(map[string]map[string]interface{}, len(indexes))
for _, index := range indexes {
rv[index.Name()] = make(map[string]interface{})
}
@@ -1202,23 +1201,16 @@ func (i *indexAliasImpl) TermFrequencies(field string, limit int, descending boo
})
}
if descending {
sort.Slice(rvTermFreqs, func(i, j int) bool {
if rvTermFreqs[i].Frequency == rvTermFreqs[j].Frequency {
// If frequencies are equal, sort by term lexicographically
return strings.Compare(rvTermFreqs[i].Term, rvTermFreqs[j].Term) < 0
}
sort.Slice(rvTermFreqs, func(i, j int) bool {
if rvTermFreqs[i].Frequency == rvTermFreqs[j].Frequency {
// If frequencies are equal, sort by term lexicographically
return rvTermFreqs[i].Term < rvTermFreqs[j].Term
}
if descending {
return rvTermFreqs[i].Frequency > rvTermFreqs[j].Frequency
})
} else {
sort.Slice(rvTermFreqs, func(i, j int) bool {
if rvTermFreqs[i].Frequency == rvTermFreqs[j].Frequency {
// If frequencies are equal, sort by term lexicographically
return strings.Compare(rvTermFreqs[i].Term, rvTermFreqs[j].Term) < 0
}
return rvTermFreqs[i].Frequency < rvTermFreqs[j].Frequency
})
}
}
return rvTermFreqs[i].Frequency < rvTermFreqs[j].Frequency
})
if limit > len(rvTermFreqs) {
limit = len(rvTermFreqs)
@@ -1272,25 +1264,22 @@ func (i *indexAliasImpl) CentroidCardinalities(field string, limit int, descendi
close(asyncResults)
}()
rvCentroidCardinalitiesResult := make([]index.CentroidCardinality, 0, limit)
rvCentroidCardinalities := make([]index.CentroidCardinality, 0, limit*len(i.indexes))
for asr := range asyncResults {
asr = append(asr, rvCentroidCardinalitiesResult...)
if descending {
sort.Slice(asr, func(i, j int) bool {
return asr[i].Cardinality > asr[j].Cardinality
})
} else {
sort.Slice(asr, func(i, j int) bool {
return asr[i].Cardinality < asr[j].Cardinality
})
}
if limit > len(asr) {
limit = len(asr)
}
rvCentroidCardinalitiesResult = asr[:limit]
rvCentroidCardinalities = append(rvCentroidCardinalities, asr...)
}
return rvCentroidCardinalitiesResult, nil
sort.Slice(rvCentroidCardinalities, func(i, j int) bool {
if descending {
return rvCentroidCardinalities[i].Cardinality > rvCentroidCardinalities[j].Cardinality
} else {
return rvCentroidCardinalities[i].Cardinality < rvCentroidCardinalities[j].Cardinality
}
})
if limit > len(rvCentroidCardinalities) {
limit = len(rvCentroidCardinalities)
}
return rvCentroidCardinalities[:limit], nil
}

View File

@@ -20,6 +20,7 @@ import (
"io"
"os"
"path/filepath"
"regexp"
"strconv"
"sync"
"sync/atomic"
@@ -859,6 +860,26 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
} else {
// build terms facet
facetBuilder := facet.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size)
// Set prefix filter if provided
if facetRequest.TermPrefix != "" {
facetBuilder.SetPrefixFilter(facetRequest.TermPrefix)
}
// Set regex filter if provided
if facetRequest.TermPattern != "" {
// Use cached compiled pattern if available, otherwise compile it now
if facetRequest.compiledPattern != nil {
facetBuilder.SetRegexFilter(facetRequest.compiledPattern)
} else {
regex, err := regexp.Compile(facetRequest.TermPattern)
if err != nil {
return nil, fmt.Errorf("error compiling regex pattern for facet '%s': %v", facetName, err)
}
facetBuilder.SetRegexFilter(regex)
}
}
facetsBuilder.Add(facetName, facetBuilder)
}
}
@@ -1304,6 +1325,9 @@ func (f *indexImplFieldDict) Cardinality() int {
// helper function to remove duplicate entries from slice of strings
func deDuplicate(fields []string) []string {
if len(fields) == 0 {
return fields
}
entries := make(map[string]struct{})
ret := []string{}
for _, entry := range fields {

View File

@@ -92,7 +92,7 @@ func DeletedFields(ori, upd *mapping.IndexMappingImpl) (map[string]*index.Update
// Compare both the mappings based on the document paths
// and create a list of index, docvalues, store differences
// for every single field possible
fieldInfo := make(map[string]*index.UpdateFieldInfo)
fieldInfo := make(map[string]*index.UpdateFieldInfo, len(oriPaths))
for path, info := range oriPaths {
err = addFieldInfo(fieldInfo, info, updPaths[path])
if err != nil {
@@ -109,13 +109,13 @@ func DeletedFields(ori, upd *mapping.IndexMappingImpl) (map[string]*index.Update
// A field cannot be completely deleted with any dynamic value turned on
if info.Deleted {
if upd.IndexDynamic {
return nil, fmt.Errorf("Mapping cannot be removed when index dynamic is true")
return nil, fmt.Errorf("mapping cannot be removed when index dynamic is true")
}
if upd.StoreDynamic {
return nil, fmt.Errorf("Mapping cannot be removed when store dynamic is true")
return nil, fmt.Errorf("mapping cannot be removed when store dynamic is true")
}
if upd.DocValuesDynamic {
return nil, fmt.Errorf("Mapping cannot be removed when docvalues dynamic is true")
return nil, fmt.Errorf("mapping cannot be removed when docvalues dynamic is true")
}
}
}
@@ -191,14 +191,14 @@ func checkUpdatedMapping(ori, upd *mapping.DocumentMapping) error {
// Simple checks to ensure no new field mappings present
// in updated
// Create a map of original field names for O(1) lookup
oriFieldNames := make(map[string]bool, len(ori.Fields))
for _, fMapping := range ori.Fields {
oriFieldNames[fMapping.Name] = true
}
for _, updFMapping := range upd.Fields {
var oriFMapping *mapping.FieldMapping
for _, fMapping := range ori.Fields {
if updFMapping.Name == fMapping.Name {
oriFMapping = fMapping
}
}
if oriFMapping == nil {
if !oriFieldNames[updFMapping.Name] {
return fmt.Errorf("updated index mapping contains new fields")
}
}
@@ -238,10 +238,8 @@ func addPathInfo(paths map[string]*pathInfo, name string, mp *mapping.DocumentMa
// Recursively add path information for all child mappings
for cName, cMapping := range mp.Properties {
var pathName string
if name == "" {
pathName = cName
} else {
pathName := cName
if name != "" {
pathName = name + "." + cName
}
addPathInfo(paths, pathName, cMapping, im, pInfo, rootName)
@@ -460,9 +458,6 @@ func addFieldInfo(fInfo map[string]*index.UpdateFieldInfo, ori, upd *pathInfo) e
}
}
}
if err != nil {
return err
}
return nil
}
@@ -567,19 +562,18 @@ func compareFieldMapping(original, updated *mapping.FieldMapping) (*index.Update
// In such a situation, any conflicting changes found will abort the update process
func validateFieldInfo(newInfo *index.UpdateFieldInfo, fInfo map[string]*index.UpdateFieldInfo,
ori *pathInfo, oriFMapInfo *fieldMapInfo) error {
// Determine field name
fieldName := oriFMapInfo.fieldMapping.Name
if fieldName == "" {
fieldName = oriFMapInfo.parent.path
}
// Construct full name with parent path
var name string
if oriFMapInfo.parent.parentPath == "" {
if oriFMapInfo.fieldMapping.Name == "" {
name = oriFMapInfo.parent.path
} else {
name = oriFMapInfo.fieldMapping.Name
}
name = fieldName
} else {
if oriFMapInfo.fieldMapping.Name == "" {
name = oriFMapInfo.parent.parentPath + "." + oriFMapInfo.parent.path
} else {
name = oriFMapInfo.parent.parentPath + "." + oriFMapInfo.fieldMapping.Name
}
name = oriFMapInfo.parent.parentPath + "." + fieldName
}
if (newInfo.Deleted || newInfo.Index || newInfo.DocValues || newInfo.Store) && ori.dynamic {
return fmt.Errorf("updated field is under a dynamic property")

View File

@@ -52,7 +52,7 @@ type DocumentMapping struct {
}
func (dm *DocumentMapping) Validate(cache *registry.Cache,
parentName string, fieldAliasCtx map[string]*FieldMapping,
path []string, fieldAliasCtx map[string]*FieldMapping,
) error {
var err error
if dm.DefaultAnalyzer != "" {
@@ -68,11 +68,7 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache,
}
}
for propertyName, property := range dm.Properties {
newParent := propertyName
if parentName != "" {
newParent = fmt.Sprintf("%s.%s", parentName, propertyName)
}
err = property.Validate(cache, newParent, fieldAliasCtx)
err = property.Validate(cache, append(path, propertyName), fieldAliasCtx)
if err != nil {
return err
}
@@ -96,7 +92,7 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache,
return err
}
}
err := validateFieldMapping(field, parentName, fieldAliasCtx)
err := validateFieldMapping(field, path, fieldAliasCtx)
if err != nil {
return err
}

View File

@@ -191,13 +191,16 @@ func (im *IndexMappingImpl) Validate() error {
return err
}
}
// fieldAliasCtx is used to detect any field alias conflicts across the entire mapping
// the map will hold the fully qualified field name to FieldMapping, so we can
// check for conflicts as we validate each DocumentMapping.
fieldAliasCtx := make(map[string]*FieldMapping)
err = im.DefaultMapping.Validate(im.cache, "", fieldAliasCtx)
err = im.DefaultMapping.Validate(im.cache, []string{}, fieldAliasCtx)
if err != nil {
return err
}
for _, docMapping := range im.TypeMapping {
err = docMapping.Validate(im.cache, "", fieldAliasCtx)
err = docMapping.Validate(im.cache, []string{}, fieldAliasCtx)
if err != nil {
return err
}

View File

@@ -38,7 +38,7 @@ func (fm *FieldMapping) processVectorBase64(propertyMightBeVector interface{},
// -----------------------------------------------------------------------------
// document validation functions
func validateFieldMapping(field *FieldMapping, parentName string,
func validateFieldMapping(field *FieldMapping, path []string,
fieldAliasCtx map[string]*FieldMapping) error {
return validateFieldType(field)
}

View File

@@ -20,6 +20,7 @@ package mapping
import (
"fmt"
"reflect"
"slices"
"github.com/blevesearch/bleve/v2/document"
"github.com/blevesearch/bleve/v2/util"
@@ -141,15 +142,27 @@ func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
if !ok {
return false
}
// Apply defaults for similarity and optimization if not set
similarity := fm.Similarity
if similarity == "" {
similarity = index.DefaultVectorSimilarityMetric
}
vectorIndexOptimizedFor := fm.VectorIndexOptimizedFor
if vectorIndexOptimizedFor == "" {
vectorIndexOptimizedFor = index.DefaultIndexOptimization
}
// normalize raw vector if similarity is cosine
if fm.Similarity == index.CosineSimilarity {
vector = NormalizeVector(vector)
// Since the vector can be multi-vector (flattened array of multiple vectors),
// we use NormalizeMultiVector to normalize each sub-vector independently.
if similarity == index.CosineSimilarity {
vector = NormalizeMultiVector(vector, fm.Dims)
}
fieldName := getFieldName(pathString, path, fm)
options := fm.Options()
field := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, vector,
fm.Dims, fm.Similarity, fm.VectorIndexOptimizedFor, options)
fm.Dims, similarity, vectorIndexOptimizedFor, options)
context.doc.AddField(field)
// "_all" composite field is not applicable for vector field
@@ -163,20 +176,29 @@ func (fm *FieldMapping) processVectorBase64(propertyMightBeVectorBase64 interfac
if !ok {
return
}
// Apply defaults for similarity and optimization if not set
similarity := fm.Similarity
if similarity == "" {
similarity = index.DefaultVectorSimilarityMetric
}
vectorIndexOptimizedFor := fm.VectorIndexOptimizedFor
if vectorIndexOptimizedFor == "" {
vectorIndexOptimizedFor = index.DefaultIndexOptimization
}
decodedVector, err := document.DecodeVector(encodedString)
if err != nil || len(decodedVector) != fm.Dims {
return
}
// normalize raw vector if similarity is cosine
if fm.Similarity == index.CosineSimilarity {
// normalize raw vector if similarity is cosine, multi-vector is not supported
// for base64 encoded vectors, so we use NormalizeVector directly.
if similarity == index.CosineSimilarity {
decodedVector = NormalizeVector(decodedVector)
}
fieldName := getFieldName(pathString, path, fm)
options := fm.Options()
field := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, decodedVector,
fm.Dims, fm.Similarity, fm.VectorIndexOptimizedFor, options)
fm.Dims, similarity, vectorIndexOptimizedFor, options)
context.doc.AddField(field)
// "_all" composite field is not applicable for vector_base64 field
@@ -186,87 +208,121 @@ func (fm *FieldMapping) processVectorBase64(propertyMightBeVectorBase64 interfac
// -----------------------------------------------------------------------------
// document validation functions
func validateFieldMapping(field *FieldMapping, parentName string,
func validateFieldMapping(field *FieldMapping, path []string,
fieldAliasCtx map[string]*FieldMapping) error {
switch field.Type {
case "vector", "vector_base64":
return validateVectorFieldAlias(field, parentName, fieldAliasCtx)
return validateVectorFieldAlias(field, path, fieldAliasCtx)
default: // non-vector field
return validateFieldType(field)
}
}
func validateVectorFieldAlias(field *FieldMapping, parentName string,
func validateVectorFieldAlias(field *FieldMapping, path []string,
fieldAliasCtx map[string]*FieldMapping) error {
if field.Name == "" {
field.Name = parentName
// fully qualified field name
pathString := encodePath(path)
// check if field has a name set, else use path to compute effective name
effectiveFieldName := getFieldName(pathString, path, field)
// Compute effective values for validation
effectiveSimilarity := field.Similarity
if effectiveSimilarity == "" {
effectiveSimilarity = index.DefaultVectorSimilarityMetric
}
effectiveOptimizedFor := field.VectorIndexOptimizedFor
if effectiveOptimizedFor == "" {
effectiveOptimizedFor = index.DefaultIndexOptimization
}
if field.Similarity == "" {
field.Similarity = index.DefaultVectorSimilarityMetric
}
if field.VectorIndexOptimizedFor == "" {
field.VectorIndexOptimizedFor = index.DefaultIndexOptimization
}
if _, exists := index.SupportedVectorIndexOptimizations[field.VectorIndexOptimizedFor]; !exists {
// if an unsupported config is provided, override to default
field.VectorIndexOptimizedFor = index.DefaultIndexOptimization
}
// following fields are not applicable for vector
// thus, we set them to default values
field.IncludeInAll = false
field.IncludeTermVectors = false
field.Store = false
field.DocValues = false
field.SkipFreqNorm = true
// # If alias is present, validate the field options as per the alias
// # If alias is present, validate the field options as per the alias.
// note: reading from a nil map is safe
if fieldAlias, ok := fieldAliasCtx[field.Name]; ok {
if fieldAlias, ok := fieldAliasCtx[effectiveFieldName]; ok {
if field.Dims != fieldAlias.Dims {
return fmt.Errorf("field: '%s', invalid alias "+
"(different dimensions %d and %d)", fieldAlias.Name, field.Dims,
"(different dimensions %d and %d)", effectiveFieldName, field.Dims,
fieldAlias.Dims)
}
if field.Similarity != fieldAlias.Similarity {
// Compare effective similarity values
aliasSimilarity := fieldAlias.Similarity
if aliasSimilarity == "" {
aliasSimilarity = index.DefaultVectorSimilarityMetric
}
if effectiveSimilarity != aliasSimilarity {
return fmt.Errorf("field: '%s', invalid alias "+
"(different similarity values %s and %s)", fieldAlias.Name,
field.Similarity, fieldAlias.Similarity)
"(different similarity values %s and %s)", effectiveFieldName,
effectiveSimilarity, aliasSimilarity)
}
// Compare effective vector index optimization values
aliasOptimizedFor := fieldAlias.VectorIndexOptimizedFor
if aliasOptimizedFor == "" {
aliasOptimizedFor = index.DefaultIndexOptimization
}
if effectiveOptimizedFor != aliasOptimizedFor {
return fmt.Errorf("field: '%s', invalid alias "+
"(different vector index optimization values %s and %s)", effectiveFieldName,
effectiveOptimizedFor, aliasOptimizedFor)
}
return nil
}
// # Validate field options
// Vector dimensions must be within allowed range
if field.Dims < MinVectorDims || field.Dims > MaxVectorDims {
return fmt.Errorf("field: '%s', invalid vector dimension: %d,"+
" value should be in range (%d, %d)", field.Name, field.Dims,
" value should be in range [%d, %d]", effectiveFieldName, field.Dims,
MinVectorDims, MaxVectorDims)
}
if _, ok := index.SupportedVectorSimilarityMetrics[field.Similarity]; !ok {
// Similarity metric must be supported
if _, ok := index.SupportedVectorSimilarityMetrics[effectiveSimilarity]; !ok {
return fmt.Errorf("field: '%s', invalid similarity "+
"metric: '%s', valid metrics are: %+v", field.Name, field.Similarity,
"metric: '%s', valid metrics are: %+v", effectiveFieldName, effectiveSimilarity,
reflect.ValueOf(index.SupportedVectorSimilarityMetrics).MapKeys())
}
// Vector index optimization must be supported
if _, ok := index.SupportedVectorIndexOptimizations[effectiveOptimizedFor]; !ok {
return fmt.Errorf("field: '%s', invalid vector index "+
"optimization: '%s', valid optimizations are: %+v", effectiveFieldName,
effectiveOptimizedFor,
reflect.ValueOf(index.SupportedVectorIndexOptimizations).MapKeys())
}
if fieldAliasCtx != nil { // writing to a nil map is unsafe
fieldAliasCtx[field.Name] = field
fieldAliasCtx[effectiveFieldName] = field
}
return nil
}
// NormalizeVector normalizes a single vector to unit length.
// It makes a copy of the input vector to avoid modifying it in-place.
func NormalizeVector(vec []float32) []float32 {
// make a copy of the vector to avoid modifying the original
// vector in-place
vecCopy := make([]float32, len(vec))
copy(vecCopy, vec)
vecCopy := slices.Clone(vec)
// normalize the vector copy using in-place normalization provided by faiss
return faiss.NormalizeVector(vecCopy)
}
// NormalizeMultiVector normalizes each sub-vector of size `dims` independently.
// For a flattened array containing multiple vectors, each sub-vector is
// normalized separately to unit length.
// It makes a copy of the input vector to avoid modifying it in-place.
func NormalizeMultiVector(vec []float32, dims int) []float32 {
if len(vec) == 0 || dims <= 0 || len(vec)%dims != 0 {
return vec
}
// Single vector - delegate to NormalizeVector
if len(vec) == dims {
return NormalizeVector(vec)
}
// Multi-vector - make a copy to avoid modifying the original
result := slices.Clone(vec)
// Normalize each sub-vector in-place
for i := 0; i < len(result); i += dims {
faiss.NormalizeVector(result[i : i+dims])
}
return result
}

View File

@@ -99,7 +99,7 @@ func (r *rescorer) rescore(ftsHits, knnHits search.DocumentMatchCollection) (sea
switch r.req.Score {
case ScoreRRF:
res := fusion.ReciprocalRankFusion(
fusionResult = fusion.ReciprocalRankFusion(
mergedHits,
r.origBoosts,
r.req.Params.ScoreRankConstant,
@@ -107,16 +107,14 @@ func (r *rescorer) rescore(ftsHits, knnHits search.DocumentMatchCollection) (sea
numKNNQueries(r.req),
r.req.Explain,
)
fusionResult = &res
case ScoreRSF:
res := fusion.RelativeScoreFusion(
fusionResult = fusion.RelativeScoreFusion(
mergedHits,
r.origBoosts,
r.req.Params.ScoreWindowSize,
numKNNQueries(r.req),
r.req.Explain,
)
fusionResult = &res
}
return fusionResult.Hits, fusionResult.Total, fusionResult.MaxScore

View File

@@ -17,8 +17,10 @@ package bleve
import (
"fmt"
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/blevesearch/bleve/v2/analysis"
@@ -147,8 +149,13 @@ type numericRange struct {
type FacetRequest struct {
Size int `json:"size"`
Field string `json:"field"`
TermPrefix string `json:"term_prefix,omitempty"`
TermPattern string `json:"term_pattern,omitempty"`
NumericRanges []*numericRange `json:"numeric_ranges,omitempty"`
DateTimeRanges []*dateTimeRange `json:"date_ranges,omitempty"`
// Compiled regex pattern (cached during validation)
compiledPattern *regexp.Regexp
}
// NewFacetRequest creates a facet on the specified
@@ -161,7 +168,26 @@ func NewFacetRequest(field string, size int) *FacetRequest {
}
}
// SetPrefixFilter sets the prefix filter for term facets.
func (fr *FacetRequest) SetPrefixFilter(prefix string) {
fr.TermPrefix = prefix
}
// SetRegexFilter sets the regex pattern filter for term facets.
func (fr *FacetRequest) SetRegexFilter(pattern string) {
fr.TermPattern = pattern
}
func (fr *FacetRequest) Validate() error {
// Validate regex pattern if provided and cache the compiled regex
if fr.TermPattern != "" {
compiled, err := regexp.Compile(fr.TermPattern)
if err != nil {
return fmt.Errorf("invalid term pattern: %v", err)
}
fr.compiledPattern = compiled
}
nrCount := len(fr.NumericRanges)
drCount := len(fr.DateTimeRanges)
if nrCount > 0 && drCount > 0 {
@@ -546,49 +572,74 @@ func (sr *SearchResult) Size() int {
}
func (sr *SearchResult) String() string {
rv := ""
rv := &strings.Builder{}
if sr.Total > 0 {
if sr.Request != nil && sr.Request.Size > 0 {
rv = fmt.Sprintf("%d matches, showing %d through %d, took %s\n", sr.Total, sr.Request.From+1, sr.Request.From+len(sr.Hits), sr.Took)
switch {
case sr.Request != nil && sr.Request.Size > 0:
start := sr.Request.From + 1
end := sr.Request.From + len(sr.Hits)
fmt.Fprintf(rv, "%d matches, showing %d through %d, took %s\n", sr.Total, start, end, sr.Took)
for i, hit := range sr.Hits {
rv += fmt.Sprintf("%5d. %s (%f)\n", i+sr.Request.From+1, hit.ID, hit.Score)
for fragmentField, fragments := range hit.Fragments {
rv += fmt.Sprintf("\t%s\n", fragmentField)
for _, fragment := range fragments {
rv += fmt.Sprintf("\t\t%s\n", fragment)
}
}
for otherFieldName, otherFieldValue := range hit.Fields {
if _, ok := hit.Fragments[otherFieldName]; !ok {
rv += fmt.Sprintf("\t%s\n", otherFieldName)
rv += fmt.Sprintf("\t\t%v\n", otherFieldValue)
}
}
rv = formatHit(rv, hit, start+i)
}
} else {
rv = fmt.Sprintf("%d matches, took %s\n", sr.Total, sr.Took)
case sr.Request == nil:
fmt.Fprintf(rv, "%d matches, took %s\n", sr.Total, sr.Took)
for i, hit := range sr.Hits {
rv = formatHit(rv, hit, i+1)
}
default:
fmt.Fprintf(rv, "%d matches, took %s\n", sr.Total, sr.Took)
}
} else {
rv = "No matches"
fmt.Fprintf(rv, "No matches\n")
}
if len(sr.Facets) > 0 {
rv += "Facets:\n"
fmt.Fprintf(rv, "Facets:\n")
for fn, f := range sr.Facets {
rv += fmt.Sprintf("%s(%d)\n", fn, f.Total)
fmt.Fprintf(rv, "%s(%d)\n", fn, f.Total)
for _, t := range f.Terms.Terms() {
rv += fmt.Sprintf("\t%s(%d)\n", t.Term, t.Count)
fmt.Fprintf(rv, "\t%s(%d)\n", t.Term, t.Count)
}
for _, n := range f.NumericRanges {
rv += fmt.Sprintf("\t%s(%d)\n", n.Name, n.Count)
fmt.Fprintf(rv, "\t%s(%d)\n", n.Name, n.Count)
}
for _, d := range f.DateRanges {
rv += fmt.Sprintf("\t%s(%d)\n", d.Name, d.Count)
fmt.Fprintf(rv, "\t%s(%d)\n", d.Name, d.Count)
}
if f.Other != 0 {
rv += fmt.Sprintf("\tOther(%d)\n", f.Other)
fmt.Fprintf(rv, "\tOther(%d)\n", f.Other)
}
}
}
return rv.String()
}
// formatHit is a helper function to format a single hit in the search result for
// the String() method of SearchResult
func formatHit(rv *strings.Builder, hit *search.DocumentMatch, hitNumber int) *strings.Builder {
fmt.Fprintf(rv, "%5d. %s (%f)\n", hitNumber, hit.ID, hit.Score)
for fragmentField, fragments := range hit.Fragments {
fmt.Fprintf(rv, "\t%s\n", fragmentField)
for _, fragment := range fragments {
fmt.Fprintf(rv, "\t\t%s\n", fragment)
}
}
for otherFieldName, otherFieldValue := range hit.Fields {
if _, ok := hit.Fragments[otherFieldName]; !ok {
fmt.Fprintf(rv, "\t%s\n", otherFieldName)
fmt.Fprintf(rv, "\t\t%v\n", otherFieldValue)
}
}
if len(hit.DecodedSort) > 0 {
fmt.Fprintf(rv, "\t_sort: [")
for k, v := range hit.DecodedSort {
if k > 0 {
fmt.Fprintf(rv, ", ")
}
fmt.Fprintf(rv, "%v", v)
}
fmt.Fprintf(rv, "]\n")
}
return rv
}

View File

@@ -15,7 +15,9 @@
package facet
import (
"bytes"
"reflect"
"regexp"
"sort"
"github.com/blevesearch/bleve/v2/search"
@@ -30,12 +32,14 @@ func init() {
}
type TermsFacetBuilder struct {
size int
field string
termsCount map[string]int
total int
missing int
sawValue bool
size int
field string
prefixBytes []byte
regex *regexp.Regexp
termsCount map[string]int
total int
missing int
sawValue bool
}
func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder {
@@ -48,7 +52,16 @@ func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder {
func (fb *TermsFacetBuilder) Size() int {
sizeInBytes := reflectStaticSizeTermsFacetBuilder + size.SizeOfPtr +
len(fb.field)
len(fb.field) +
len(fb.prefixBytes) +
size.SizeOfPtr // regex pointer (does not include actual regexp.Regexp object size)
// Estimate regex object size if present.
if fb.regex != nil {
// This is only the static size of regexp.Regexp struct, not including heap allocations.
sizeInBytes += int(reflect.TypeOf(*fb.regex).Size())
// NOTE: Actual memory usage of regexp.Regexp may be higher due to internal allocations.
}
for k := range fb.termsCount {
sizeInBytes += size.SizeOfString + len(k) +
@@ -62,10 +75,39 @@ func (fb *TermsFacetBuilder) Field() string {
return fb.field
}
// SetPrefixFilter sets the prefix filter for term facets.
func (fb *TermsFacetBuilder) SetPrefixFilter(prefix string) {
if prefix != "" {
fb.prefixBytes = []byte(prefix)
} else {
fb.prefixBytes = nil
}
}
// SetRegexFilter sets the compiled regex filter for term facets.
func (fb *TermsFacetBuilder) SetRegexFilter(regex *regexp.Regexp) {
fb.regex = regex
}
func (fb *TermsFacetBuilder) UpdateVisitor(term []byte) {
fb.sawValue = true
fb.termsCount[string(term)] = fb.termsCount[string(term)] + 1
// Total represents all terms visited, not just matching ones.
// This is necessary for the "Other" calculation.
fb.total++
// Fast prefix check on []byte - zero allocation
if len(fb.prefixBytes) > 0 && !bytes.HasPrefix(term, fb.prefixBytes) {
return
}
// Fast regex check on []byte - zero allocation
if fb.regex != nil && !fb.regex.Match(term) {
return
}
// Only convert to string if term matches filters
termStr := string(term)
fb.sawValue = true
fb.termsCount[termStr] = fb.termsCount[termStr] + 1
}
func (fb *TermsFacetBuilder) StartDoc() {

View File

@@ -15,7 +15,6 @@
package query
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -203,7 +202,7 @@ func (q *BooleanQuery) Searcher(ctx context.Context, i index.IndexReader, m mapp
return false
}
// Compare document IDs
cmp := bytes.Compare(refDoc.IndexInternalID, d.IndexInternalID)
cmp := refDoc.IndexInternalID.Compare(d.IndexInternalID)
if cmp < 0 {
// filterSearcher is behind the current document, Advance() it
refDoc, err = filterSearcher.Advance(sctx, d.IndexInternalID)
@@ -211,7 +210,7 @@ func (q *BooleanQuery) Searcher(ctx context.Context, i index.IndexReader, m mapp
return false
}
// After advance, check if they're now equal
return bytes.Equal(refDoc.IndexInternalID, d.IndexInternalID)
cmp = refDoc.IndexInternalID.Compare(d.IndexInternalID)
}
// cmp >= 0: either equal (match) or filterSearcher is ahead (no match)
return cmp == 0

View File

@@ -53,7 +53,7 @@ func (q *KNNQuery) SetK(k int64) {
q.K = k
}
func (q *KNNQuery) SetFieldVal(field string) {
func (q *KNNQuery) SetField(field string) {
q.VectorField = field
}

View File

@@ -88,7 +88,10 @@ func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents [
func (s *DisjunctionQueryScorer) ScoreAndExplBreakdown(ctx *search.SearchContext, constituents []*search.DocumentMatch,
matchingIdxs []int, originalPositions []int, countTotal int) *search.DocumentMatch {
scoreBreakdown := make(map[int]float64)
rv := constituents[0]
if rv.ScoreBreakdown == nil {
rv.ScoreBreakdown = make(map[int]float64, len(constituents))
}
var childrenExplanations []*search.Explanation
if s.options.Explain {
// since we want to notify which expl belongs to which matched searcher within the disjunction searcher
@@ -104,7 +107,7 @@ func (s *DisjunctionQueryScorer) ScoreAndExplBreakdown(ctx *search.SearchContext
// scorer used in disjunction heap searcher
index = matchingIdxs[i]
}
scoreBreakdown[index] = docMatch.Score
rv.ScoreBreakdown[index] = docMatch.Score
if s.options.Explain {
childrenExplanations[index] = docMatch.Expl
}
@@ -113,9 +116,6 @@ func (s *DisjunctionQueryScorer) ScoreAndExplBreakdown(ctx *search.SearchContext
if s.options.Explain {
explBreakdown = &search.Explanation{Children: childrenExplanations}
}
rv := constituents[0]
rv.ScoreBreakdown = scoreBreakdown
rv.Expl = explBreakdown
rv.FieldTermLocations = search.MergeFieldTermLocations(
rv.FieldTermLocations, constituents[1:])

View File

@@ -207,20 +207,29 @@ func (dm *DocumentMatch) Reset() *DocumentMatch {
indexInternalID := dm.IndexInternalID
// remember the []interface{} used for sort
sort := dm.Sort
// remember the []string used for decoded sort
decodedSort := dm.DecodedSort
// remember the FieldTermLocations backing array
ftls := dm.FieldTermLocations
for i := range ftls { // recycle the ArrayPositions of each location
ftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0]
}
// remember the score breakdown map
scoreBreakdown := dm.ScoreBreakdown
// clear out the score breakdown map
clear(scoreBreakdown)
// idiom to copy over from empty DocumentMatch (0 allocations)
*dm = DocumentMatch{}
// reuse the []byte already allocated (and reset len to 0)
dm.IndexInternalID = indexInternalID[:0]
// reuse the []interface{} already allocated (and reset len to 0)
dm.Sort = sort[:0]
dm.DecodedSort = dm.DecodedSort[:0]
// reuse the []string already allocated (and reset len to 0)
dm.DecodedSort = decodedSort[:0]
// reuse the FieldTermLocations already allocated (and reset len to 0)
dm.FieldTermLocations = ftls[:0]
// reuse the score breakdown map already allocated (after clearing it)
dm.ScoreBreakdown = scoreBreakdown
return dm
}

View File

@@ -84,7 +84,7 @@ func (s *KNNSearcher) VectorOptimize(ctx context.Context, octx index.VectorOptim
func (s *KNNSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (
*search.DocumentMatch, error) {
knnMatch, err := s.vectorReader.Next(s.vd.Reset())
knnMatch, err := s.vectorReader.Advance(ID, s.vd.Reset())
if err != nil {
return nil, err
}

View File

@@ -288,10 +288,15 @@ func createKNNQuery(req *SearchRequest, knnFilterResults map[int]index.EligibleD
// If it's a filtered kNN but has no eligible filter hits, then
// do not run the kNN query.
if selector, exists := knnFilterResults[i]; exists && selector == nil {
// if the kNN query is filtered and has no eligible filter hits, then
// do not run the kNN query, so we add a match_none query to the subQueries.
// this will ensure that the score breakdown is set to 0 for this kNN query.
subQueries = append(subQueries, NewMatchNoneQuery())
kArray = append(kArray, 0)
continue
}
knnQuery := query.NewKNNQuery(knn.Vector)
knnQuery.SetFieldVal(knn.Field)
knnQuery.SetField(knn.Field)
knnQuery.SetK(knn.K)
knnQuery.SetBoost(knn.Boost.Value())
knnQuery.SetParams(knn.Params)
@@ -381,7 +386,7 @@ func addSortAndFieldsToKNNHits(req *SearchRequest, knnHits []*search.DocumentMat
return nil
}
func (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader, preSearch bool) ([]*search.DocumentMatch, error) {
func (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader, preSearch bool) (knnHits []*search.DocumentMatch, err error) {
// Maps the index of a KNN query in the request to its pre-filter result:
// - If the KNN query is **not filtered**, the value will be `nil`.
// - If the KNN query **is filtered**, the value will be an eligible document selector
@@ -401,21 +406,33 @@ func (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, rea
continue
}
// Applies to all supported types of queries.
filterSearcher, _ := filterQ.Searcher(ctx, reader, i.m, search.SearcherOptions{
filterSearcher, err := filterQ.Searcher(ctx, reader, i.m, search.SearcherOptions{
Score: "none", // just want eligible hits --> don't compute scores if not needed
})
if err != nil {
return nil, err
}
// Using the index doc count to determine collector size since we do not
// have an estimate of the number of eligible docs in the index yet.
indexDocCount, err := i.DocCount()
if err != nil {
// close the searcher before returning
filterSearcher.Close()
return nil, err
}
filterColl := collector.NewEligibleCollector(int(indexDocCount))
err = filterColl.Collect(ctx, filterSearcher, reader)
if err != nil {
// close the searcher before returning
filterSearcher.Close()
return nil, err
}
knnFilterResults[idx] = filterColl.EligibleSelector()
// Close the filter searcher, as we are done with it.
err = filterSearcher.Close()
if err != nil {
return nil, err
}
}
// Add the filter hits when creating the kNN query
@@ -429,12 +446,17 @@ func (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, rea
if err != nil {
return nil, err
}
defer func() {
if serr := knnSearcher.Close(); err == nil && serr != nil {
err = serr
}
}()
knnCollector := collector.NewKNNCollector(kArray, sumOfK)
err = knnCollector.Collect(ctx, knnSearcher, reader)
if err != nil {
return nil, err
}
knnHits := knnCollector.Results()
knnHits = knnCollector.Results()
if !preSearch {
knnHits = finalizeKNNResults(req, knnHits)
}

View File

@@ -19,15 +19,11 @@ package zap
import (
"encoding/binary"
"encoding/json"
"math"
"reflect"
"github.com/RoaringBitmap/roaring/v2"
"github.com/RoaringBitmap/roaring/v2/roaring64"
"github.com/bits-and-blooms/bitset"
index "github.com/blevesearch/bleve_index_api"
faiss "github.com/blevesearch/go-faiss"
segment "github.com/blevesearch/scorch_segment_api/v2"
)
@@ -272,45 +268,7 @@ func (vpItr *VecPostingsIterator) BytesWritten() uint64 {
return 0
}
// vectorIndexWrapper conforms to scorch_segment_api's VectorIndex interface
type vectorIndexWrapper struct {
search func(qVector []float32, k int64,
params json.RawMessage) (segment.VecPostingsList, error)
searchWithFilter func(qVector []float32, k int64, eligibleDocIDs []uint64,
params json.RawMessage) (segment.VecPostingsList, error)
close func()
size func() uint64
obtainKCentroidCardinalitiesFromIVFIndex func(limit int, descending bool) (
[]index.CentroidCardinality, error)
}
func (i *vectorIndexWrapper) Search(qVector []float32, k int64,
params json.RawMessage) (
segment.VecPostingsList, error) {
return i.search(qVector, k, params)
}
func (i *vectorIndexWrapper) SearchWithFilter(qVector []float32, k int64,
eligibleDocIDs []uint64, params json.RawMessage) (
segment.VecPostingsList, error) {
return i.searchWithFilter(qVector, k, eligibleDocIDs, params)
}
func (i *vectorIndexWrapper) Close() {
i.close()
}
func (i *vectorIndexWrapper) Size() uint64 {
return i.size()
}
func (i *vectorIndexWrapper) ObtainKCentroidCardinalitiesFromIVFIndex(limit int, descending bool) (
[]index.CentroidCardinality, error) {
return i.obtainKCentroidCardinalitiesFromIVFIndex(limit, descending)
}
// InterpretVectorIndex returns a construct of closures (vectorIndexWrapper)
// InterpretVectorIndex returns a struct based implementation (vectorIndexWrapper)
// that will allow the caller to -
// (1) search within an attached vector index
// (2) search limited to a subset of documents within an attached vector index
@@ -319,248 +277,18 @@ func (i *vectorIndexWrapper) ObtainKCentroidCardinalitiesFromIVFIndex(limit int,
func (sb *SegmentBase) InterpretVectorIndex(field string, requiresFiltering bool,
except *roaring.Bitmap) (
segment.VectorIndex, error) {
// Params needed for the closures
var vecIndex *faiss.IndexImpl
var vecDocIDMap map[int64]uint32
var docVecIDMap map[uint32][]int64
var vectorIDsToExclude []int64
var fieldIDPlus1 uint16
var vecIndexSize uint64
// Utility function to add the corresponding docID and scores for each vector
// returned after the kNN query to the newly
// created vecPostingsList
addIDsToPostingsList := func(pl *VecPostingsList, ids []int64, scores []float32) {
for i := 0; i < len(ids); i++ {
vecID := ids[i]
// Checking if it's present in the vecDocIDMap.
// If -1 is returned as an ID(insufficient vectors), this will ensure
// it isn't added to the final postings list.
if docID, ok := vecDocIDMap[vecID]; ok {
code := getVectorCode(docID, scores[i])
pl.postings.Add(code)
}
}
}
var (
wrapVecIndex = &vectorIndexWrapper{
search: func(qVector []float32, k int64, params json.RawMessage) (
segment.VecPostingsList, error) {
// 1. returned postings list (of type PostingsList) has two types of information - docNum and its score.
// 2. both the values can be represented using roaring bitmaps.
// 3. the Iterator (of type PostingsIterator) returned would operate in terms of VecPostings.
// 4. VecPostings would just have the docNum and the score. Every call of Next()
// and Advance just returns the next VecPostings. The caller would do a vp.Number()
// and the Score() to get the corresponding values
rv := &VecPostingsList{
except: nil, // todo: handle the except bitmap within postings iterator.
postings: roaring64.New(),
}
if vecIndex == nil || vecIndex.D() != len(qVector) {
// vector index not found or dimensionality mismatched
return rv, nil
}
scores, ids, err := vecIndex.SearchWithoutIDs(qVector, k,
vectorIDsToExclude, params)
if err != nil {
return nil, err
}
addIDsToPostingsList(rv, ids, scores)
return rv, nil
},
searchWithFilter: func(qVector []float32, k int64,
eligibleDocIDs []uint64, params json.RawMessage) (
segment.VecPostingsList, error) {
// 1. returned postings list (of type PostingsList) has two types of information - docNum and its score.
// 2. both the values can be represented using roaring bitmaps.
// 3. the Iterator (of type PostingsIterator) returned would operate in terms of VecPostings.
// 4. VecPostings would just have the docNum and the score. Every call of Next()
// and Advance just returns the next VecPostings. The caller would do a vp.Number()
// and the Score() to get the corresponding values
rv := &VecPostingsList{
except: nil, // todo: handle the except bitmap within postings iterator.
postings: roaring64.New(),
}
if vecIndex == nil || vecIndex.D() != len(qVector) {
// vector index not found or dimensionality mismatched
return rv, nil
}
// Check and proceed only if non-zero documents eligible per the filter query.
if len(eligibleDocIDs) == 0 {
return rv, nil
}
// If every element in the index is eligible (full selectivity),
// then this can basically be considered unfiltered kNN.
if len(eligibleDocIDs) == int(sb.numDocs) {
scores, ids, err := vecIndex.SearchWithoutIDs(qVector, k,
vectorIDsToExclude, params)
if err != nil {
return nil, err
}
addIDsToPostingsList(rv, ids, scores)
return rv, nil
}
// vector IDs corresponding to the local doc numbers to be
// considered for the search
vectorIDsToInclude := make([]int64, 0, len(eligibleDocIDs))
for _, id := range eligibleDocIDs {
vecIDs := docVecIDMap[uint32(id)]
// In the common case where vecIDs has only one element, which occurs
// when a document has only one vector field, we can
// avoid the unnecessary overhead of slice unpacking (append(vecIDs...)).
// Directly append the single element for efficiency.
if len(vecIDs) == 1 {
vectorIDsToInclude = append(vectorIDsToInclude, vecIDs[0])
} else {
vectorIDsToInclude = append(vectorIDsToInclude, vecIDs...)
}
}
// In case a doc has invalid vector fields but valid non-vector fields,
// filter hit IDs may be ineligible for the kNN since the document does
// not have any/valid vectors.
if len(vectorIDsToInclude) == 0 {
return rv, nil
}
// If the index is not an IVF index, then the search can be
// performed directly, using the Flat index.
if !vecIndex.IsIVFIndex() {
// vector IDs corresponding to the local doc numbers to be
// considered for the search
scores, ids, err := vecIndex.SearchWithIDs(qVector, k,
vectorIDsToInclude, params)
if err != nil {
return nil, err
}
addIDsToPostingsList(rv, ids, scores)
return rv, nil
}
// Determining which clusters, identified by centroid ID,
// have at least one eligible vector and hence, ought to be
// probed.
clusterVectorCounts, err := vecIndex.ObtainClusterVectorCountsFromIVFIndex(vectorIDsToInclude)
if err != nil {
return nil, err
}
var selector faiss.Selector
// If there are more elements to be included than excluded, it
// might be quicker to use an exclusion selector as a filter
// instead of an inclusion selector.
if float32(len(eligibleDocIDs))/float32(len(docVecIDMap)) > 0.5 {
// Use a bitset to efficiently track eligible document IDs.
// This reduces the lookup cost when checking if a document ID is eligible,
// compared to using a map or slice.
bs := bitset.New(uint(len(eligibleDocIDs)))
for _, docID := range eligibleDocIDs {
bs.Set(uint(docID))
}
ineligibleVectorIDs := make([]int64, 0, len(vecDocIDMap)-len(vectorIDsToInclude))
for docID, vecIDs := range docVecIDMap {
// Check if the document ID is NOT in the eligible set, marking it as ineligible.
if !bs.Test(uint(docID)) {
// In the common case where vecIDs has only one element, which occurs
// when a document has only one vector field, we can
// avoid the unnecessary overhead of slice unpacking (append(vecIDs...)).
// Directly append the single element for efficiency.
if len(vecIDs) == 1 {
ineligibleVectorIDs = append(ineligibleVectorIDs, vecIDs[0])
} else {
ineligibleVectorIDs = append(ineligibleVectorIDs, vecIDs...)
}
}
}
selector, err = faiss.NewIDSelectorNot(ineligibleVectorIDs)
} else {
selector, err = faiss.NewIDSelectorBatch(vectorIDsToInclude)
}
if err != nil {
return nil, err
}
// If no error occurred during the creation of the selector, then
// it should be deleted once the search is complete.
defer selector.Delete()
// Ordering the retrieved centroid IDs by increasing order
// of distance i.e. decreasing order of proximity to query vector.
centroidIDs := make([]int64, 0, len(clusterVectorCounts))
for centroidID := range clusterVectorCounts {
centroidIDs = append(centroidIDs, centroidID)
}
closestCentroidIDs, centroidDistances, err :=
vecIndex.ObtainClustersWithDistancesFromIVFIndex(qVector, centroidIDs)
if err != nil {
return nil, err
}
// Getting the nprobe value set at index time.
nprobe := int(vecIndex.GetNProbe())
// Determining the minimum number of centroids to be probed
// to ensure that at least 'k' vectors are collected while
// examining at least 'nprobe' centroids.
var eligibleDocsTillNow int64
minEligibleCentroids := len(closestCentroidIDs)
for i, centroidID := range closestCentroidIDs {
eligibleDocsTillNow += clusterVectorCounts[centroidID]
// Stop once we've examined at least 'nprobe' centroids and
// collected at least 'k' vectors.
if eligibleDocsTillNow >= k && i+1 >= nprobe {
minEligibleCentroids = i + 1
break
}
}
// Search the clusters specified by 'closestCentroidIDs' for
// vectors whose IDs are present in 'vectorIDsToInclude'
scores, ids, err := vecIndex.SearchClustersFromIVFIndex(
selector, closestCentroidIDs, minEligibleCentroids,
k, qVector, centroidDistances, params)
if err != nil {
return nil, err
}
addIDsToPostingsList(rv, ids, scores)
return rv, nil
},
close: func() {
// skipping the closing because the index is cached and it's being
// deferred to a later point of time.
sb.vecIndexCache.decRef(fieldIDPlus1)
},
size: func() uint64 {
return vecIndexSize
},
obtainKCentroidCardinalitiesFromIVFIndex: func(limit int, descending bool) ([]index.CentroidCardinality, error) {
if vecIndex == nil || !vecIndex.IsIVFIndex() {
return nil, nil
}
cardinalities, centroids, err := vecIndex.ObtainKCentroidCardinalitiesFromIVFIndex(limit, descending)
if err != nil {
return nil, err
}
centroidCardinalities := make([]index.CentroidCardinality, len(cardinalities))
for i, cardinality := range cardinalities {
centroidCardinalities[i] = index.CentroidCardinality{
Centroid: centroids[i],
Cardinality: cardinality,
}
}
return centroidCardinalities, nil
},
}
err error
)
fieldIDPlus1 = sb.fieldsMap[field]
rv := &vectorIndexWrapper{sb: sb}
fieldIDPlus1 := sb.fieldsMap[field]
if fieldIDPlus1 <= 0 {
return wrapVecIndex, nil
return rv, nil
}
rv.fieldIDPlus1 = fieldIDPlus1
vectorSection := sb.fieldsSectionsMap[fieldIDPlus1-1][SectionFaissVectorIndex]
// check if the field has a vector section in the segment.
if vectorSection <= 0 {
return wrapVecIndex, nil
return rv, nil
}
pos := int(vectorSection)
@@ -574,15 +302,19 @@ func (sb *SegmentBase) InterpretVectorIndex(field string, requiresFiltering bool
pos += n
}
vecIndex, vecDocIDMap, docVecIDMap, vectorIDsToExclude, err =
var err error
rv.vecIndex, rv.vecDocIDMap, rv.docVecIDMap, rv.vectorIDsToExclude, err =
sb.vecIndexCache.loadOrCreate(fieldIDPlus1, sb.mem[pos:], requiresFiltering,
except)
if vecIndex != nil {
vecIndexSize = vecIndex.Size()
if err != nil {
return nil, err
}
return wrapVecIndex, err
if rv.vecIndex != nil {
rv.vecIndexSize = rv.vecIndex.Size()
}
return rv, nil
}
func (sb *SegmentBase) UpdateFieldStats(stats segment.FieldStats) {

View File

@@ -0,0 +1,645 @@
// Copyright (c) 2025 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build vectors
// +build vectors
package zap
import (
"encoding/json"
"math"
"slices"
"github.com/RoaringBitmap/roaring/v2/roaring64"
"github.com/bits-and-blooms/bitset"
index "github.com/blevesearch/bleve_index_api"
faiss "github.com/blevesearch/go-faiss"
segment "github.com/blevesearch/scorch_segment_api/v2"
)
// MaxMultiVectorDocSearchRetries limits repeated searches when deduplicating
// multi-vector documents. Each retry excludes previously seen vectors to find
// new unique documents. Acts as a safeguard against pathological data distributions.
var MaxMultiVectorDocSearchRetries = 100
// vectorIndexWrapper conforms to scorch_segment_api's VectorIndex interface
type vectorIndexWrapper struct {
vecIndex *faiss.IndexImpl
vecDocIDMap map[int64]uint32
docVecIDMap map[uint32][]int64
vectorIDsToExclude []int64
fieldIDPlus1 uint16
vecIndexSize uint64
sb *SegmentBase
}
func (v *vectorIndexWrapper) Search(qVector []float32, k int64,
params json.RawMessage) (
segment.VecPostingsList, error) {
// 1. returned postings list (of type PostingsList) has two types of information - docNum and its score.
// 2. both the values can be represented using roaring bitmaps.
// 3. the Iterator (of type PostingsIterator) returned would operate in terms of VecPostings.
// 4. VecPostings would just have the docNum and the score. Every call of Next()
// and Advance just returns the next VecPostings. The caller would do a vp.Number()
// and the Score() to get the corresponding values
rv := &VecPostingsList{
except: nil, // todo: handle the except bitmap within postings iterator.
postings: roaring64.New(),
}
if v.vecIndex == nil || v.vecIndex.D() != len(qVector) {
// vector index not found or dimensionality mismatched
return rv, nil
}
if v.sb.numDocs == 0 {
return rv, nil
}
rs, err := v.searchWithoutIDs(qVector, k,
v.vectorIDsToExclude, params)
if err != nil {
return nil, err
}
v.addIDsToPostingsList(rv, rs)
return rv, nil
}
func (v *vectorIndexWrapper) SearchWithFilter(qVector []float32, k int64,
eligibleDocIDs []uint64, params json.RawMessage) (
segment.VecPostingsList, error) {
// If every element in the index is eligible (full selectivity),
// then this can basically be considered unfiltered kNN.
if len(eligibleDocIDs) == int(v.sb.numDocs) {
return v.Search(qVector, k, params)
}
// 1. returned postings list (of type PostingsList) has two types of information - docNum and its score.
// 2. both the values can be represented using roaring bitmaps.
// 3. the Iterator (of type PostingsIterator) returned would operate in terms of VecPostings.
// 4. VecPostings would just have the docNum and the score. Every call of Next()
// and Advance just returns the next VecPostings. The caller would do a vp.Number()
// and the Score() to get the corresponding values
rv := &VecPostingsList{
except: nil, // todo: handle the except bitmap within postings iterator.
postings: roaring64.New(),
}
if v.vecIndex == nil || v.vecIndex.D() != len(qVector) {
// vector index not found or dimensionality mismatched
return rv, nil
}
// Check and proceed only if non-zero documents eligible per the filter query.
if len(eligibleDocIDs) == 0 {
return rv, nil
}
// vector IDs corresponding to the local doc numbers to be
// considered for the search
vectorIDsToInclude := make([]int64, 0, len(eligibleDocIDs))
for _, id := range eligibleDocIDs {
vecIDs := v.docVecIDMap[uint32(id)]
// In the common case where vecIDs has only one element, which occurs
// when a document has only one vector field, we can
// avoid the unnecessary overhead of slice unpacking (append(vecIDs...)).
// Directly append the single element for efficiency.
if len(vecIDs) == 1 {
vectorIDsToInclude = append(vectorIDsToInclude, vecIDs[0])
} else {
vectorIDsToInclude = append(vectorIDsToInclude, vecIDs...)
}
}
// In case a doc has invalid vector fields but valid non-vector fields,
// filter hit IDs may be ineligible for the kNN since the document does
// not have any/valid vectors.
if len(vectorIDsToInclude) == 0 {
return rv, nil
}
// If the index is not an IVF index, then the search can be
// performed directly, using the Flat index.
if !v.vecIndex.IsIVFIndex() {
// vector IDs corresponding to the local doc numbers to be
// considered for the search
rs, err := v.searchWithIDs(qVector, k,
vectorIDsToInclude, params)
if err != nil {
return nil, err
}
v.addIDsToPostingsList(rv, rs)
return rv, nil
}
// Determining which clusters, identified by centroid ID,
// have at least one eligible vector and hence, ought to be
// probed.
clusterVectorCounts, err := v.vecIndex.ObtainClusterVectorCountsFromIVFIndex(vectorIDsToInclude)
if err != nil {
return nil, err
}
var ids []int64
var include bool
// If there are more elements to be included than excluded, it
// might be quicker to use an exclusion selector as a filter
// instead of an inclusion selector.
if float32(len(eligibleDocIDs))/float32(len(v.docVecIDMap)) > 0.5 {
// Use a bitset to efficiently track eligible document IDs.
// This reduces the lookup cost when checking if a document ID is eligible,
// compared to using a map or slice.
bs := bitset.New(uint(v.sb.numDocs))
for _, docID := range eligibleDocIDs {
bs.Set(uint(docID))
}
ineligibleVectorIDs := make([]int64, 0, len(v.vecDocIDMap)-len(vectorIDsToInclude))
for docID, vecIDs := range v.docVecIDMap {
// Check if the document ID is NOT in the eligible set, marking it as ineligible.
if !bs.Test(uint(docID)) {
// In the common case where vecIDs has only one element, which occurs
// when a document has only one vector field, we can
// avoid the unnecessary overhead of slice unpacking (append(vecIDs...)).
// Directly append the single element for efficiency.
if len(vecIDs) == 1 {
ineligibleVectorIDs = append(ineligibleVectorIDs, vecIDs[0])
} else {
ineligibleVectorIDs = append(ineligibleVectorIDs, vecIDs...)
}
}
}
ids = ineligibleVectorIDs
include = false
} else {
ids = vectorIDsToInclude
include = true
}
// Ordering the retrieved centroid IDs by increasing order
// of distance i.e. decreasing order of proximity to query vector.
centroidIDs := make([]int64, 0, len(clusterVectorCounts))
for centroidID := range clusterVectorCounts {
centroidIDs = append(centroidIDs, centroidID)
}
closestCentroidIDs, centroidDistances, err :=
v.vecIndex.ObtainClustersWithDistancesFromIVFIndex(qVector, centroidIDs)
if err != nil {
return nil, err
}
// Getting the nprobe value set at index time.
nprobe := int(v.vecIndex.GetNProbe())
// Determining the minimum number of centroids to be probed
// to ensure that at least 'k' vectors are collected while
// examining at least 'nprobe' centroids.
// centroidsToProbe range: [nprobe, number of eligible centroids]
var eligibleVecsTillNow int64
centroidsToProbe := len(closestCentroidIDs)
for i, centroidID := range closestCentroidIDs {
eligibleVecsTillNow += clusterVectorCounts[centroidID]
// Stop once we've examined at least 'nprobe' centroids and
// collected at least 'k' vectors.
if eligibleVecsTillNow >= k && i+1 >= nprobe {
centroidsToProbe = i + 1
break
}
}
// Search the clusters specified by 'closestCentroidIDs' for
// vectors whose IDs are present in 'vectorIDsToInclude'
rs, err := v.searchClustersFromIVFIndex(
ids, include, closestCentroidIDs, centroidsToProbe,
k, qVector, centroidDistances, params)
if err != nil {
return nil, err
}
v.addIDsToPostingsList(rv, rs)
return rv, nil
}
func (v *vectorIndexWrapper) Close() {
// skipping the closing because the index is cached and it's being
// deferred to a later point of time.
v.sb.vecIndexCache.decRef(v.fieldIDPlus1)
}
func (v *vectorIndexWrapper) Size() uint64 {
return v.vecIndexSize
}
func (v *vectorIndexWrapper) ObtainKCentroidCardinalitiesFromIVFIndex(limit int, descending bool) (
[]index.CentroidCardinality, error) {
if v.vecIndex == nil || !v.vecIndex.IsIVFIndex() {
return nil, nil
}
cardinalities, centroids, err := v.vecIndex.ObtainKCentroidCardinalitiesFromIVFIndex(limit, descending)
if err != nil {
return nil, err
}
centroidCardinalities := make([]index.CentroidCardinality, len(cardinalities))
for i, cardinality := range cardinalities {
centroidCardinalities[i] = index.CentroidCardinality{
Centroid: centroids[i],
Cardinality: cardinality,
}
}
return centroidCardinalities, nil
}
// Utility function to add the corresponding docID and scores for each unique
// docID retrieved from the vector index search to the newly created vecPostingsList
func (v *vectorIndexWrapper) addIDsToPostingsList(pl *VecPostingsList, rs resultSet) {
rs.iterate(func(docID uint32, score float32) {
// transform the docID and score to vector code format
code := getVectorCode(docID, score)
// add to postings list, this ensures ordered storage
// based on the docID since it occupies the upper 32 bits
pl.postings.Add(code)
})
}
// docSearch performs a search on the vector index to retrieve
// top k documents based on the provided search function.
// It handles deduplication of documents that may have multiple
// vectors associated with them.
// The prepareNextIter function is used to set up the state
// for the next iteration, if more searches are needed to find
// k unique documents. The callback recieves the number of iterations
// done so far and the vector ids retrieved in the last search. While preparing
// the next iteration, if its decided that no further searches are needed,
// the prepareNextIter function can decide whether to continue searching or not
func (v *vectorIndexWrapper) docSearch(k int64, numDocs uint64,
search func() (scores []float32, labels []int64, err error),
prepareNextIter func(numIter int, labels []int64) bool) (resultSet, error) {
// create a result set to hold top K docIDs and their scores
rs := newResultSet(k, numDocs)
// flag to indicate if we have exhausted the vector index
var exhausted bool
// keep track of number of iterations done, we execute the loop more than once only when
// we have multi-vector documents leading to duplicates in docIDs retrieved
numIter := 0
// get the metric type of the index to help with deduplication logic
metricType := v.vecIndex.MetricType()
// we keep searching until we have k unique docIDs or we have exhausted the vector index
// or we have reached the maximum number of deduplication iterations allowed
for numIter < MaxMultiVectorDocSearchRetries && rs.size() < k && !exhausted {
// search the vector index
numIter++
scores, labels, err := search()
if err != nil {
return nil, err
}
// process the retrieved ids and scores, getting the corresponding docIDs
// for each vector id retrieved, and storing the best score for each unique docID
// the moment we see a -1 for a vector id, we stop processing further since
// it indicates there are no more vectors to be retrieved and break out of the loop
// by setting the exhausted flag
for i, vecID := range labels {
if vecID == -1 {
exhausted = true
break
}
docID, exists := v.getDocIDForVectorID(vecID)
if !exists {
continue
}
score := scores[i]
prevScore, exists := rs.get(docID)
if !exists {
// first time seeing this docID, so just store it
rs.put(docID, score)
continue
}
// we have seen this docID before, so we must compare scores
// check the index metric type first to check how we compare distances/scores
// and store the best score for the docID accordingly
// for inner product, higher the score, better the match
// for euclidean distance, lower the score/distance, better the match
// so we invert the comparison accordingly
switch metricType {
case faiss.MetricInnerProduct: // similarity metrics like dot product => higher is better
if score > prevScore {
rs.put(docID, score)
}
case faiss.MetricL2:
fallthrough
default: // distance metrics like euclidean distance => lower is better
if score < prevScore {
rs.put(docID, score)
}
}
}
// if we still have less than k unique docIDs, prepare for the next iteration, provided
// we have not exhausted the index
if rs.size() < k && !exhausted {
// prepare state for next iteration
shouldContinue := prepareNextIter(numIter, labels)
if !shouldContinue {
break
}
}
}
// at this point we either have k unique docIDs or we have exhausted
// the vector index or we have reached the maximum number of deduplication iterations allowed
// or the prepareNextIter function decided to break out of the loop
return rs, nil
}
// searchWithoutIDs performs a search on the vector index to retrieve the top K documents while
// excluding any vector IDs specified in the exclude slice.
func (v *vectorIndexWrapper) searchWithoutIDs(qVector []float32, k int64, exclude []int64, params json.RawMessage) (
resultSet, error) {
return v.docSearch(k, v.sb.numDocs,
func() ([]float32, []int64, error) {
return v.vecIndex.SearchWithoutIDs(qVector, k, exclude, params)
},
func(numIter int, labels []int64) bool {
// if this is the first loop iteration and we have < k unique docIDs,
// we must clone the existing exclude slice before appending to it
// to avoid modifying the original slice passed in by the caller
if numIter == 1 {
exclude = slices.Clone(exclude)
}
// prepare the exclude list for the next iteration by adding
// the vector ids retrieved in this iteration
exclude = append(exclude, labels...)
// with exclude list updated, we can proceed to the next iteration
return true
})
}
// searchWithIDs performs a search on the vector index to retrieve the top K documents while only
// considering the vector IDs specified in the include slice.
func (v *vectorIndexWrapper) searchWithIDs(qVector []float32, k int64, include []int64, params json.RawMessage) (
resultSet, error) {
// if the number of iterations > 1, we will be modifying the include slice
// to exclude vector ids already seen, so we use this set to track the
// include set for the next iteration, this is reused across iterations
// and allocated only once, when numIter == 1
var includeSet map[int64]struct{}
return v.docSearch(k, v.sb.numDocs,
func() ([]float32, []int64, error) {
return v.vecIndex.SearchWithIDs(qVector, k, include, params)
},
func(numIter int, labels []int64) bool {
// if this is the first loop iteration and we have < k unique docIDs,
// we clone the existing include slice before modifying it
if numIter == 1 {
include = slices.Clone(include)
// build the include set for subsequent iterations
includeSet = make(map[int64]struct{}, len(include))
for _, id := range include {
includeSet[id] = struct{}{}
}
}
// prepare the include list for the next iteration
// by removing the vector ids retrieved in this iteration
// from the include set
for _, id := range labels {
delete(includeSet, id)
}
// now build the next include slice from the set
include = include[:0]
for id := range includeSet {
include = append(include, id)
}
// only continue searching if we still have vector ids to include
return len(include) != 0
})
}
// searchClustersFromIVFIndex performs a search on the IVF vector index to retrieve the top K documents
// while either including or excluding the vector IDs specified in the ids slice, depending on the include flag.
// It takes into account the eligible centroid IDs and ensures that at least centroidsToProbe are probed.
// If after a few iterations we haven't found enough documents, it dynamically increases the number of
// clusters searched (up to the number of eligible centroids) to ensure we can find k unique documents.
func (v *vectorIndexWrapper) searchClustersFromIVFIndex(ids []int64, include bool, eligibleCentroidIDs []int64,
centroidsToProbe int, k int64, x, centroidDis []float32, params json.RawMessage) (
resultSet, error) {
// if the number of iterations > 1, we will be modifying the include slice
// to exclude vector ids already seen, so we use this set to track the
// include set for the next iteration, this is reused across iterations
// and allocated only once, when numIter == 1
var includeSet map[int64]struct{}
var totalEligibleCentroids = len(eligibleCentroidIDs)
// Threshold for when to start increasing: after 2 iterations without
// finding enough documents, we start increasing up to the number of centroidsToProbe
// up to the total number of eligible centroids available
const nprobeIncreaseThreshold = 2
return v.docSearch(k, v.sb.numDocs,
func() ([]float32, []int64, error) {
// build the selector based on whatever ids is as of now and the
// include/exclude flag
selector, err := v.getSelector(ids, include)
if err != nil {
return nil, nil, err
}
// once the main search is done we must free the selector
defer selector.Delete()
return v.vecIndex.SearchClustersFromIVFIndex(selector, eligibleCentroidIDs,
centroidsToProbe, k, x, centroidDis, params)
},
func(numIter int, labels []int64) bool {
// if this is the first loop iteration and we have < k unique docIDs,
// we must clone the existing ids slice before modifying it to avoid
// modifying the original slice passed in by the caller
if numIter == 1 {
ids = slices.Clone(ids)
if include {
// build the include set for subsequent iterations
// by adding all the ids initially present in the ids slice
includeSet = make(map[int64]struct{}, len(ids))
for _, id := range ids {
includeSet[id] = struct{}{}
}
}
}
// if we have iterated atleast nprobeIncreaseThreshold times
// and still have not found enough unique docIDs, we increase
// the number of centroids to probe for the next iteration
// to try and find more vectors/documents
if numIter >= nprobeIncreaseThreshold && centroidsToProbe < len(eligibleCentroidIDs) {
// Calculate how much to increase: increase by 50% of the remaining centroids to probe,
// but at least by 1 to ensure progress.
increaseAmount := max((totalEligibleCentroids-centroidsToProbe)/2, 1)
// Update centroidsToProbe, ensuring it does not exceed the total eligible centroids
centroidsToProbe = min(centroidsToProbe+increaseAmount, len(eligibleCentroidIDs))
}
// prepare the exclude/include list for the next iteration
if include {
// removing the vector ids retrieved in this iteration
// from the include set and rebuild the ids slice from the set
for _, id := range labels {
delete(includeSet, id)
}
// now build the next include slice from the set
ids = ids[:0]
for id := range includeSet {
ids = append(ids, id)
}
// only continue searching if we still have vector ids to include
return len(ids) != 0
} else {
// appending the vector ids retrieved in this iteration
// to the exclude list
ids = append(ids, labels...)
// with exclude list updated, we can proceed to the next iteration
return true
}
})
}
// Utility function to get a faiss.Selector based on the include/exclude flag
// and the vector ids provided, if include is true, it returns an inclusion selector,
// else it returns an exclusion selector. The caller must ensure to free the selector
// by calling selector.Delete() when done using it.
func (v *vectorIndexWrapper) getSelector(ids []int64, include bool) (selector faiss.Selector, err error) {
if include {
selector, err = faiss.NewIDSelectorBatch(ids)
} else {
selector, err = faiss.NewIDSelectorNot(ids)
}
if err != nil {
return nil, err
}
return selector, nil
}
// Utility function to get the docID for a given vectorID, used for the
// deduplication logic, to map vectorIDs back to their corresponding docIDs
func (v *vectorIndexWrapper) getDocIDForVectorID(vecID int64) (uint32, bool) {
docID, exists := v.vecDocIDMap[vecID]
return docID, exists
}
// resultSet is a data structure to hold (docID, score) pairs while ensuring
// that each docID is unique. It supports efficient insertion, retrieval,
// and iteration over the stored pairs.
type resultSet interface {
// Add a (docID, score) pair to the result set.
put(docID uint32, score float32)
// Get the score for a given docID. Returns false if docID not present.
get(docID uint32) (float32, bool)
// Iterate over all (docID, score) pairs in the result set.
iterate(func(docID uint32, score float32))
// Get the size of the result set.
size() int64
}
// resultSetSliceThreshold defines the threshold ratio of k to total documents
// in the index, below which a map-based resultSet is used, and above which
// a slice-based resultSet is used.
// It is derived using the following reasoning:
//
// Let N = total number of documents
// Let K = number of top K documents to retrieve
//
// Memory usage if the Result Set uses a map[uint32]float32 of size K underneath:
//
// ~20 bytes per entry (key + value + map overhead)
// Total ≈ 20 * K bytes
//
// Memory usage if the Result Set uses a slice of float32 of size N underneath:
//
// 4 bytes per entry
// Total ≈ 4 * N bytes
//
// We want the threshold below which a map is more memory-efficient than a slice:
//
// 20K < 4N
// K/N < 4/20
//
// Therefore, if the ratio of K to N is less than 0.2 (4/20), we use a map-based resultSet.
const resultSetSliceThreshold float64 = 0.2
// newResultSet creates a new resultSet
func newResultSet(k int64, numDocs uint64) resultSet {
// if numDocs is zero (empty index), just use map-based resultSet as its a no-op
// else decide based the percent of documents being retrieved. If we require
// greater than 20% of total documents, use slice-based resultSet for better memory efficiency
// else use map-based resultSet
if numDocs == 0 || float64(k)/float64(numDocs) < resultSetSliceThreshold {
return newResultSetMap(k)
}
return newResultSetSlice(numDocs)
}
type resultSetMap struct {
data map[uint32]float32
}
func newResultSetMap(k int64) resultSet {
return &resultSetMap{
data: make(map[uint32]float32, k),
}
}
func (rs *resultSetMap) put(docID uint32, score float32) {
rs.data[docID] = score
}
func (rs *resultSetMap) get(docID uint32) (float32, bool) {
score, exists := rs.data[docID]
return score, exists
}
func (rs *resultSetMap) iterate(f func(docID uint32, score float32)) {
for docID, score := range rs.data {
f(docID, score)
}
}
func (rs *resultSetMap) size() int64 {
return int64(len(rs.data))
}
type resultSetSlice struct {
count int64
data []float32
}
func newResultSetSlice(numDocs uint64) resultSet {
data := make([]float32, numDocs)
// scores can be negative, so initialize to a sentinel value which is NaN
sentinel := float32(math.NaN())
for i := range data {
data[i] = sentinel
}
return &resultSetSlice{
count: 0,
data: data,
}
}
func (rs *resultSetSlice) put(docID uint32, score float32) {
// only increment count if this docID was not already present
if math.IsNaN(float64(rs.data[docID])) {
rs.count++
}
rs.data[docID] = score
}
func (rs *resultSetSlice) get(docID uint32) (float32, bool) {
score := rs.data[docID]
if math.IsNaN(float64(score)) {
return 0, false
}
return score, true
}
func (rs *resultSetSlice) iterate(f func(docID uint32, score float32)) {
for docID, score := range rs.data {
if !math.IsNaN(float64(score)) {
f(uint32(docID), score)
}
}
}
func (rs *resultSetSlice) size() int64 {
return rs.count
}

View File

@@ -0,0 +1,60 @@
# Changelog
## [0.6.0]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.5.0...v0.6.0)
### Added
- New `StringGraphemes` and `BytesGraphemes` methods, for iterating over the
widths of grapheme clusters.
### Changed
- Added ASCII fast paths
## [0.5.0]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.4.1...v0.5.0)
### Added
- Unicode 16 support
- Improved emoji presentation handling per Unicode TR51
### Changed
- Corrected VS15 (U+FE0E) handling: now preserves base character width (no-op) per Unicode TR51
- Performance optimizations: reduced property lookups
### Fixed
- VS15 variation selector now correctly preserves base character width instead of forcing width 1
## [0.4.1]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.4.0...v0.4.1)
### Changed
- Updated uax29 dependency
- Improved flag handling
## [0.4.0]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.3.1...v0.4.0)
### Added
- Support for variation selectors (VS15, VS16) and regional indicator pairs (flags)
## [0.3.1]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.3.0...v0.3.1)
### Added
- Fuzz testing support
### Changed
- Updated stringish dependency
## [0.3.0]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.2.0...v0.3.0)
### Changed
- Dropped compatibility with go-runewidth
- Trie implementation cleanup

View File

@@ -5,6 +5,7 @@ A high-performance Go package for measuring the monospace display width of strin
[![Documentation](https://pkg.go.dev/badge/github.com/clipperhouse/displaywidth.svg)](https://pkg.go.dev/github.com/clipperhouse/displaywidth)
[![Test](https://github.com/clipperhouse/displaywidth/actions/workflows/gotest.yml/badge.svg)](https://github.com/clipperhouse/displaywidth/actions/workflows/gotest.yml)
[![Fuzz](https://github.com/clipperhouse/displaywidth/actions/workflows/gofuzz.yml/badge.svg)](https://github.com/clipperhouse/displaywidth/actions/workflows/gofuzz.yml)
## Install
```bash
go get github.com/clipperhouse/displaywidth
@@ -32,84 +33,91 @@ func main() {
}
```
For most purposes, you should use the `String` or `Bytes` methods.
### Options
You can specify East Asian Width and Strict Emoji Neutral settings. If
unspecified, the default is `EastAsianWidth: false, StrictEmojiNeutral: true`.
You can specify East Asian Width settings. When false (default),
[East Asian Ambiguous characters](https://www.unicode.org/reports/tr11/#Ambiguous)
are treated as width 1. When true, East Asian Ambiguous characters are treated
as width 2.
```go
options := displaywidth.Options{
EastAsianWidth: true,
StrictEmojiNeutral: false,
myOptions := displaywidth.Options{
EastAsianWidth: true,
}
width := options.String("Hello, 世界!")
width := myOptions.String("Hello, 世界!")
fmt.Println(width)
```
## Details
## Technical details
This package implements the Unicode East Asian Width standard (UAX #11) and is
intended to be compatible with `go-runewidth`. It operates on bytes without
decoding runes for better performance.
This package implements the Unicode East Asian Width standard
([UAX #11](https://www.unicode.org/reports/tr11/)), and handles
[version selectors](https://en.wikipedia.org/wiki/Variation_Selectors_(Unicode_block)),
and [regional indicator pairs](https://en.wikipedia.org/wiki/Regional_indicator_symbol)
(flags). We implement [Unicode TR51](https://unicode.org/reports/tr51/).
`clipperhouse/displaywidth`, `mattn/go-runewidth`, and `rivo/uniseg` will
give the same outputs for most real-world text. See extensive details in the
[compatibility analysis](comparison/COMPATIBILITY_ANALYSIS.md).
If you wish to investigate the core logic, see the `lookupProperties` and `width`
functions in [width.go](width.go#L135). The essential trie generation logic is in
`buildPropertyBitmap` in [unicode.go](internal/gen/unicode.go#L317).
I (@clipperhouse) am keeping an eye on [emerging standards and test suites](https://www.jeffquast.com/post/state-of-terminal-emulation-2025/).
## Prior Art
[mattn/go-runewidth](https://github.com/mattn/go-runewidth)
[rivo/uniseg](https://github.com/rivo/uniseg)
[x/text/width](https://pkg.go.dev/golang.org/x/text/width)
[x/text/internal/triegen](https://pkg.go.dev/golang.org/x/text/internal/triegen)
## Benchmarks
Part of my motivation is the insight that we can avoid decoding runes for better performance.
```bash
cd comparison
go test -bench=. -benchmem
```
```
goos: darwin
goarch: arm64
pkg: github.com/clipperhouse/displaywidth
pkg: github.com/clipperhouse/displaywidth/comparison
cpu: Apple M2
BenchmarkStringDefault/displaywidth-8 10537 ns/op 160.10 MB/s 0 B/op 0 allocs/op
BenchmarkStringDefault/go-runewidth-8 14162 ns/op 119.12 MB/s 0 B/op 0 allocs/op
BenchmarkString_EAW/displaywidth-8 10776 ns/op 156.55 MB/s 0 B/op 0 allocs/op
BenchmarkString_EAW/go-runewidth-8 23987 ns/op 70.33 MB/s 0 B/op 0 allocs/op
BenchmarkString_StrictEmoji/displaywidth-8 10892 ns/op 154.88 MB/s 0 B/op 0 allocs/op
BenchmarkString_StrictEmoji/go-runewidth-8 14552 ns/op 115.93 MB/s 0 B/op 0 allocs/op
BenchmarkString_ASCII/displaywidth-8 1116 ns/op 114.72 MB/s 0 B/op 0 allocs/op
BenchmarkString_ASCII/go-runewidth-8 1178 ns/op 108.67 MB/s 0 B/op 0 allocs/op
BenchmarkString_Unicode/displaywidth-8 896.9 ns/op 148.29 MB/s 0 B/op 0 allocs/op
BenchmarkString_Unicode/go-runewidth-8 1434 ns/op 92.72 MB/s 0 B/op 0 allocs/op
BenchmarkStringWidth_Emoji/displaywidth-8 3033 ns/op 238.74 MB/s 0 B/op 0 allocs/op
BenchmarkStringWidth_Emoji/go-runewidth-8 4841 ns/op 149.56 MB/s 0 B/op 0 allocs/op
BenchmarkString_Mixed/displaywidth-8 4064 ns/op 124.74 MB/s 0 B/op 0 allocs/op
BenchmarkString_Mixed/go-runewidth-8 4696 ns/op 107.97 MB/s 0 B/op 0 allocs/op
BenchmarkString_ControlChars/displaywidth-8 320.6 ns/op 102.93 MB/s 0 B/op 0 allocs/op
BenchmarkString_ControlChars/go-runewidth-8 373.8 ns/op 88.28 MB/s 0 B/op 0 allocs/op
BenchmarkRuneDefault/displaywidth-8 335.5 ns/op 411.35 MB/s 0 B/op 0 allocs/op
BenchmarkRuneDefault/go-runewidth-8 681.2 ns/op 202.58 MB/s 0 B/op 0 allocs/op
BenchmarkRuneWidth_EAW/displaywidth-8 146.7 ns/op 374.80 MB/s 0 B/op 0 allocs/op
BenchmarkRuneWidth_EAW/go-runewidth-8 495.6 ns/op 110.98 MB/s 0 B/op 0 allocs/op
BenchmarkRuneWidth_ASCII/displaywidth-8 63.00 ns/op 460.33 MB/s 0 B/op 0 allocs/op
BenchmarkRuneWidth_ASCII/go-runewidth-8 68.90 ns/op 420.91 MB/s 0 B/op 0 allocs/op
BenchmarkString_Mixed/clipperhouse/displaywidth-8 10469 ns/op 161.15 MB/s 0 B/op 0 allocs/op
BenchmarkString_Mixed/mattn/go-runewidth-8 14250 ns/op 118.39 MB/s 0 B/op 0 allocs/op
BenchmarkString_Mixed/rivo/uniseg-8 19258 ns/op 87.60 MB/s 0 B/op 0 allocs/op
BenchmarkString_EastAsian/clipperhouse/displaywidth-8 10518 ns/op 160.39 MB/s 0 B/op 0 allocs/op
BenchmarkString_EastAsian/mattn/go-runewidth-8 23827 ns/op 70.80 MB/s 0 B/op 0 allocs/op
BenchmarkString_EastAsian/rivo/uniseg-8 19537 ns/op 86.35 MB/s 0 B/op 0 allocs/op
BenchmarkString_ASCII/clipperhouse/displaywidth-8 1027 ns/op 124.61 MB/s 0 B/op 0 allocs/op
BenchmarkString_ASCII/mattn/go-runewidth-8 1166 ns/op 109.78 MB/s 0 B/op 0 allocs/op
BenchmarkString_ASCII/rivo/uniseg-8 1551 ns/op 82.52 MB/s 0 B/op 0 allocs/op
BenchmarkString_Emoji/clipperhouse/displaywidth-8 3164 ns/op 228.84 MB/s 0 B/op 0 allocs/op
BenchmarkString_Emoji/mattn/go-runewidth-8 4728 ns/op 153.13 MB/s 0 B/op 0 allocs/op
BenchmarkString_Emoji/rivo/uniseg-8 6489 ns/op 111.57 MB/s 0 B/op 0 allocs/op
BenchmarkRune_Mixed/clipperhouse/displaywidth-8 3429 ns/op 491.96 MB/s 0 B/op 0 allocs/op
BenchmarkRune_Mixed/mattn/go-runewidth-8 5308 ns/op 317.81 MB/s 0 B/op 0 allocs/op
BenchmarkRune_EastAsian/clipperhouse/displaywidth-8 3419 ns/op 493.49 MB/s 0 B/op 0 allocs/op
BenchmarkRune_EastAsian/mattn/go-runewidth-8 15321 ns/op 110.11 MB/s 0 B/op 0 allocs/op
BenchmarkRune_ASCII/clipperhouse/displaywidth-8 254.4 ns/op 503.19 MB/s 0 B/op 0 allocs/op
BenchmarkRune_ASCII/mattn/go-runewidth-8 264.3 ns/op 484.31 MB/s 0 B/op 0 allocs/op
BenchmarkRune_Emoji/clipperhouse/displaywidth-8 1374 ns/op 527.02 MB/s 0 B/op 0 allocs/op
BenchmarkRune_Emoji/mattn/go-runewidth-8 2210 ns/op 327.66 MB/s 0 B/op 0 allocs/op
```
I use a similar technique in [this grapheme cluster library](https://github.com/clipperhouse/uax29).
## Compatibility
`displaywidth` will mostly give the same outputs as `go-runewidth`, but there are some differences:
- Unicode category Mn (Nonspacing Mark): `displaywidth` will return width 0, `go-runewidth` may return width 1 for some runes.
- Unicode category Cf (Format): `displaywidth` will return width 0, `go-runewidth` may return width 1 for some runes.
- Unicode category Mc (Spacing Mark): `displaywidth` will return width 1, `go-runewidth` may return width 0 for some runes.
- Unicode category Cs (Surrogate): `displaywidth` will return width 0, `go-runewidth` may return width 1 for some runes. Surrogates are not valid UTF-8; some packages may turn them into the replacement character (U+FFFD).
- Unicode category Zl (Line separator): `displaywidth` will return width 0, `go-runewidth` may return width 1.
- Unicode category Zp (Paragraph separator): `displaywidth` will return width 0, `go-runewidth` may return width 1.
- Unicode Noncharacters (U+FFFE and U+FFFF): `displaywidth` will return width 0, `go-runewidth` may return width 1.
See `TestCompatibility` for more details.

View File

@@ -0,0 +1,72 @@
package displaywidth
import (
"github.com/clipperhouse/stringish"
"github.com/clipperhouse/uax29/v2/graphemes"
)
// Graphemes is an iterator over grapheme clusters.
//
// Iterate using the Next method, and get the width of the current grapheme
// using the Width method.
type Graphemes[T stringish.Interface] struct {
iter graphemes.Iterator[T]
options Options
}
// Next advances the iterator to the next grapheme cluster.
func (g *Graphemes[T]) Next() bool {
return g.iter.Next()
}
// Value returns the current grapheme cluster.
func (g *Graphemes[T]) Value() T {
return g.iter.Value()
}
// Width returns the display width of the current grapheme cluster.
func (g *Graphemes[T]) Width() int {
return graphemeWidth(g.Value(), g.options)
}
// StringGraphemes returns an iterator over grapheme clusters for the given
// string.
//
// Iterate using the Next method, and get the width of the current grapheme
// using the Width method.
func StringGraphemes(s string) Graphemes[string] {
return DefaultOptions.StringGraphemes(s)
}
// StringGraphemes returns an iterator over grapheme clusters for the given
// string, with the given options.
//
// Iterate using the Next method, and get the width of the current grapheme
// using the Width method.
func (options Options) StringGraphemes(s string) Graphemes[string] {
return Graphemes[string]{
iter: graphemes.FromString(s),
options: options,
}
}
// BytesGraphemes returns an iterator over grapheme clusters for the given
// []byte.
//
// Iterate using the Next method, and get the width of the current grapheme
// using the Width method.
func BytesGraphemes(s []byte) Graphemes[[]byte] {
return DefaultOptions.BytesGraphemes(s)
}
// BytesGraphemes returns an iterator over grapheme clusters for the given
// []byte, with the given options.
//
// Iterate using the Next method, and get the width of the current grapheme
// using the Width method.
func (options Options) BytesGraphemes(s []byte) Graphemes[[]byte] {
return Graphemes[[]byte]{
iter: graphemes.FromBytes(s),
options: options,
}
}

91
vendor/github.com/clipperhouse/displaywidth/tables.go generated vendored Normal file
View File

@@ -0,0 +1,91 @@
package displaywidth
// propertyWidths is a jump table of sorts, instead of a switch
var propertyWidths = [5]int{
_Default: 1,
_Zero_Width: 0,
_East_Asian_Wide: 2,
_East_Asian_Ambiguous: 1,
_Emoji: 2,
}
// asciiWidths is a lookup table for single-byte character widths. Printable
// ASCII characters have width 1, control characters have width 0.
//
// It is intended for valid single-byte UTF-8, which means <128.
//
// If you look up an index >= 128, that is either:
// - invalid UTF-8, or
// - a multi-byte UTF-8 sequence, in which case you should be operating on
// the grapheme cluster, and not using this table
//
// We will return a default value of 1 in those cases, so as not to panic.
var asciiWidths = [256]int8{
// Control characters (0x00-0x1F): width 0
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// Printable ASCII (0x20-0x7E): width 1
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
// DEL (0x7F): width 0
0,
// >= 128
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
}
// asciiProperties is a lookup table for single-byte character properties.
// It is intended for valid single-byte UTF-8, which means <128.
//
// If you look up an index >= 128, that is either:
// - invalid UTF-8, or
// - a multi-byte UTF-8 sequence, in which case you should be operating on
// the grapheme cluster, and not using this table
//
// We will return a default value of _Default in those cases, so as not to
// panic.
var asciiProperties = [256]property{
// Control characters (0x00-0x1F): _Zero_Width
_Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width,
_Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width,
_Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width,
_Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width, _Zero_Width,
// Printable ASCII (0x20-0x7E): _Default
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default,
// DEL (0x7F): _Zero_Width
_Zero_Width,
// >= 128
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
_Default, _Default, _Default, _Default, _Default, _Default, _Default, _Default,
}

View File

File diff suppressed because it is too large Load Diff

View File

@@ -7,153 +7,205 @@ import (
"github.com/clipperhouse/uax29/v2/graphemes"
)
// String calculates the display width of a string
// using the [DefaultOptions]
// Options allows you to specify the treatment of ambiguous East Asian
// characters. When EastAsianWidth is false (default), ambiguous East Asian
// characters are treated as width 1. When EastAsianWidth is true, ambiguous
// East Asian characters are treated as width 2.
type Options struct {
EastAsianWidth bool
}
// DefaultOptions is the default options for the display width
// calculation, which is EastAsianWidth: false.
var DefaultOptions = Options{EastAsianWidth: false}
// String calculates the display width of a string,
// by iterating over grapheme clusters in the string
// and summing their widths.
func String(s string) int {
return DefaultOptions.String(s)
}
// Bytes calculates the display width of a []byte
// using the [DefaultOptions]
// String calculates the display width of a string, for the given options, by
// iterating over grapheme clusters in the string and summing their widths.
func (options Options) String(s string) int {
// Optimization: no need to parse grapheme
switch len(s) {
case 0:
return 0
case 1:
return int(asciiWidths[s[0]])
}
width := 0
g := graphemes.FromString(s)
for g.Next() {
width += graphemeWidth(g.Value(), options)
}
return width
}
// Bytes calculates the display width of a []byte,
// by iterating over grapheme clusters in the byte slice
// and summing their widths.
func Bytes(s []byte) int {
return DefaultOptions.Bytes(s)
}
// Bytes calculates the display width of a []byte, for the given options, by
// iterating over grapheme clusters in the slice and summing their widths.
func (options Options) Bytes(s []byte) int {
// Optimization: no need to parse grapheme
switch len(s) {
case 0:
return 0
case 1:
return int(asciiWidths[s[0]])
}
width := 0
g := graphemes.FromBytes(s)
for g.Next() {
width += graphemeWidth(g.Value(), options)
}
return width
}
// Rune calculates the display width of a rune. You
// should almost certainly use [String] or [Bytes] for
// most purposes.
//
// The smallest unit of display width is a grapheme
// cluster, not a rune. Iterating over runes to measure
// width is incorrect in many cases.
func Rune(r rune) int {
return DefaultOptions.Rune(r)
}
type Options struct {
EastAsianWidth bool
StrictEmojiNeutral bool
}
var DefaultOptions = Options{
EastAsianWidth: false,
StrictEmojiNeutral: true,
}
// String calculates the display width of a string
// for the given options
func (options Options) String(s string) int {
if len(s) == 0 {
return 0
}
total := 0
g := graphemes.FromString(s)
for g.Next() {
// The first character in the grapheme cluster determines the width;
// modifiers and joiners do not contribute to the width.
props, _ := lookupProperties(g.Value())
total += props.width(options)
}
return total
}
// BytesOptions calculates the display width of a []byte
// for the given options
func (options Options) Bytes(s []byte) int {
if len(s) == 0 {
return 0
}
total := 0
g := graphemes.FromBytes(s)
for g.Next() {
// The first character in the grapheme cluster determines the width;
// modifiers and joiners do not contribute to the width.
props, _ := lookupProperties(g.Value())
total += props.width(options)
}
return total
}
// Rune calculates the display width of a rune, for the given options.
//
// You should almost certainly use [String] or [Bytes] for most purposes.
//
// The smallest unit of display width is a grapheme cluster, not a rune.
// Iterating over runes to measure width is incorrect in many cases.
func (options Options) Rune(r rune) int {
// Fast path for ASCII
if r < utf8.RuneSelf {
if isASCIIControl(byte(r)) {
// Control (0x00-0x1F) and DEL (0x7F)
return 0
}
// ASCII printable (0x20-0x7E)
return 1
return int(asciiWidths[byte(r)])
}
// Surrogates (U+D800-U+DFFF) are invalid UTF-8 and have zero width
// Other packages might turn them into the replacement character (U+FFFD)
// in which case, we won't see it.
// Surrogates (U+D800-U+DFFF) are invalid UTF-8.
if r >= 0xD800 && r <= 0xDFFF {
return 0
}
// Stack-allocated to avoid heap allocation
var buf [4]byte // UTF-8 is at most 4 bytes
var buf [4]byte
n := utf8.EncodeRune(buf[:], r)
// Skip the grapheme iterator and directly lookup properties
props, _ := lookupProperties(buf[:n])
return props.width(options)
// Skip the grapheme iterator
return lookupProperties(buf[:n]).width(options)
}
func isASCIIControl(b byte) bool {
return b < 0x20 || b == 0x7F
}
const defaultWidth = 1
// is returns true if the property flag is set
func (p property) is(flag property) bool {
return p&flag != 0
}
// lookupProperties returns the properties for the first character in a string
func lookupProperties[T stringish.Interface](s T) (property, int) {
if len(s) == 0 {
return 0, 0
// graphemeWidth returns the display width of a grapheme cluster.
// The passed string must be a single grapheme cluster.
func graphemeWidth[T stringish.Interface](s T, options Options) int {
// Optimization: no need to look up properties
switch len(s) {
case 0:
return 0
case 1:
return int(asciiWidths[s[0]])
}
// Fast path for ASCII characters (single byte)
b := s[0]
if b < utf8.RuneSelf { // Single-byte ASCII
if isASCIIControl(b) {
// Control characters (0x00-0x1F) and DEL (0x7F) - width 0
return _ZeroWidth, 1
return lookupProperties(s).width(options)
}
// isRIPrefix checks if the slice matches the Regional Indicator prefix
// (F0 9F 87). It assumes len(s) >= 3.
func isRIPrefix[T stringish.Interface](s T) bool {
return s[0] == 0xF0 && s[1] == 0x9F && s[2] == 0x87
}
// isVS16 checks if the slice matches VS16 (U+FE0F) UTF-8 encoding
// (EF B8 8F). It assumes len(s) >= 3.
func isVS16[T stringish.Interface](s T) bool {
return s[0] == 0xEF && s[1] == 0xB8 && s[2] == 0x8F
}
// lookupProperties returns the properties for a grapheme.
// The passed string must be at least one byte long.
//
// Callers must handle zero and single-byte strings upstream, both as an
// optimization, and to reduce the scope of this function.
func lookupProperties[T stringish.Interface](s T) property {
l := len(s)
if s[0] < utf8.RuneSelf {
// Check for variation selector after ASCII (e.g., keycap sequences like 1⃣)
if l >= 4 {
// Subslice may help eliminate bounds checks
vs := s[1:4]
if isVS16(vs) {
// VS16 requests emoji presentation (width 2)
return _Emoji
}
// VS15 (0x8E) requests text presentation but does not affect width,
// in my reading of Unicode TR51. Falls through to _Default.
}
// ASCII printable characters (0x20-0x7E) - width 1
// Return 0 properties, width calculation will default to 1
return 0, 1
return asciiProperties[s[0]]
}
// Use the generated trie for lookup
props, size := lookup(s)
return property(props), size
// Regional indicator pair (flag)
if l >= 8 {
// Subslice may help eliminate bounds checks
ri := s[:8]
// First rune
if isRIPrefix(ri[0:3]) {
b3 := ri[3]
if b3 >= 0xA6 && b3 <= 0xBF {
// Second rune
if isRIPrefix(ri[4:7]) {
b7 := ri[7]
if b7 >= 0xA6 && b7 <= 0xBF {
return _Emoji
}
}
}
}
}
p, sz := lookup(s)
// Variation Selectors
if sz > 0 && l >= sz+3 {
// Subslice may help eliminate bounds checks
vs := s[sz : sz+3]
if isVS16(vs) {
// VS16 requests emoji presentation (width 2)
return _Emoji
}
// VS15 (0x8E) requests text presentation but does not affect width,
// in my reading of Unicode TR51. Falls through to return the base
// character's property.
}
return property(p)
}
// width determines the display width of a character based on its properties
const _Default property = 0
const boundsCheck = property(len(propertyWidths) - 1)
// width determines the display width of a character based on its properties,
// and configuration options
func (p property) width(options Options) int {
if p == 0 {
// Character not in trie, use default behavior
return defaultWidth
}
if p.is(_ZeroWidth) {
return 0
}
if options.EastAsianWidth {
if p.is(_East_Asian_Ambiguous) {
return 2
}
if p.is(_East_Asian_Ambiguous|_Emoji) && !options.StrictEmojiNeutral {
return 2
}
}
if p.is(_East_Asian_Full_Wide) {
if options.EastAsianWidth && p == _East_Asian_Ambiguous {
return 2
}
// Default width for all other characters
return defaultWidth
// Bounds check may help the compiler eliminate its bounds check,
// and safety of course.
if p > boundsCheck {
return 1 // default width
}
return propertyWidths[p]
}

View File

@@ -1,5 +1,9 @@
An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode version 15.0.0.
[![Documentation](https://pkg.go.dev/badge/github.com/clipperhouse/uax29/v2/graphemes.svg)](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
![Tests](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
![Fuzz](https://github.com/clipperhouse/uax29/actions/workflows/gofuzz.yml/badge.svg)
## Quick start
```
@@ -18,15 +22,14 @@ for tokens.Next() { // Next() returns true until end of data
}
```
[![Documentation](https://pkg.go.dev/badge/github.com/clipperhouse/uax29/v2/graphemes.svg)](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
_A grapheme is a “single visible character”, which might be a simple as a single letter, or a complex emoji that consists of several Unicode code points._
## Conformance
We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Tests29). Status:
We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Tests29).
![Go](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
![Tests](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
![Fuzz](https://github.com/clipperhouse/uax29/actions/workflows/gofuzz.yml/badge.svg)
## APIs
@@ -71,9 +74,18 @@ for tokens.Next() { // Next() returns true until end of data
}
```
### Performance
### Benchmarks
On a Mac M2 laptop, we see around 200MB/s, or around 100 million graphemes per second. You should see ~constant memory, and no allocations.
On a Mac M2 laptop, we see around 200MB/s, or around 100 million graphemes per second, and no allocations.
```
goos: darwin
goarch: arm64
pkg: github.com/clipperhouse/uax29/graphemes/comparative
cpu: Apple M2
BenchmarkGraphemes/clipperhouse/uax29-8 173805 ns/op 201.16 MB/s 0 B/op 0 allocs/op
BenchmarkGraphemes/rivo/uniseg-8 2045128 ns/op 17.10 MB/s 0 B/op 0 allocs/op
```
### Invalid inputs

View File

@@ -1,8 +1,11 @@
package graphemes
import "github.com/clipperhouse/uax29/v2/internal/iterators"
import (
"github.com/clipperhouse/stringish"
"github.com/clipperhouse/uax29/v2/internal/iterators"
)
type Iterator[T iterators.Stringish] struct {
type Iterator[T stringish.Interface] struct {
*iterators.Iterator[T]
}

View File

@@ -3,7 +3,7 @@ package graphemes
import (
"bufio"
"github.com/clipperhouse/uax29/v2/internal/iterators"
"github.com/clipperhouse/stringish"
)
// is determines if lookup intersects propert(ies)
@@ -18,7 +18,7 @@ const _Ignore = _Extend
// See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
var SplitFunc bufio.SplitFunc = splitFunc[[]byte]
func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T, err error) {
func splitFunc[T stringish.Interface](data T, atEOF bool) (advance int, token T, err error) {
var empty T
if len(data) == 0 {
return 0, empty, nil

View File

@@ -1,10 +1,10 @@
package graphemes
import "github.com/clipperhouse/stringish"
// generated by github.com/clipperhouse/uax29/v2
// from https://www.unicode.org/Public/15.0.0/ucd/auxiliary/GraphemeBreakProperty.txt
import "github.com/clipperhouse/uax29/v2/internal/iterators"
type property uint16
const (
@@ -27,7 +27,7 @@ const (
// lookup returns the trie value for the first UTF-8 encoding in s and
// the width in bytes of this encoding. The size will be 0 if s does not
// hold enough bytes to complete the encoding. len(s) must be greater than 0.
func lookup[T iterators.Stringish](s T) (v property, sz int) {
func lookup[T stringish.Interface](s T) (v property, sz int) {
c0 := s[0]
switch {
case c0 < 0x80: // is ASCII

View File

@@ -1,14 +1,12 @@
package iterators
type Stringish interface {
[]byte | string
}
import "github.com/clipperhouse/stringish"
type SplitFunc[T Stringish] func(T, bool) (int, T, error)
type SplitFunc[T stringish.Interface] func(T, bool) (int, T, error)
// Iterator is a generic iterator for words that are either []byte or string.
// Iterate while Next() is true, and access the word via Value().
type Iterator[T Stringish] struct {
type Iterator[T stringish.Interface] struct {
split SplitFunc[T]
data T
start int
@@ -16,7 +14,7 @@ type Iterator[T Stringish] struct {
}
// New creates a new Iterator for the given data and SplitFunc.
func New[T Stringish](split SplitFunc[T], data T) *Iterator[T] {
func New[T stringish.Interface](split SplitFunc[T], data T) *Iterator[T] {
return &Iterator[T]{
split: split,
data: data,
@@ -83,3 +81,20 @@ func (iter *Iterator[T]) Reset() {
iter.start = 0
iter.pos = 0
}
func (iter *Iterator[T]) First() T {
if len(iter.data) == 0 {
return iter.data
}
advance, _, err := iter.split(iter.data, true)
if err != nil {
panic(err)
}
if advance <= 0 {
panic("SplitFunc returned a zero or negative advance")
}
if advance > len(iter.data) {
panic("SplitFunc advanced beyond the end of the data")
}
return iter.data[:advance]
}

View File

@@ -1,6 +1,6 @@
// Package chi is a small, idiomatic and composable router for building HTTP services.
//
// chi requires Go 1.14 or newer.
// chi supports the four most recent major versions of Go.
//
// Example:
//

View File

@@ -2,6 +2,7 @@ package middleware
import (
"net/http"
"slices"
"strings"
)
@@ -29,13 +30,7 @@ func contentEncoding(ce string, charsets ...string) bool {
_, ce = split(strings.ToLower(ce), ";")
_, ce = split(ce, "charset=")
ce, _ = split(ce, ";")
for _, c := range charsets {
if ce == c {
return true
}
}
return false
return slices.Contains(charsets, ce)
}
// Split a string in two parts, cleaning any whitespace.

View File

@@ -25,7 +25,7 @@ const RequestIDKey ctxKeyRequestID = 0
var RequestIDHeader = "X-Request-Id"
var prefix string
var reqid uint64
var reqid atomic.Uint64
// A quick note on the statistics here: we're trying to calculate the chance that
// two randomly generated base62 prefixes will collide. We use the formula from
@@ -69,7 +69,7 @@ func RequestID(next http.Handler) http.Handler {
ctx := r.Context()
requestID := r.Header.Get(RequestIDHeader)
if requestID == "" {
myid := atomic.AddUint64(&reqid, 1)
myid := reqid.Add(1)
requestID = fmt.Sprintf("%s-%06d", prefix, myid)
}
ctx = context.WithValue(ctx, RequestIDKey, requestID)
@@ -92,5 +92,5 @@ func GetReqID(ctx context.Context) string {
// NextRequestID generates the next request ID in the sequence.
func NextRequestID() uint64 {
return atomic.AddUint64(&reqid, 1)
return reqid.Add(1)
}

View File

@@ -47,15 +47,22 @@ func RedirectSlashes(next http.Handler) http.Handler {
} else {
path = r.URL.Path
}
if len(path) > 1 && path[len(path)-1] == '/' {
// Trim all leading and trailing slashes (e.g., "//evil.com", "/some/path//")
path = "/" + strings.Trim(path, "/")
// Normalize backslashes to forward slashes to prevent "/\evil.com" style redirects
// that some clients may interpret as protocol-relative.
path = strings.ReplaceAll(path, `\`, `/`)
// Collapse leading/trailing slashes and force a single leading slash.
path := "/" + strings.Trim(path, "/")
if r.URL.RawQuery != "" {
path = fmt.Sprintf("%s?%s", path, r.URL.RawQuery)
}
http.Redirect(w, r, path, 301)
return
}
next.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)

View File

@@ -467,8 +467,10 @@ func (mx *Mux) routeHTTP(w http.ResponseWriter, r *http.Request) {
// Find the route
if _, _, h := mx.tree.FindRoute(rctx, method, routePath); h != nil {
if supportsPathValue {
setPathValue(rctx, r)
// Set http.Request path values from our request context
for i, key := range rctx.URLParams.Keys {
value := rctx.URLParams.Values[i]
r.SetPathValue(key, value)
}
if supportsPattern {
setPattern(rctx, r)

View File

@@ -1,21 +0,0 @@
//go:build go1.22 && !tinygo
// +build go1.22,!tinygo
package chi
import "net/http"
// supportsPathValue is true if the Go version is 1.22 and above.
//
// If this is true, `net/http.Request` has methods `SetPathValue` and `PathValue`.
const supportsPathValue = true
// setPathValue sets the path values in the Request value
// based on the provided request context.
func setPathValue(rctx *Context, r *http.Request) {
for i, key := range rctx.URLParams.Keys {
value := rctx.URLParams.Values[i]
r.SetPathValue(key, value)
}
}

View File

@@ -1,19 +0,0 @@
//go:build !go1.22 || tinygo
// +build !go1.22 tinygo
package chi
import "net/http"
// supportsPathValue is true if the Go version is 1.22 and above.
//
// If this is true, `net/http.Request` has methods `SetPathValue` and `PathValue`.
const supportsPathValue = false
// setPathValue sets the path values in the Request value
// based on the provided request context.
//
// setPathValue is only supported in Go 1.22 and above so
// this is just a blank function so that it compiles.
func setPathValue(rctx *Context, r *http.Request) {
}

View File

@@ -71,6 +71,7 @@ func RegisterMethod(method string) {
}
mt := methodTyp(2 << n)
methodMap[method] = mt
reverseMethodMap[mt] = method
mALL |= mt
}
@@ -328,7 +329,7 @@ func (n *node) replaceChild(label, tail byte, child *node) {
func (n *node) getEdge(ntyp nodeTyp, label, tail byte, prefix string) *node {
nds := n.children[ntyp]
for i := 0; i < len(nds); i++ {
for i := range nds {
if nds[i].label == label && nds[i].tail == tail {
if ntyp == ntRegexp && nds[i].prefix != prefix {
continue
@@ -429,9 +430,7 @@ func (n *node) findRoute(rctx *Context, method methodTyp, path string) *node {
}
// serially loop through each node grouped by the tail delimiter
for idx := 0; idx < len(nds); idx++ {
xn = nds[idx]
for _, xn = range nds {
// label for param nodes is the delimiter byte
p := strings.IndexByte(xsearch, xn.tail)
@@ -770,20 +769,14 @@ func patParamKeys(pattern string) []string {
}
}
// longestPrefix finds the length of the shared prefix
// of two strings
func longestPrefix(k1, k2 string) int {
max := len(k1)
if l := len(k2); l < max {
max = l
}
var i int
for i = 0; i < max; i++ {
// longestPrefix finds the length of the shared prefix of two strings
func longestPrefix(k1, k2 string) (i int) {
for i = 0; i < min(len(k1), len(k2)); i++ {
if k1[i] != k2[i] {
break
}
}
return i
return
}
type nodes []*node

View File

@@ -23,7 +23,7 @@ A [Go](http://golang.org) client for the [NATS messaging system](https://nats.io
go get github.com/nats-io/nats.go@latest
# To get a specific version:
go get github.com/nats-io/nats.go@v1.47.0
go get github.com/nats-io/nats.go@v1.48.0
# Note that the latest major version for NATS Server is v2:
go get github.com/nats-io/nats-server/v2@latest

View File

@@ -95,7 +95,7 @@ func (nc *Conn) oldRequestWithContext(ctx context.Context, subj string, hdr, dat
s.AutoUnsubscribe(1)
defer s.Unsubscribe()
err = nc.publish(subj, inbox, hdr, data)
err = nc.publish(subj, inbox, false, hdr, data)
if err != nil {
return nil, err
}

View File

@@ -107,7 +107,7 @@ func (c *EncodedConn) Publish(subject string, v any) error {
if err != nil {
return err
}
return c.Conn.publish(subject, _EMPTY_, nil, b)
return c.Conn.publish(subject, _EMPTY_, false, nil, b)
}
// PublishRequest will perform a Publish() expecting a response on the
@@ -120,7 +120,7 @@ func (c *EncodedConn) PublishRequest(subject, reply string, v any) error {
if err != nil {
return err
}
return c.Conn.publish(subject, reply, nil, b)
return c.Conn.publish(subject, reply, true, nil, b)
}
// Request will create an Inbox and perform a Request() call

View File

@@ -568,7 +568,6 @@ iter, _ := cons.Messages(jetstream.PullMaxMessages(10), jetstream.PullMaxBytes(1
request. If the value is set too low, the consumer will stall and not be able
to consume messages.
- `PullExpiry(time.Duration)` - timeout on a single pull request to the server
type PullThresholdMessages int
- `PullThresholdMessages(int)` - amount of messages which triggers refilling the
buffer
- `PullThresholdBytes(int)` - amount of bytes which triggers refilling the

View File

@@ -67,6 +67,11 @@ type (
// without additional checks. After the channel is closed,
// MessageBatch.Error() should be checked to see if there was an error
// during message delivery (e.g. missing heartbeat).
//
// NOTE: Fetch has worse performance when used to continuously retrieve
// messages in comparison to Messages or Consume methods, as it does not
// perform any optimizations (e.g. overlapping pull requests) and new
// subscription is created for each execution.
Fetch(batch int, opts ...FetchOpt) (MessageBatch, error)
// FetchBytes is used to retrieve up to a provided bytes from the
@@ -88,6 +93,11 @@ type (
// without additional checks. After the channel is closed,
// MessageBatch.Error() should be checked to see if there was an error
// during message delivery (e.g. missing heartbeat).
//
// NOTE: FetchBytes has worse performance when used to continuously
// retrieve messages in comparison to Messages or Consume methods, as it
// does not perform any optimizations (e.g. overlapping pull requests)
// and new subscription is created for each execution.
FetchBytes(maxBytes int, opts ...FetchOpt) (MessageBatch, error)
// FetchNoWait is used to retrieve up to a provided number of messages
@@ -102,6 +112,11 @@ type (
// without additional checks. After the channel is closed,
// MessageBatch.Error() should be checked to see if there was an error
// during message delivery (e.g. missing heartbeat).
//
// NOTE: FetchNoWait has worse performance when used to continuously
// retrieve messages in comparison to Messages or Consume methods, as it
// does not perform any optimizations (e.g. overlapping pull requests)
// and new subscription is created for each execution.
FetchNoWait(batch int) (MessageBatch, error)
// Consume will continuously receive messages and handle them

View File

@@ -246,6 +246,11 @@ type (
Mirror *StreamSource `json:"mirror,omitempty"`
// Sources defines the configuration for sources of a KeyValue store.
// If no subject transforms are defined, it is assumed that a source is
// also a KV store and subject transforms will be set to correctly map
// keys from the source KV to the current one. If subject transforms are
// defined, they will be used as is. This allows using non-kv streams as
// sources.
Sources []*StreamSource `json:"sources,omitempty"`
// Compression sets the underlying stream compression.
@@ -471,7 +476,6 @@ const (
kvSubjectsTmpl = "$KV.%s.>"
kvSubjectsPreTmpl = "$KV.%s."
kvSubjectsPreDomainTmpl = "%s.$KV.%s."
kvNoPending = "0"
)
const (
@@ -685,8 +689,14 @@ func (js *jetStream) prepareKeyValueConfig(ctx context.Context, cfg KeyValueConf
scfg.Mirror = m
scfg.MirrorDirect = true
} else if len(cfg.Sources) > 0 {
// For now we do not allow direct subjects for sources. If that is desired a user could use stream API directly.
for _, ss := range cfg.Sources {
// if subject transforms are already set, then use as is.
// this allows for full control of the source, e.g. using non-KV streams.
// Note that in this case, the Name is not modified and full stream name must be provided.
if len(ss.SubjectTransforms) > 0 {
scfg.Sources = append(scfg.Sources, ss)
continue
}
var sourceBucketName string
if strings.HasPrefix(ss.Name, kvBucketNamePre) {
sourceBucketName = ss.Name[len(kvBucketNamePre):]
@@ -1291,6 +1301,8 @@ func (kv *kvs) WatchFiltered(ctx context.Context, keys []string, opts ...WatchOp
return nil, err
}
sub.SetClosedHandler(func(_ string) {
w.mu.Lock()
defer w.mu.Unlock()
close(w.updates)
})
// If there were no pending messages at the time of the creation

View File

@@ -105,7 +105,11 @@ func (p *pushConsumer) Consume(handler MessageHandler, opts ...PushConsumeOpt) (
}
var err error
sub.subscription, err = p.js.conn.Subscribe(p.info.Config.DeliverSubject, internalHandler)
if p.info.Config.DeliverGroup != "" {
sub.subscription, err = p.js.conn.QueueSubscribe(p.info.Config.DeliverSubject, p.info.Config.DeliverGroup, internalHandler)
} else {
sub.subscription, err = p.js.conn.Subscribe(p.info.Config.DeliverSubject, internalHandler)
}
if err != nil {
return nil, err
}

View File

@@ -1132,7 +1132,7 @@ func (js *js) PublishMsgAsync(m *Msg, opts ...PubOpt) (PubAckFuture, error) {
if err != nil {
return nil, err
}
if err := js.nc.publish(m.Subject, reply, hdr, m.Data); err != nil {
if err := js.nc.publish(m.Subject, reply, false, hdr, m.Data); err != nil {
js.clearPAF(id)
return nil, err
}
@@ -3560,7 +3560,7 @@ func (js *js) apiRequestWithContext(ctx context.Context, subj string, data []byt
}
if js.opts.shouldTrace {
ctrace := js.opts.ctrace
if ctrace.RequestSent != nil {
if ctrace.ResponseReceived != nil {
ctrace.ResponseReceived(subj, resp.Data, resp.Header)
}
}

View File

@@ -354,7 +354,6 @@ const (
kvSubjectsTmpl = "$KV.%s.>"
kvSubjectsPreTmpl = "$KV.%s."
kvSubjectsPreDomainTmpl = "%s.$KV.%s."
kvNoPending = "0"
)
// Regex for valid keys and buckets.

View File

@@ -48,7 +48,7 @@ import (
// Default Constants
const (
Version = "1.47.0"
Version = "1.48.0"
DefaultURL = "nats://127.0.0.1:4222"
DefaultPort = 4222
DefaultMaxReconnect = 60
@@ -534,6 +534,11 @@ type Options struct {
// WebSocketConnectionHeadersHandler is an optional callback handler for generating token used for WebSocket connections.
WebSocketConnectionHeadersHandler WebSocketHeadersHandler
// SkipSubjectValidation will disable publish subject validation.
// NOTE: This is not recommended in general, as the performance gain is minimal
// and may lead to breaking protocol.
SkipSubjectValidation bool
}
const (
@@ -1512,6 +1517,20 @@ func WebSocketConnectionHeadersHandler(cb WebSocketHeadersHandler) Option {
}
}
// SkipSubjectValidation is an Option to skip subject validation when
// publishing messages.
// By default, subject validation is performed to ensure that subjects
// are valid according to NATS subject syntax (no spaces newlines and tabs).
// NOTE: It is not recommended to use this option as the performance gain
// is minimal and disabling subject validation can lead breaking protocol
// rules.
func SkipSubjectValidation() Option {
return func(o *Options) error {
o.SkipSubjectValidation = true
return nil
}
}
// Handler processing
// SetDisconnectHandler will set the disconnect event handler.
@@ -3916,7 +3935,7 @@ func (nc *Conn) kickFlusher() {
// argument is left untouched and needs to be correctly interpreted on
// the receiver.
func (nc *Conn) Publish(subj string, data []byte) error {
return nc.publish(subj, _EMPTY_, nil, data)
return nc.publish(subj, _EMPTY_, false, nil, data)
}
// Header represents the optional Header for a NATS message,
@@ -4059,27 +4078,71 @@ func (nc *Conn) PublishMsg(m *Msg) error {
if err != nil {
return err
}
return nc.publish(m.Subject, m.Reply, hdr, m.Data)
validateReply := m.Reply != _EMPTY_
return nc.publish(m.Subject, m.Reply, validateReply, hdr, m.Data)
}
// PublishRequest will perform a Publish() expecting a response on the
// reply subject. Use Request() for automatically waiting for a response
// inline.
func (nc *Conn) PublishRequest(subj, reply string, data []byte) error {
return nc.publish(subj, reply, nil, data)
return nc.publish(subj, reply, true, nil, data)
}
// Used for handrolled Itoa
const digits = "0123456789"
// validateSubject checks if the subject contains characters that break the NATS protocol.
// Uses an adaptive algorithm: manual loop for short subjects (< 16 chars) and
// SIMD-optimized strings.IndexByte for longer subjects.
func validateSubject(subj string) error {
if subj == "" {
return ErrBadSubject
}
// Adaptive threshold based on benchmark data showing crossover at ~15-20 characters.
const lengthThreshold = 16
if len(subj) < lengthThreshold {
// Fast path for short subjects (< 16 chars)
// Short-circuit on non-control characters.
for i := range len(subj) {
c := subj[i]
if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
return ErrBadSubject
}
}
return nil
}
// Optimized path for long subjects (>= 16 chars)
// Uses SIMD-optimized strings.IndexByte (processes 16+ bytes per instruction)
if strings.IndexByte(subj, ' ') >= 0 ||
strings.IndexByte(subj, '\t') >= 0 ||
strings.IndexByte(subj, '\r') >= 0 ||
strings.IndexByte(subj, '\n') >= 0 {
return ErrBadSubject
}
return nil
}
// publish is the internal function to publish messages to a nats-server.
// Sends a protocol data message by queuing into the bufio writer
// and kicking the flush go routine. These writes should be protected.
func (nc *Conn) publish(subj, reply string, hdr, data []byte) error {
func (nc *Conn) publish(subj, reply string, validateReply bool, hdr, data []byte) error {
if nc == nil {
return ErrInvalidConnection
}
if subj == "" {
if !nc.Opts.SkipSubjectValidation {
if err := validateSubject(subj); err != nil {
return err
}
if validateReply {
if err := validateSubject(reply); err != nil {
return ErrBadSubject
}
}
} else if subj == _EMPTY_ {
return ErrBadSubject
}
nc.mu.Lock()
@@ -4245,7 +4308,7 @@ func (nc *Conn) createNewRequestAndSend(subj string, hdr, data []byte) (chan *Ms
}
nc.mu.Unlock()
if err := nc.publish(subj, respInbox, hdr, data); err != nil {
if err := nc.publish(subj, respInbox, false, hdr, data); err != nil {
return nil, token, err
}
@@ -4341,7 +4404,7 @@ func (nc *Conn) oldRequest(subj string, hdr, data []byte, timeout time.Duration)
s.AutoUnsubscribe(1)
defer s.Unsubscribe()
err = nc.publish(subj, inbox, hdr, data)
err = nc.publish(subj, inbox, false, hdr, data)
if err != nil {
return nil, err
}

View File

@@ -657,6 +657,13 @@ func Mark(names ...string) {
// It is similar to Dbg but formats the output as JSON for better readability. It is thread-safe and respects
// the loggers configuration (e.g., enabled, level, suspend, handler, middleware).
func Output(values ...interface{}) {
defaultLogger.output(2, values...)
}
// Inspect logs one or more values in a **developer-friendly, deeply introspective format** at Info level.
// It includes the caller file and line number, and reveals **all fields** — including:
func Inspect(values ...interface{}) {
o := NewInspector(defaultLogger)
o.Log(2, values...)
}

View File

@@ -79,7 +79,7 @@ func (o *Inspector) Log(skip int, values ...interface{}) {
}
// Construct log message with file, line, and JSON data
msg := fmt.Sprintf("[%s:%d] DUMP: %s", shortFile, line, string(jsonData))
msg := fmt.Sprintf("[%s:%d] INSPECT: %s", shortFile, line, string(jsonData))
o.logger.log(lx.LevelInfo, lx.ClassText, msg, nil, false)
}
}

View File

@@ -350,17 +350,58 @@ func (l *Logger) Dump(values ...interface{}) {
}
}
// Output logs data in a human-readable JSON format at Info level, including caller file and line information.
// It is similar to Dbg but formats the output as JSON for better readability. It is thread-safe and respects
// the logger's configuration (e.g., enabled, level, suspend, handler, middleware).
// Example:
//
// logger := New("app").Enable()
// x := map[string]int{"key": 42}
// logger.Output(x) // Output: [app] INFO: [file.go:123] JSON: {"key": 42}
//
// Logger method to provide access to Output functionality
// Output logs each value as pretty-printed JSON for REST debugging.
// Each value is logged on its own line with [file:line] and a blank line after the header.
// Ideal for inspecting outgoing/incoming REST payloads.
func (l *Logger) Output(values ...interface{}) {
l.output(2, values...)
}
func (l *Logger) output(skip int, values ...interface{}) {
if !l.shouldLog(lx.LevelInfo) {
return
}
_, file, line, ok := runtime.Caller(skip)
if !ok {
return
}
shortFile := file
if idx := strings.LastIndex(file, "/"); idx >= 0 {
shortFile = file[idx+1:]
}
header := fmt.Sprintf("[%s:%d] JSON:\n", shortFile, line)
for _, v := range values {
// Always pretty-print with indent
b, err := json.MarshalIndent(v, " ", " ")
if err != nil {
b, _ = json.MarshalIndent(map[string]any{
"value": fmt.Sprintf("%+v", v),
"error": err.Error(),
}, " ", " ")
}
l.log(lx.LevelInfo, lx.ClassText, header+string(b), nil, false)
}
}
// Inspect logs one or more values in a **developer-friendly, deeply introspective format** at Info level.
// It includes the caller file and line number, and reveals **all fields** — including:
//
// - Private (unexported) fields → prefixed with `(field)`
// - Embedded structs (inlined)
// - Pointers and nil values → shown as `*(field)` or `nil`
// - Full struct nesting and type information
//
// This method uses `NewInspector` under the hood, which performs **full reflection-based traversal**.
// It is **not** meant for production logging or REST APIs — use `Output` for that.
//
// Ideal for:
// - Debugging complex internal state
// - Inspecting structs with private fields
// - Understanding struct embedding and pointer behavior
func (l *Logger) Inspect(values ...interface{}) {
o := NewInspector(l)
o.Log(2, values...)
}

View File

@@ -28,7 +28,7 @@ go get github.com/olekukonko/tablewriter@v0.0.5
#### Latest Version
The latest stable version
```bash
go get github.com/olekukonko/tablewriter@v1.1.1
go get github.com/olekukonko/tablewriter@v1.1.2
```
**Warning:** Version `v1.0.0` contains missing functionality and should not be used.
@@ -62,7 +62,7 @@ func main() {
data := [][]string{
{"Package", "Version", "Status"},
{"tablewriter", "v0.0.5", "legacy"},
{"tablewriter", "v1.1.1", "latest"},
{"tablewriter", "v1.1.2", "latest"},
}
table := tablewriter.NewWriter(os.Stdout)
@@ -77,7 +77,7 @@ func main() {
│ PACKAGE │ VERSION │ STATUS │
├─────────────┼─────────┼────────┤
│ tablewriter │ v0.0.5 │ legacy │
│ tablewriter │ v1.1.1 │ latest │
│ tablewriter │ v1.1.2 │ latest │
└─────────────┴─────────┴────────┘
```

View File

@@ -1,6 +1,8 @@
package tablewriter
import (
"github.com/mattn/go-runewidth"
"github.com/olekukonko/tablewriter/pkg/twwidth"
"github.com/olekukonko/tablewriter/tw"
)
@@ -218,3 +220,16 @@ func WithTableMax(width int) Option {
}
}
}
// Deprecated: use WithEastAsian instead.
// WithCondition provides a way to set a custom global runewidth.Condition
// that will be used for all subsequent display width calculations by the twwidth (twdw) package.
//
// The runewidth.Condition object allows for more fine-grained control over how rune widths
// are determined, beyond just toggling EastAsianWidth. This could include settings for
// ambiguous width characters or other future properties of runewidth.Condition.
func WithCondition(cond *runewidth.Condition) Option {
return func(target *Table) {
twwidth.SetCondition(cond)
}
}

View File

@@ -3,8 +3,8 @@ package tablewriter
import (
"reflect"
"github.com/mattn/go-runewidth"
"github.com/olekukonko/ll"
"github.com/olekukonko/tablewriter/pkg/twcache"
"github.com/olekukonko/tablewriter/pkg/twwidth"
"github.com/olekukonko/tablewriter/tw"
)
@@ -471,22 +471,48 @@ func WithStreaming(c tw.StreamConfig) Option {
func WithStringer(stringer interface{}) Option {
return func(t *Table) {
t.stringer = stringer
t.stringerCacheMu.Lock()
t.stringerCache = make(map[reflect.Type]reflect.Value)
t.stringerCacheMu.Unlock()
t.stringerCache = twcache.NewLRU[reflect.Type, reflect.Value](tw.DefaultCacheStringCapacity)
if t.logger != nil {
t.logger.Debug("Stringer updated, cache cleared")
}
}
}
// WithStringerCache enables caching for the stringer function.
// Logs the change if debugging is enabled.
// WithStringerCache enables the default LRU caching for the stringer function.
// It initializes the cache with a default capacity if one does not already exist.
func WithStringerCache() Option {
return func(t *Table) {
t.stringerCacheEnabled = true
// Initialize default cache if strictly necessary (nil),
// or if you want to ensure the default implementation is used.
if t.stringerCache == nil {
// NewLRU returns (Instance, error). We ignore the error here assuming capacity > 0.
cache := twcache.NewLRU[reflect.Type, reflect.Value](tw.DefaultCacheStringCapacity)
t.stringerCache = cache
}
if t.logger != nil {
t.logger.Debug("Option: WithStringerCache enabled")
t.logger.Debug("Option: WithStringerCache enabled (Default LRU)")
}
}
}
// WithStringerCacheCustom enables caching for the stringer function using a specific implementation.
// Passing nil disables caching entirely.
func WithStringerCacheCustom(cache twcache.Cache[reflect.Type, reflect.Value]) Option {
return func(t *Table) {
if cache == nil {
t.stringerCache = nil
if t.logger != nil {
t.logger.Debug("Option: WithStringerCacheCustom called with nil (Caching Disabled)")
}
return
}
// Set the custom cache and enable the flag
t.stringerCache = cache
if t.logger != nil {
t.logger.Debug("Option: WithStringerCacheCustom enabled")
}
}
}
@@ -629,27 +655,20 @@ func WithRendition(rendition tw.Rendition) Option {
}
// WithEastAsian configures the global East Asian width calculation setting.
// - enable=true: Enables East Asian width calculations. CJK and ambiguous characters
// - state=tw.On: Enables East Asian width calculations. CJK and ambiguous characters
// are typically measured as double width.
// - enable=false: Disables East Asian width calculations. Characters are generally
// - state=tw.Off: Disables East Asian width calculations. Characters are generally
// measured as single width, subject to Unicode standards.
//
// This setting affects all subsequent display width calculations using the twdw package.
func WithEastAsian(enable bool) Option {
func WithEastAsian(state tw.State) Option {
return func(target *Table) {
twwidth.SetEastAsian(enable)
}
}
// WithCondition provides a way to set a custom global runewidth.Condition
// that will be used for all subsequent display width calculations by the twwidth (twdw) package.
//
// The runewidth.Condition object allows for more fine-grained control over how rune widths
// are determined, beyond just toggling EastAsianWidth. This could include settings for
// ambiguous width characters or other future properties of runewidth.Condition.
func WithCondition(cond *runewidth.Condition) Option {
return func(target *Table) {
twwidth.SetCondition(cond)
if state.Enabled() {
twwidth.SetEastAsian(true)
}
if state.Disabled() {
twwidth.SetEastAsian(false)
}
}
}

View File

@@ -0,0 +1,12 @@
package twcache
// Cache defines a generic interface for a key-value storage with type constraints on keys and values.
// The keys must be of a type that supports comparison.
// Add inserts a new key-value pair, potentially evicting an item if necessary.
// Get retrieves a value associated with the given key, returning a boolean to indicate if the key was found.
// Purge clears all items from the cache.
type Cache[K comparable, V any] interface {
Add(key K, value V) (evicted bool)
Get(key K) (value V, ok bool)
Purge()
}

View File

@@ -0,0 +1,289 @@
package twcache
import (
"sync"
"sync/atomic"
)
// EvictCallback is a function called when an entry is evicted.
// This includes evictions during Purge or Resize operations.
type EvictCallback[K comparable, V any] func(key K, value V)
// LRU is a thread-safe, generic LRU cache with a fixed size.
// It has zero dependencies, high performance, and full features.
type LRU[K comparable, V any] struct {
size int
items map[K]*entry[K, V]
head *entry[K, V] // Most Recently Used
tail *entry[K, V] // Least Recently Used
onEvict EvictCallback[K, V]
mu sync.Mutex
hits atomic.Int64
misses atomic.Int64
}
// entry represents a single item in the LRU linked list.
// It holds the key, value, and pointers to prev/next entries.
type entry[K comparable, V any] struct {
key K
value V
prev *entry[K, V]
next *entry[K, V]
}
// NewLRU creates a new LRU cache with the given size.
// Returns nil if size <= 0, acting as a disabled cache.
// Caps size at 100,000 for reasonableness.
func NewLRU[K comparable, V any](size int) *LRU[K, V] {
return NewLRUEvict[K, V](size, nil)
}
// NewLRUEvict creates a new LRU cache with an eviction callback.
// The callback is optional and called on evictions.
// Returns nil if size <= 0.
func NewLRUEvict[K comparable, V any](size int, onEvict EvictCallback[K, V]) *LRU[K, V] {
if size <= 0 {
return nil // nil = disabled cache (fast path in hot code)
}
if size > 100_000 {
size = 100_000 // reasonable upper bound
}
return &LRU[K, V]{
size: size,
items: make(map[K]*entry[K, V], size),
onEvict: onEvict,
}
}
// GetOrCompute retrieves a value or computes it if missing.
// Ensures no double computation under concurrency.
// Ideal for expensive computations like twwidth.
func (c *LRU[K, V]) GetOrCompute(key K, compute func() V) V {
if c == nil || c.size <= 0 {
return compute()
}
c.mu.Lock()
if e, ok := c.items[key]; ok {
c.moveToFront(e)
c.hits.Add(1)
c.mu.Unlock()
return e.value
}
c.misses.Add(1)
value := compute() // expensive work only on real miss
// Double-check: someone might have added it while computing
if e, ok := c.items[key]; ok {
e.value = value
c.moveToFront(e)
c.mu.Unlock()
return value
}
// Evict if needed
if len(c.items) >= c.size {
c.removeOldest()
}
e := &entry[K, V]{key: key, value: value}
c.addToFront(e)
c.items[key] = e
c.mu.Unlock()
return value
}
// Get retrieves a value by key if it exists.
// Returns the value and true if found, else zero and false.
// Updates the entry to most recently used.
func (c *LRU[K, V]) Get(key K) (V, bool) {
if c == nil || c.size <= 0 {
var zero V
return zero, false
}
c.mu.Lock()
defer c.mu.Unlock()
e, ok := c.items[key]
if !ok {
c.misses.Add(1)
var zero V
return zero, false
}
c.hits.Add(1)
c.moveToFront(e)
return e.value, true
}
// Add inserts or updates a key-value pair.
// Evicts the oldest if cache is full.
// Returns true if an eviction occurred.
func (c *LRU[K, V]) Add(key K, value V) (evicted bool) {
if c == nil || c.size <= 0 {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
if e, ok := c.items[key]; ok {
e.value = value
c.moveToFront(e)
return false
}
if len(c.items) >= c.size {
c.removeOldest()
evicted = true
}
e := &entry[K, V]{key: key, value: value}
c.addToFront(e)
c.items[key] = e
return evicted
}
// Remove deletes a key from the cache.
// Returns true if the key was found and removed.
func (c *LRU[K, V]) Remove(key K) bool {
if c == nil || c.size <= 0 {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
e, ok := c.items[key]
if !ok {
return false
}
c.removeNode(e)
delete(c.items, key)
return true
}
// Purge clears all entries from the cache.
// Calls onEvict for each entry if set.
// Resets hit/miss counters.
func (c *LRU[K, V]) Purge() {
if c == nil || c.size <= 0 {
return
}
c.mu.Lock()
if c.onEvict != nil {
for key, e := range c.items {
c.onEvict(key, e.value)
}
}
c.items = make(map[K]*entry[K, V], c.size)
c.head = nil
c.tail = nil
c.hits.Store(0)
c.misses.Store(0)
c.mu.Unlock()
}
// Len returns the current number of items in the cache.
func (c *LRU[K, V]) Len() int {
if c == nil || c.size <= 0 {
return 0
}
c.mu.Lock()
n := len(c.items)
c.mu.Unlock()
return n
}
// Cap returns the maximum capacity of the cache.
func (c *LRU[K, V]) Cap() int {
if c == nil {
return 0
}
return c.size
}
// HitRate returns the cache hit ratio (0.0 to 1.0).
// Based on hits / (hits + misses).
func (c *LRU[K, V]) HitRate() float64 {
h := c.hits.Load()
m := c.misses.Load()
total := h + m
if total == 0 {
return 0.0
}
return float64(h) / float64(total)
}
// RemoveOldest removes and returns the least recently used item.
// Returns key, value, and true if an item was removed.
// Calls onEvict if set.
func (c *LRU[K, V]) RemoveOldest() (key K, value V, ok bool) {
if c == nil || c.size <= 0 {
return
}
c.mu.Lock()
defer c.mu.Unlock()
if c.tail == nil {
return
}
key = c.tail.key
value = c.tail.value
c.removeOldest()
return key, value, true
}
// moveToFront moves an entry to the front (MRU position).
func (c *LRU[K, V]) moveToFront(e *entry[K, V]) {
if c.head == e {
return
}
c.removeNode(e)
c.addToFront(e)
}
// addToFront adds an entry to the front of the list.
func (c *LRU[K, V]) addToFront(e *entry[K, V]) {
e.prev = nil
e.next = c.head
if c.head != nil {
c.head.prev = e
}
c.head = e
if c.tail == nil {
c.tail = e
}
}
// removeNode removes an entry from the linked list.
func (c *LRU[K, V]) removeNode(e *entry[K, V]) {
if e.prev != nil {
e.prev.next = e.next
} else {
c.head = e.next
}
if e.next != nil {
e.next.prev = e.prev
} else {
c.tail = e.prev
}
e.prev = nil
e.next = nil
}
// removeOldest removes the tail entry (LRU).
// Calls onEvict if set and deletes from map.
func (c *LRU[K, V]) removeOldest() {
if c.tail == nil {
return
}
e := c.tail
if c.onEvict != nil {
c.onEvict(e.key, e.value)
}
c.removeNode(e)
delete(c.items, e.key)
}

View File

@@ -8,42 +8,53 @@ import (
"github.com/clipperhouse/displaywidth"
"github.com/mattn/go-runewidth"
"github.com/olekukonko/tablewriter/pkg/twcache"
)
// globalOptions holds the global displaywidth configuration, including East Asian width settings.
var globalOptions displaywidth.Options
const (
cacheCapacity = 8192
// mu protects access to condition and widthCache for thread safety.
cachePrefix = "0:"
cacheEastAsianPrefix = "1:"
)
// Options allows for configuring width calculation on a per-call basis.
type Options struct {
EastAsianWidth bool
}
// globalOptions holds the global displaywidth configuration, including East Asian width settings.
var globalOptions Options
// mu protects access to globalOptions for thread safety.
var mu sync.Mutex
// widthCache stores memoized results of Width calculations to improve performance.
var widthCache *twcache.LRU[string, int]
// ansi is a compiled regular expression for stripping ANSI escape codes from strings.
var ansi = Filter()
func init() {
globalOptions = newOptions()
widthCache = make(map[cacheKey]int)
}
func newOptions() displaywidth.Options {
// go-runewidth has default logic based on env variables and locale,
// we want to keep that compatibility
// Initialize global options by detecting from the environment,
// which is the one key feature we get from go-runewidth.
cond := runewidth.NewCondition()
options := displaywidth.Options{
EastAsianWidth: cond.EastAsianWidth,
StrictEmojiNeutral: cond.StrictEmojiNeutral,
globalOptions = Options{
EastAsianWidth: cond.EastAsianWidth,
}
return options
widthCache = twcache.NewLRU[string, int](cacheCapacity)
}
// cacheKey is used as a key for memoizing string width results in widthCache.
type cacheKey struct {
str string // Input string
eastAsianWidth bool // East Asian width setting
// makeCacheKey generates a string key for the LRU cache from the input string
// and the current East Asian width setting.
// Prefix "0:" for false, "1:" for true.
func makeCacheKey(str string, eastAsianWidth bool) string {
if eastAsianWidth {
return cacheEastAsianPrefix + str
}
return cachePrefix + str
}
// widthCache stores memoized results of Width calculations to improve performance.
var widthCache map[cacheKey]int
// Filter compiles and returns a regular expression for matching ANSI escape sequences,
// including CSI (Control Sequence Introducer) and OSC (Operating System Command) sequences.
// The returned regex can be used to strip ANSI codes from strings.
@@ -62,20 +73,25 @@ func Filter() *regexp.Regexp {
return regexp.MustCompile("(" + regCSI + "|" + regOSC + ")")
}
// SetEastAsian enables or disables East Asian width handling for width calculations.
// When the setting changes, the width cache is cleared to ensure accuracy.
// SetOptions sets the global options for width calculation.
// This function is thread-safe.
func SetOptions(opts Options) {
mu.Lock()
defer mu.Unlock()
if globalOptions.EastAsianWidth != opts.EastAsianWidth {
globalOptions = opts
widthCache.Purge()
}
}
// SetEastAsian enables or disables East Asian width handling globally.
// This function is thread-safe.
//
// Example:
//
// twdw.SetEastAsian(true) // Enable East Asian width handling
func SetEastAsian(enable bool) {
mu.Lock()
defer mu.Unlock()
if globalOptions.EastAsianWidth != enable {
globalOptions.EastAsianWidth = enable
widthCache = make(map[cacheKey]int) // Clear cache on setting change
}
SetOptions(Options{EastAsianWidth: enable})
}
// IsEastAsian returns the current East Asian width setting.
@@ -92,85 +108,67 @@ func IsEastAsian() bool {
return globalOptions.EastAsianWidth
}
// SetCondition updates the global runewidth.Condition used for width calculations.
// This method is kept for backward compatibility. The condition is converted to
// displaywidth.Options internally for better performance.
// Deprecated: use SetOptions with the new twwidth.Options struct instead.
// This function is kept for backward compatibility.
func SetCondition(cond *runewidth.Condition) {
mu.Lock()
defer mu.Unlock()
widthCache = make(map[cacheKey]int) // Clear cache on setting change
globalOptions = conditionToOptions(cond)
}
// Convert runewidth.Condition to displaywidth.Options
func conditionToOptions(cond *runewidth.Condition) displaywidth.Options {
return displaywidth.Options{
EastAsianWidth: cond.EastAsianWidth,
StrictEmojiNeutral: cond.StrictEmojiNeutral,
newEastAsianWidth := cond.EastAsianWidth
if globalOptions.EastAsianWidth != newEastAsianWidth {
globalOptions.EastAsianWidth = newEastAsianWidth
widthCache.Purge()
}
}
// Width calculates the visual width of a string, excluding ANSI escape sequences,
// using the go-runewidth package for accurate Unicode handling. It accounts for the
// current East Asian width setting and caches results for performance.
// Width calculates the visual width of a string using the global cache for performance.
// It excludes ANSI escape sequences and accounts for the global East Asian width setting.
// This function is thread-safe.
//
// Example:
//
// width := twdw.Width("Hello\x1b[31mWorld") // Returns 10
func Width(str string) int {
mu.Lock()
key := cacheKey{str: str, eastAsianWidth: globalOptions.EastAsianWidth}
if w, found := widthCache[key]; found {
mu.Unlock()
currentEA := IsEastAsian()
key := makeCacheKey(str, currentEA)
if w, found := widthCache.Get(key); found {
return w
}
mu.Unlock()
options := newOptions()
options.EastAsianWidth = key.eastAsianWidth
opts := displaywidth.Options{EastAsianWidth: currentEA}
stripped := ansi.ReplaceAllLiteralString(str, "")
calculatedWidth := options.String(stripped)
mu.Lock()
widthCache[key] = calculatedWidth
mu.Unlock()
calculatedWidth := opts.String(stripped)
widthCache.Add(key, calculatedWidth)
return calculatedWidth
}
// WidthNoCache calculates the visual width of a string without using or
// updating the global cache. It uses the current global East Asian width setting.
// This function is intended for internal use (e.g., benchmarking) and is thread-safe.
// WidthWithOptions calculates the visual width of a string with specific options,
// bypassing the global settings and cache. This is useful for one-shot calculations
// where global state is not desired.
func WidthWithOptions(str string, opts Options) int {
dwOpts := displaywidth.Options{EastAsianWidth: opts.EastAsianWidth}
stripped := ansi.ReplaceAllLiteralString(str, "")
return dwOpts.String(stripped)
}
// WidthNoCache calculates the visual width of a string without using the global cache.
//
// Example:
//
// width := twdw.WidthNoCache("Hello\x1b[31mWorld") // Returns 10
func WidthNoCache(str string) int {
mu.Lock()
currentEA := globalOptions.EastAsianWidth
mu.Unlock()
options := newOptions()
options.EastAsianWidth = currentEA
stripped := ansi.ReplaceAllLiteralString(str, "")
return options.String(stripped)
// This function's behavior is equivalent to a one-shot calculation
// using the current global options. The WidthWithOptions function
// does not interact with the cache, thus fulfilling the requirement.
return WidthWithOptions(str, Options{EastAsianWidth: IsEastAsian()})
}
// Display calculates the visual width of a string, excluding ANSI escape sequences,
// using the provided runewidth condition. Unlike Width, it does not use caching
// and is intended for cases where a specific condition is required.
// This function is thread-safe with respect to the provided condition.
//
// Example:
//
// cond := runewidth.NewCondition()
// width := twdw.Display(cond, "Hello\x1b[31mWorld") // Returns 10
// Deprecated: use WidthWithOptions with the new twwidth.Options struct instead.
// This function is kept for backward compatibility.
func Display(cond *runewidth.Condition, str string) int {
options := conditionToOptions(cond)
return options.String(ansi.ReplaceAllLiteralString(str, ""))
opts := Options{EastAsianWidth: cond.EastAsianWidth}
return WidthWithOptions(str, opts)
}
// Truncate shortens a string to fit within a specified visual width, optionally
@@ -217,31 +215,34 @@ func Truncate(s string, maxWidth int, suffix ...string) string {
// Case 3: String fits completely or fits with suffix.
// Here, maxWidth is the total budget for the line.
if sDisplayWidth <= maxWidth {
// If the string contains ANSI, we must ensure it ends with a reset
// to prevent bleeding, even if we don't truncate.
safeS := s
if strings.Contains(s, "\x1b") && !strings.HasSuffix(s, "\x1b[0m") {
safeS += "\x1b[0m"
}
if len(suffixStr) == 0 { // No suffix.
return s
return safeS
}
// Suffix is provided. Check if s + suffix fits.
if sDisplayWidth+suffixDisplayWidth <= maxWidth {
return s + suffixStr
return safeS + suffixStr
}
// s fits, but s + suffix is too long. Return s.
return s
// s fits, but s + suffix is too long. Return s (with reset if needed).
return safeS
}
// Case 4: String needs truncation (sDisplayWidth > maxWidth).
// maxWidth is the total budget for the final string (content + suffix).
// Capture the global EastAsianWidth setting once for consistent use
mu.Lock()
currentGlobalEastAsianWidth := globalOptions.EastAsianWidth
mu.Unlock()
currentGlobalEastAsianWidth := IsEastAsian()
// Special case for EastAsian true: if only suffix fits, return suffix.
// This was derived from previous test behavior.
if len(suffixStr) > 0 && currentGlobalEastAsianWidth {
provisionalContentWidth := maxWidth - suffixDisplayWidth
if provisionalContentWidth == 0 { // Exactly enough space for suffix only
return suffixStr // <<<< MODIFIED: No ANSI reset here
return suffixStr
}
}
@@ -263,7 +264,6 @@ func Truncate(s string, maxWidth int, suffix ...string) string {
}
return "" // Cannot fit anything.
}
// If targetContentForIteration is 0, loop won't run, result will be empty string, then suffix is added.
var contentBuf bytes.Buffer
var currentContentDisplayWidth int
@@ -271,8 +271,7 @@ func Truncate(s string, maxWidth int, suffix ...string) string {
inAnsiSequence := false
ansiWrittenToContent := false
options := newOptions()
options.EastAsianWidth = currentGlobalEastAsianWidth
dwOpts := displaywidth.Options{EastAsianWidth: currentGlobalEastAsianWidth}
for _, r := range s {
if r == '\x1b' {
@@ -306,7 +305,7 @@ func Truncate(s string, maxWidth int, suffix ...string) string {
ansiSeqBuf.Reset()
}
} else { // Normal character
runeDisplayWidth := options.Rune(r)
runeDisplayWidth := dwOpts.Rune(r)
if targetContentForIteration == 0 { // No budget for content at all
break
}
@@ -320,32 +319,51 @@ func Truncate(s string, maxWidth int, suffix ...string) string {
result := contentBuf.String()
// Suffix is added if:
// 1. A suffix string is provided.
// 2. Truncation actually happened (sDisplayWidth > maxWidth originally)
// OR if the content part is empty but a suffix is meant to be shown
// (e.g. targetContentForIteration was 0).
if len(suffixStr) > 0 {
// Add suffix if we are in the truncation path (sDisplayWidth > maxWidth)
// OR if targetContentForIteration was 0 (meaning only suffix should be shown)
// but we must ensure we don't exceed original maxWidth.
// The logic above for targetContentForIteration already ensures space.
needsReset := false
// Condition for reset: if styling was active in 's' and might affect suffix
if (ansiWrittenToContent || (inAnsiSequence && strings.Contains(s, "\x1b["))) && (currentContentDisplayWidth > 0 || ansiWrittenToContent) {
if !strings.HasSuffix(result, "\x1b[0m") {
needsReset = true
}
} else if currentContentDisplayWidth > 0 && strings.Contains(result, "\x1b[") && !strings.HasSuffix(result, "\x1b[0m") && strings.Contains(s, "\x1b[") {
// If result has content and ANSI, and original had ANSI, and result not already reset
// Determine if we need to append a reset sequence to prevent color bleeding.
// This is needed if we wrote any ANSI codes or if the input had active codes
// that we might have cut off or left open.
needsReset := false
if (ansiWrittenToContent || (inAnsiSequence && strings.Contains(s, "\x1b["))) && (currentContentDisplayWidth > 0 || ansiWrittenToContent) {
if !strings.HasSuffix(result, "\x1b[0m") {
needsReset = true
}
} else if currentContentDisplayWidth > 0 && strings.Contains(result, "\x1b[") && !strings.HasSuffix(result, "\x1b[0m") && strings.Contains(s, "\x1b[") {
needsReset = true
}
if needsReset {
result += "\x1b[0m"
}
if needsReset {
result += "\x1b[0m"
}
// Suffix is added if provided.
if len(suffixStr) > 0 {
result += suffixStr
}
return result
}
// SetCacheCapacity changes the cache size dynamically
// If capacity <= 0, disables caching entirely
func SetCacheCapacity(capacity int) {
mu.Lock()
defer mu.Unlock()
if capacity <= 0 {
widthCache = nil // nil = fully disabled
return
}
newCache := twcache.NewLRU[string, int](capacity)
widthCache = newCache
}
// GetCacheStats returns current cache statistics
func GetCacheStats() (size, capacity int, hitRate float64) {
mu.Lock()
defer mu.Unlock()
if widthCache == nil {
return 0, 0, 0
}
return widthCache.Len(), widthCache.Cap(), widthCache.HitRate()
}

View File

@@ -8,11 +8,11 @@ import (
"reflect"
"runtime"
"strings"
"sync"
"github.com/olekukonko/errors"
"github.com/olekukonko/ll"
"github.com/olekukonko/ll/lh"
"github.com/olekukonko/tablewriter/pkg/twcache"
"github.com/olekukonko/tablewriter/pkg/twwarp"
"github.com/olekukonko/tablewriter/pkg/twwidth"
"github.com/olekukonko/tablewriter/renderer"
@@ -52,9 +52,7 @@ type Table struct {
streamRowCounter int // Counter for rows rendered in streaming mode (0-indexed logical rows)
// cache
stringerCache map[reflect.Type]reflect.Value // Cache for stringer reflection
stringerCacheMu sync.RWMutex // Mutex for thread-safe cache access
stringerCacheEnabled bool // Flag to enable/disable caching
stringerCache twcache.Cache[reflect.Type, reflect.Value] // Cache for stringer reflection
batchRenderNumCols int
isBatchRenderNumColsSet bool
@@ -126,8 +124,7 @@ func NewTable(w io.Writer, opts ...Option) *Table {
streamRowCounter: 0,
// Cache
stringerCache: make(map[reflect.Type]reflect.Value),
stringerCacheEnabled: false, // Disabled by default
stringerCache: twcache.NewLRU[reflect.Type, reflect.Value](tw.DefaultCacheStringCapacity),
}
// set Options
@@ -483,10 +480,11 @@ func (t *Table) Reset() {
t.streamRowCounter = 0
// The stringer and its cache are part of the table's configuration,
if t.stringerCacheEnabled {
t.stringerCacheMu.Lock()
t.stringerCache = make(map[reflect.Type]reflect.Value)
t.stringerCacheMu.Unlock()
if t.stringerCache == nil {
t.stringerCache = twcache.NewLRU[reflect.Type, reflect.Value](tw.DefaultCacheStringCapacity)
t.logger.Debug("Reset(): Stringer cache reset to default capacity.")
} else {
t.stringerCache.Purge()
t.logger.Debug("Reset(): Stringer cache cleared.")
}

View File

@@ -8,6 +8,8 @@ const (
Success = 1 // Operation succeeded
MinimumColumnWidth = 8
DefaultCacheStringCapacity = 10 * 1024 // 10 KB
)
const (

View File

@@ -1064,17 +1064,13 @@ func (t *Table) convertToStringer(input interface{}) ([]string, error) {
t.logger.Debugf("convertToString attempt %v using %v", input, t.stringer)
inputType := reflect.TypeOf(input)
stringerFuncVal := reflect.ValueOf(t.stringer)
stringerFuncType := stringerFuncVal.Type()
// Cache lookup (simplified, actual cache logic can be more complex)
if t.stringerCacheEnabled {
t.stringerCacheMu.RLock()
cachedFunc, ok := t.stringerCache[inputType]
t.stringerCacheMu.RUnlock()
if ok {
// Add proper type checking for cachedFunc against input here if necessary
// Cache lookup using twcache.LRU
// This assumes t.stringerCache is *twcache.LRU[reflect.Type, reflect.Value]
if t.stringerCache != nil {
if cachedFunc, ok := t.stringerCache.Get(inputType); ok {
t.logger.Debugf("convertToStringer: Cache hit for type %v", inputType)
// We can proceed to call it immediately because it's already been validated/cached
results := cachedFunc.Call([]reflect.Value{reflect.ValueOf(input)})
if len(results) == 1 && results[0].Type() == reflect.TypeOf([]string{}) {
return results[0].Interface().([]string), nil
@@ -1082,6 +1078,9 @@ func (t *Table) convertToStringer(input interface{}) ([]string, error) {
}
}
stringerFuncVal := reflect.ValueOf(t.stringer)
stringerFuncType := stringerFuncVal.Type()
// Robust type checking for the stringer function
validSignature := stringerFuncVal.Kind() == reflect.Func &&
stringerFuncType.NumIn() == 1 &&
@@ -1105,10 +1104,6 @@ func (t *Table) convertToStringer(input interface{}) ([]string, error) {
}
} else if paramType.Kind() == reflect.Interface || (paramType.Kind() == reflect.Ptr && paramType.Elem().Kind() != reflect.Interface) {
// If input is nil, it can be assigned if stringer expects an interface or a pointer type
// (but not a pointer to an interface, which is rare for stringers).
// A nil value for a concrete type parameter would cause a panic on Call.
// So, if paramType is not an interface/pointer, and input is nil, it's an issue.
// This needs careful handling. For now, assume assignable if interface/pointer.
assignable = true
}
@@ -1120,7 +1115,6 @@ func (t *Table) convertToStringer(input interface{}) ([]string, error) {
if input == nil {
// If input is nil, we must pass a zero value of the stringer's parameter type
// if that type is a pointer or interface.
// Passing reflect.ValueOf(nil) directly will cause issues if paramType is concrete.
callArgs = []reflect.Value{reflect.Zero(paramType)}
} else {
callArgs = []reflect.Value{reflect.ValueOf(input)}
@@ -1128,10 +1122,9 @@ func (t *Table) convertToStringer(input interface{}) ([]string, error) {
resultValues := stringerFuncVal.Call(callArgs)
if t.stringerCacheEnabled && inputType != nil { // Only cache if inputType is valid
t.stringerCacheMu.Lock()
t.stringerCache[inputType] = stringerFuncVal
t.stringerCacheMu.Unlock()
// Add to cache if enabled (not nil) and input type is valid
if t.stringerCache != nil && inputType != nil {
t.stringerCache.Add(inputType, stringerFuncVal)
}
return resultValues[0].Interface().([]string), nil

View File

@@ -1,3 +1,19 @@
## 2.27.5
### Fixes
Don't make a new formatter for each GinkgoT(); that's just silly and uses precious memory
## 2.27.4
### Fixes
- CurrentTreeConstructionNodeReport: fix for nested container nodes [59bc751]
## 2.27.3
### Fixes
report exit result in case of failure [1c9f356]
fix data race [ece19c8]
## 2.27.2
### Fixes

View File

@@ -9,6 +9,7 @@ import (
"path/filepath"
"regexp"
"strings"
"sync/atomic"
"syscall"
"time"
@@ -159,12 +160,15 @@ func runSerial(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig t
func runParallel(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig types.ReporterConfig, cliConfig types.CLIConfig, goFlagsConfig types.GoFlagsConfig, additionalArgs []string) TestSuite {
type procResult struct {
proc int
exitResult string
passed bool
hasProgrammaticFocus bool
}
numProcs := cliConfig.ComputedProcs()
procOutput := make([]*bytes.Buffer, numProcs)
procExitResult := make([]string, numProcs)
coverProfiles := []string{}
blockProfiles := []string{}
@@ -224,16 +228,20 @@ func runParallel(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig
args = append(args, additionalArgs...)
cmd, buf := buildAndStartCommand(suite, args, false)
var exited atomic.Bool
procOutput[proc-1] = buf
server.RegisterAlive(proc, func() bool { return cmd.ProcessState == nil || !cmd.ProcessState.Exited() })
server.RegisterAlive(proc, func() bool { return !exited.Load() })
go func() {
cmd.Wait()
exitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
procResults <- procResult{
proc: proc,
exitResult: cmd.ProcessState.String(),
passed: (exitStatus == 0) || (exitStatus == types.GINKGO_FOCUS_EXIT_CODE),
hasProgrammaticFocus: exitStatus == types.GINKGO_FOCUS_EXIT_CODE,
}
exited.Store(true)
}()
}
@@ -242,6 +250,7 @@ func runParallel(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig
result := <-procResults
passed = passed && result.passed
suite.HasProgrammaticFocus = suite.HasProgrammaticFocus || result.hasProgrammaticFocus
procExitResult[result.proc-1] = result.exitResult
}
if passed {
suite.State = TestSuiteStatePassed
@@ -261,6 +270,8 @@ func runParallel(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig
for proc := 1; proc <= cliConfig.ComputedProcs(); proc++ {
fmt.Fprintf(formatter.ColorableStdErr, formatter.F("{{bold}}Output from proc %d:{{/}}\n", proc))
fmt.Fprintln(os.Stderr, formatter.Fi(1, "%s", procOutput[proc-1].String()))
fmt.Fprintf(formatter.ColorableStdErr, formatter.F("{{bold}}Exit result of proc %d:{{/}}\n", proc))
fmt.Fprintln(os.Stderr, formatter.Fi(1, "%s\n", procExitResult[proc-1]))
}
fmt.Fprintf(os.Stderr, "** End **")
}

View File

@@ -208,9 +208,12 @@ func (suite *Suite) PushNode(node Node) error {
// Ensure that code running in the body of the container node
// has access to information about the current container node(s).
// The current one (nil in top-level container nodes, non-nil in an
// embedded container node) gets restored when the node is done.
oldConstructionNodeReport := suite.currentConstructionNodeReport
suite.currentConstructionNodeReport = constructionNodeReportForTreeNode(suite.tree)
defer func() {
suite.currentConstructionNodeReport = nil
suite.currentConstructionNodeReport = oldConstructionNodeReport
}()
node.Body(nil)

View File

@@ -27,6 +27,11 @@ type ginkgoWriterInterface interface {
type ginkgoRecoverFunc func()
type attachProgressReporterFunc func(func() string) func()
var formatters = map[bool]formatter.Formatter{
true: formatter.NewWithNoColorBool(true),
false: formatter.NewWithNoColorBool(false),
}
func New(writer ginkgoWriterInterface, fail failFunc, skip skipFunc, cleanup cleanupFunc, report reportFunc, addReportEntry addReportEntryFunc, ginkgoRecover ginkgoRecoverFunc, attachProgressReporter attachProgressReporterFunc, randomSeed int64, parallelProcess int, parallelTotal int, noColor bool, offset int) *ginkgoTestingTProxy {
return &ginkgoTestingTProxy{
fail: fail,
@@ -41,7 +46,7 @@ func New(writer ginkgoWriterInterface, fail failFunc, skip skipFunc, cleanup cle
randomSeed: randomSeed,
parallelProcess: parallelProcess,
parallelTotal: parallelTotal,
f: formatter.NewWithNoColorBool(noColor),
f: formatters[noColor], //minimize allocations by reusing formatters
}
}

View File

@@ -1,3 +1,3 @@
package types
const VERSION = "2.27.2"
const VERSION = "2.27.5"

View File

@@ -1,3 +1,14 @@
## 1.39.0
### Features
Add `MatchErrorStrictly` which only passes if `errors.Is(actual, expected)` returns true. `MatchError`, by contrast, will fallback to string comparison.
## 1.38.3
### Fixes
make string formatitng more consistent for users who use format.Object directly
## 1.38.2
- roll back to go 1.23.0 [c404969]

View File

@@ -262,7 +262,7 @@ func Object(object any, indentation uint) string {
if err, ok := object.(error); ok && !isNilValue(value) { // isNilValue check needed here to avoid nil deref due to boxed nil
commonRepresentation += "\n" + IndentString(err.Error(), indentation) + "\n" + indent
}
return fmt.Sprintf("%s<%s>: %s%s", indent, formatType(value), commonRepresentation, formatValue(value, indentation))
return fmt.Sprintf("%s<%s>: %s%s", indent, formatType(value), commonRepresentation, formatValue(value, indentation, true))
}
/*
@@ -306,7 +306,7 @@ func formatType(v reflect.Value) string {
}
}
func formatValue(value reflect.Value, indentation uint) string {
func formatValue(value reflect.Value, indentation uint, isTopLevel bool) string {
if indentation > MaxDepth {
return "..."
}
@@ -367,11 +367,11 @@ func formatValue(value reflect.Value, indentation uint) string {
case reflect.Func:
return fmt.Sprintf("0x%x", value.Pointer())
case reflect.Ptr:
return formatValue(value.Elem(), indentation)
return formatValue(value.Elem(), indentation, isTopLevel)
case reflect.Slice:
return truncateLongStrings(formatSlice(value, indentation))
case reflect.String:
return truncateLongStrings(formatString(value.String(), indentation))
return truncateLongStrings(formatString(value.String(), indentation, isTopLevel))
case reflect.Array:
return truncateLongStrings(formatSlice(value, indentation))
case reflect.Map:
@@ -392,8 +392,8 @@ func formatValue(value reflect.Value, indentation uint) string {
}
}
func formatString(object any, indentation uint) string {
if indentation == 1 {
func formatString(object any, indentation uint, isTopLevel bool) string {
if isTopLevel {
s := fmt.Sprintf("%s", object)
components := strings.Split(s, "\n")
result := ""
@@ -416,14 +416,14 @@ func formatString(object any, indentation uint) string {
func formatSlice(v reflect.Value, indentation uint) string {
if v.Kind() == reflect.Slice && v.Type().Elem().Kind() == reflect.Uint8 && isPrintableString(string(v.Bytes())) {
return formatString(v.Bytes(), indentation)
return formatString(v.Bytes(), indentation, false)
}
l := v.Len()
result := make([]string, l)
longest := 0
for i := 0; i < l; i++ {
result[i] = formatValue(v.Index(i), indentation+1)
for i := range l {
result[i] = formatValue(v.Index(i), indentation+1, false)
if len(result[i]) > longest {
longest = len(result[i])
}
@@ -443,7 +443,7 @@ func formatMap(v reflect.Value, indentation uint) string {
longest := 0
for i, key := range v.MapKeys() {
value := v.MapIndex(key)
result[i] = fmt.Sprintf("%s: %s", formatValue(key, indentation+1), formatValue(value, indentation+1))
result[i] = fmt.Sprintf("%s: %s", formatValue(key, indentation+1, false), formatValue(value, indentation+1, false))
if len(result[i]) > longest {
longest = len(result[i])
}
@@ -462,10 +462,10 @@ func formatStruct(v reflect.Value, indentation uint) string {
l := v.NumField()
result := []string{}
longest := 0
for i := 0; i < l; i++ {
for i := range l {
structField := t.Field(i)
fieldEntry := v.Field(i)
representation := fmt.Sprintf("%s: %s", structField.Name, formatValue(fieldEntry, indentation+1))
representation := fmt.Sprintf("%s: %s", structField.Name, formatValue(fieldEntry, indentation+1, false))
result = append(result, representation)
if len(representation) > longest {
longest = len(representation)
@@ -479,7 +479,7 @@ func formatStruct(v reflect.Value, indentation uint) string {
}
func formatInterface(v reflect.Value, indentation uint) string {
return fmt.Sprintf("<%s>%s", formatType(v.Elem()), formatValue(v.Elem(), indentation))
return fmt.Sprintf("<%s>%s", formatType(v.Elem()), formatValue(v.Elem(), indentation, false))
}
func isNilValue(a reflect.Value) bool {

View File

@@ -22,7 +22,7 @@ import (
"github.com/onsi/gomega/types"
)
const GOMEGA_VERSION = "1.38.2"
const GOMEGA_VERSION = "1.39.0"
const nilGomegaPanic = `You are trying to make an assertion, but haven't registered Gomega's fail handler.
If you're using Ginkgo then you probably forgot to put your assertion in an It().

View File

@@ -146,6 +146,24 @@ func MatchError(expected any, functionErrorDescription ...any) types.GomegaMatch
}
}
// MatchErrorStrictly succeeds iff actual is a non-nil error that matches the passed in
// expected error according to errors.Is(actual, expected).
//
// This behavior differs from MatchError where
//
// Expect(errors.New("some error")).To(MatchError(errors.New("some error")))
//
// succeeds, but errors.Is would return false so:
//
// Expect(errors.New("some error")).To(MatchErrorStrictly(errors.New("some error")))
//
// fails.
func MatchErrorStrictly(expected error) types.GomegaMatcher {
return &matchers.MatchErrorStrictlyMatcher{
Expected: expected,
}
}
// BeClosed succeeds if actual is a closed channel.
// It is an error to pass a non-channel to BeClosed, it is also an error to pass nil
//
@@ -515,8 +533,8 @@ func HaveExistingField(field string) types.GomegaMatcher {
// and even interface values.
//
// actual := 42
// Expect(actual).To(HaveValue(42))
// Expect(&actual).To(HaveValue(42))
// Expect(actual).To(HaveValue(Equal(42)))
// Expect(&actual).To(HaveValue(Equal(42)))
func HaveValue(matcher types.GomegaMatcher) types.GomegaMatcher {
return &matchers.HaveValueMatcher{
Matcher: matcher,

View File

@@ -39,7 +39,7 @@ func (matcher *HaveKeyMatcher) Match(actual any) (success bool, err error) {
}
keys := reflect.ValueOf(actual).MapKeys()
for i := 0; i < len(keys); i++ {
for i := range keys {
success, err := keyMatcher.Match(keys[i].Interface())
if err != nil {
return false, fmt.Errorf("HaveKey's key matcher failed with:\n%s%s", format.Indent, err.Error())

View File

@@ -52,7 +52,7 @@ func (matcher *HaveKeyWithValueMatcher) Match(actual any) (success bool, err err
}
keys := reflect.ValueOf(actual).MapKeys()
for i := 0; i < len(keys); i++ {
for i := range keys {
success, err := keyMatcher.Match(keys[i].Interface())
if err != nil {
return false, fmt.Errorf("HaveKeyWithValue's key matcher failed with:\n%s%s", format.Indent, err.Error())

View File

@@ -0,0 +1,39 @@
package matchers
import (
"errors"
"fmt"
"github.com/onsi/gomega/format"
)
type MatchErrorStrictlyMatcher struct {
Expected error
}
func (matcher *MatchErrorStrictlyMatcher) Match(actual any) (success bool, err error) {
if isNil(matcher.Expected) {
return false, fmt.Errorf("Expected error is nil, use \"ToNot(HaveOccurred())\" to explicitly check for nil errors")
}
if isNil(actual) {
return false, fmt.Errorf("Expected an error, got nil")
}
if !isError(actual) {
return false, fmt.Errorf("Expected an error. Got:\n%s", format.Object(actual, 1))
}
actualErr := actual.(error)
return errors.Is(actualErr, matcher.Expected), nil
}
func (matcher *MatchErrorStrictlyMatcher) FailureMessage(actual any) (message string) {
return format.Message(actual, "to match error", matcher.Expected)
}
func (matcher *MatchErrorStrictlyMatcher) NegatedFailureMessage(actual any) (message string) {
return format.Message(actual, "not to match error", matcher.Expected)
}

View File

@@ -1,6 +1,9 @@
package edge
import . "github.com/onsi/gomega/matchers/support/goraph/node"
import (
. "github.com/onsi/gomega/matchers/support/goraph/node"
"slices"
)
type Edge struct {
Node1 int
@@ -20,13 +23,7 @@ func (ec EdgeSet) Free(node Node) bool {
}
func (ec EdgeSet) Contains(edge Edge) bool {
for _, e := range ec {
if e == edge {
return true
}
}
return false
return slices.Contains(ec, edge)
}
func (ec EdgeSet) FindByNodes(node1, node2 Node) (Edge, bool) {

View File

@@ -1,28 +0,0 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ocdav
import (
// initialize reva registries by importing the relevant loader packages
// see cmd/revad/runtime/loader.go for other loaders if a service is not found
_ "github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/loader"
_ "github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/token/loader"
_ "github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/tokenwriter/loader"
_ "github.com/opencloud-eu/reva/v2/pkg/token/manager/loader"
)

View File

@@ -1,376 +0,0 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ocdav
import (
"context"
"crypto/tls"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/config"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storage/favorite"
"github.com/rs/zerolog"
"go-micro.dev/v4/broker"
"go.opentelemetry.io/otel/trace"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
TLSConfig *tls.Config
Broker broker.Broker
Address string
Logger zerolog.Logger
Context context.Context
// Metrics *metrics.Metrics
// Flags []cli.Flag
Name string
JWTSecret string
FavoriteManager favorite.Manager
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
TracesExporter string
TraceProvider trace.TracerProvider
MetricsEnabled bool
MetricsNamespace string
MetricsSubsystem string
// ocdav.* is internal so we need to set config options individually
config config.Config
lockSystem ocdav.LockSystem
AllowCredentials bool
AllowedOrigins []string
AllowedHeaders []string
AllowedMethods []string
AllowDepthInfinity bool
RegisterTTL time.Duration
RegisterInterval time.Duration
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// TLSConfig provides a function to set the TLSConfig option.
func TLSConfig(config *tls.Config) Option {
return func(o *Options) {
o.TLSConfig = config
}
}
// Broker provides a function to set the Broker option.
func Broker(b broker.Broker) Option {
return func(o *Options) {
o.Broker = b
}
}
// Address provides a function to set the address option.
func Address(val string) Option {
return func(o *Options) {
o.Address = val
}
}
func AllowDepthInfinity(val bool) Option {
return func(o *Options) {
o.AllowDepthInfinity = val
}
}
// JWTSecret provides a function to set the jwt secret option.
func JWTSecret(s string) Option {
return func(o *Options) {
o.JWTSecret = s
}
}
// MachineAuthAPIKey provides a function to set the machine auth api key option.
func MachineAuthAPIKey(s string) Option {
return func(o *Options) {
o.config.MachineAuthAPIKey = s
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Logger provides a function to set the logger option.
func Logger(val zerolog.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Name provides a function to set the Name option.
func Name(val string) Option {
return func(o *Options) {
o.Name = val
}
}
// Prefix provides a function to set the prefix config option.
func Prefix(val string) Option {
return func(o *Options) {
o.config.Prefix = val
}
}
// FilesNamespace provides a function to set the FilesNamespace config option.
func FilesNamespace(val string) Option {
return func(o *Options) {
o.config.FilesNamespace = val
}
}
// WebdavNamespace provides a function to set the WebdavNamespace config option.
func WebdavNamespace(val string) Option {
return func(o *Options) {
o.config.WebdavNamespace = val
}
}
// SharesNamespace provides a function to set the SharesNamespace config option.
func SharesNamespace(val string) Option {
return func(o *Options) {
o.config.SharesNamespace = val
}
}
// OCMNamespace provides a function to set the OCMNamespace config option.
func OCMNamespace(val string) Option {
return func(o *Options) {
o.config.OCMNamespace = val
}
}
// GatewaySvc provides a function to set the GatewaySvc config option.
func GatewaySvc(val string) Option {
return func(o *Options) {
o.config.GatewaySvc = val
}
}
// Timeout provides a function to set the Timeout config option.
func Timeout(val int64) Option {
return func(o *Options) {
o.config.Timeout = val
}
}
// Insecure provides a function to set the Insecure config option.
func Insecure(val bool) Option {
return func(o *Options) {
o.config.Insecure = val
}
}
// PublicURL provides a function to set the PublicURL config option.
func PublicURL(val string) Option {
return func(o *Options) {
o.config.PublicURL = val
}
}
// FavoriteManager provides a function to set the FavoriteManager option.
func FavoriteManager(val favorite.Manager) Option {
return func(o *Options) {
o.FavoriteManager = val
}
}
// GatewaySelector provides a function to set the GatewaySelector option.
func GatewaySelector(val pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewaySelector = val
}
}
// LockSystem provides a function to set the LockSystem option.
func LockSystem(val ocdav.LockSystem) Option {
return func(o *Options) {
o.lockSystem = val
}
}
// WithTracesExporter option
func WithTracesExporter(exporter string) Option {
return func(o *Options) {
o.TracesExporter = exporter
}
}
// WithTracingExporter option
// Deprecated: unused
func WithTracingExporter(exporter string) Option {
return func(o *Options) {}
}
// WithTraceProvider option
func WithTraceProvider(provider trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = provider
}
}
// Version provides a function to set the Version config option.
func Version(val string) Option {
return func(o *Options) {
o.config.Version = val
}
}
// VersionString provides a function to set the VersionString config option.
func VersionString(val string) Option {
return func(o *Options) {
o.config.VersionString = val
}
}
// Edition provides a function to set the Edition config option.
func Edition(val string) Option {
return func(o *Options) {
o.config.Edition = val
}
}
// Product provides a function to set the Product config option.
func Product(val string) Option {
return func(o *Options) {
o.config.Product = val
}
}
// ProductName provides a function to set the ProductName config option.
func ProductName(val string) Option {
return func(o *Options) {
o.config.ProductName = val
}
}
// ProductVersion provides a function to set the ProductVersion config option.
func ProductVersion(val string) Option {
return func(o *Options) {
o.config.ProductVersion = val
}
}
// MetricsEnabled provides a function to set the MetricsEnabled config option.
func MetricsEnabled(val bool) Option {
return func(o *Options) {
o.MetricsEnabled = val
}
}
// MetricsNamespace provides a function to set the MetricsNamespace config option.
func MetricsNamespace(val string) Option {
return func(o *Options) {
o.MetricsNamespace = val
}
}
// MetricsSubsystem provides a function to set the MetricsSubsystem config option.
func MetricsSubsystem(val string) Option {
return func(o *Options) {
o.MetricsSubsystem = val
}
}
// AllowCredentials provides a function to set the AllowCredentials option.
func AllowCredentials(val bool) Option {
return func(o *Options) {
o.AllowCredentials = val
}
}
// AllowedOrigins provides a function to set the AllowedOrigins option.
func AllowedOrigins(val []string) Option {
return func(o *Options) {
o.AllowedOrigins = val
}
}
// AllowedMethods provides a function to set the AllowedMethods option.
func AllowedMethods(val []string) Option {
return func(o *Options) {
o.AllowedMethods = val
}
}
// AllowedHeaders provides a function to set the AllowedHeaders option.
func AllowedHeaders(val []string) Option {
return func(o *Options) {
o.AllowedHeaders = val
}
}
// ItemNameInvalidChars provides a function to set forbidden characters in file or folder names
func ItemNameInvalidChars(chars []string) Option {
return func(o *Options) {
o.config.NameValidation.InvalidChars = chars
}
}
// ItemNameMaxLength provides a function to set the maximum length of a file or folder name
func ItemNameMaxLength(i int) Option {
return func(o *Options) {
o.config.NameValidation.MaxLength = i
}
}
// RegisterTTL provides a function to set the RegisterTTL option.
func RegisterTTL(ttl time.Duration) Option {
return func(o *Options) {
o.RegisterTTL = ttl
}
}
// RegisterInterval provides a function to set the RegisterInterval option.
func RegisterInterval(interval time.Duration) Option {
return func(o *Options) {
o.RegisterInterval = interval
}
}
// URLSigningSharedSecret provides a function to set the URLSigningSharedSecret config option.
func URLSigningSharedSecret(secret string) Option {
return func(o *Options) {
o.config.URLSigningSharedSecret = secret
}
}

View File

@@ -1,229 +0,0 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ocdav
import (
"net/http"
"strings"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
httpServer "github.com/go-micro/plugins/v4/server/http"
"github.com/opencloud-eu/opencloud/pkg/registry"
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/appctx"
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth"
cors2 "github.com/opencloud-eu/reva/v2/internal/http/interceptors/cors"
revaLogMiddleware "github.com/opencloud-eu/reva/v2/internal/http/interceptors/log"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/global"
"github.com/opencloud-eu/reva/v2/pkg/storage/favorite/memory"
rtrace "github.com/opencloud-eu/reva/v2/pkg/trace"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"go-micro.dev/v4"
"go-micro.dev/v4/server"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
func init() {
// register method with chi before any routing is set up
chi.RegisterMethod(ocdav.MethodPropfind)
chi.RegisterMethod(ocdav.MethodProppatch)
chi.RegisterMethod(ocdav.MethodLock)
chi.RegisterMethod(ocdav.MethodUnlock)
chi.RegisterMethod(ocdav.MethodCopy)
chi.RegisterMethod(ocdav.MethodMove)
chi.RegisterMethod(ocdav.MethodMkcol)
chi.RegisterMethod(ocdav.MethodReport)
}
const (
// ServerName to use when announcing the service to the registry
ServerName = "ocdav"
)
// Service initializes the ocdav service and underlying http server.
func Service(opts ...Option) (micro.Service, error) {
sopts := newOptions(opts...)
// set defaults
if err := setDefaults(&sopts); err != nil {
return nil, err
}
sopts.Logger = sopts.Logger.With().Str("name", sopts.Name).Logger()
srv := httpServer.NewServer(
server.Broker(sopts.Broker),
server.TLSConfig(sopts.TLSConfig),
server.Name(sopts.Name),
server.Address(sopts.Address), // Address defaults to ":0" and will pick any free port
server.Version(sopts.config.VersionString),
server.RegisterTTL(sopts.RegisterTTL),
server.RegisterInterval(sopts.RegisterInterval),
)
revaService, err := ocdav.NewWith(&sopts.config, sopts.FavoriteManager, sopts.lockSystem, &sopts.Logger, sopts.GatewaySelector)
if err != nil {
return nil, err
}
r := chi.NewRouter()
tp := sopts.TraceProvider
if tp == nil {
tp = rtrace.NewTracerProvider(sopts.Name, sopts.TracesExporter)
}
if err := useMiddlewares(r, &sopts, revaService, tp); err != nil {
return nil, err
}
r.Handle("/*", revaService.Handler())
_ = chi.Walk(r, func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error {
sopts.Logger.Debug().Str("service", "ocdav").Str("method", method).Str("route", route).Int("middlewares", len(middlewares)).Msg("serving endpoint")
return nil
})
hd := srv.NewHandler(r)
if err := srv.Handle(hd); err != nil {
return nil, err
}
service := micro.NewService(
micro.Server(srv),
micro.Registry(registry.GetRegistry()),
)
// finally, return the service so it can be Run() by the caller himself
return service, nil
}
func setDefaults(sopts *Options) error {
// set defaults
if sopts.Name == "" {
sopts.Name = ServerName
}
if sopts.lockSystem == nil {
selector, err := pool.GatewaySelector(sopts.config.GatewaySvc)
if err != nil {
return errors.Wrap(err, "error getting gateway selector")
}
sopts.lockSystem = ocdav.NewCS3LS(selector)
}
if sopts.FavoriteManager == nil {
sopts.FavoriteManager, _ = memory.New(map[string]interface{}{})
}
if !strings.HasPrefix(sopts.config.Prefix, "/") {
sopts.config.Prefix = "/" + sopts.config.Prefix
}
if sopts.config.VersionString == "" {
sopts.config.VersionString = "0.0.0"
}
sopts.config.AllowPropfindDepthInfinitiy = sopts.AllowDepthInfinity
return nil
}
func useMiddlewares(r *chi.Mux, sopts *Options, svc global.Service, tp trace.TracerProvider) error {
// auth
for _, v := range svc.Unprotected() {
sopts.Logger.Info().Str("url", v).Msg("unprotected URL")
}
authMiddle, err := auth.New(map[string]interface{}{
"gatewaysvc": sopts.config.GatewaySvc,
"token_managers": map[string]interface{}{
"jwt": map[string]interface{}{
"secret": sopts.JWTSecret,
},
},
}, svc.Unprotected(), tp)
if err != nil {
return err
}
// log
lm := revaLogMiddleware.New()
cors, _, err := cors2.New(map[string]interface{}{
"allow_credentials": sopts.AllowCredentials,
"allowed_methods": sopts.AllowedMethods,
"allowed_headers": sopts.AllowedHeaders,
"allowed_origins": sopts.AllowedOrigins,
})
if err != nil {
return err
}
// tracing
tm := traceHandler(tp, "ocdav")
// metrics
pm := func(h http.Handler) http.Handler { return h }
if sopts.MetricsEnabled {
namespace := sopts.MetricsNamespace
if namespace == "" {
namespace = "reva"
}
subsystem := sopts.MetricsSubsystem
if subsystem == "" {
subsystem = "ocdav"
}
counter := promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "http_requests_total",
Help: "The total number of processed " + subsystem + " HTTP requests for " + namespace,
})
pm = func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h.ServeHTTP(w, r)
counter.Inc()
})
}
}
// ctx
cm := appctx.New(sopts.Logger, tp)
// request-id
rm := middleware.RequestID
// actually register
r.Use(pm, tm, lm, authMiddle, rm, cm, cors)
return nil
}
func traceHandler(tp trace.TracerProvider, name string) func(http.Handler) http.Handler {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := rtrace.Propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
t := tp.Tracer("reva")
ctx, span := t.Start(ctx, name)
defer span.End()
rtrace.Propagator.Inject(ctx, propagation.HeaderCarrier(r.Header))
h.ServeHTTP(w, r.WithContext(ctx))
})
}
}

Some files were not shown because too many files have changed in this diff Show More