mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-07-16 02:22:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

[full-ci] chore: reva bump -2.47.0 (#3127)

Browse Source
This commit is contained in:
Viktor Scharf
2026-07-14 13:15:54 +02:00
committed by GitHub
parent a53eb44083
commit 4d511a3a42
182 changed files with 5796 additions and 4298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
vendor/github.com/Masterminds/semver/v3/.gitignore generated vendored
View File

@@ -1 +1,2 @@
_fuzz/
_fuzz/
.devcontainer/

59
vendor/github.com/Masterminds/semver/v3/.golangci.yml generated vendored
View File

@@ -1,27 +1,42 @@
run:
deadline: 2m
version: "2"
linters:
disable-all: true
default: none
enable:
- misspell
- govet
- staticcheck
- errcheck
- unparam
- ineffassign
- nakedret
- gocyclo
- dupl
- goimports
- revive
- errcheck
- gocyclo
- gosec
- gosimple
- typecheck
- govet
- ineffassign
- misspell
- nakedret
- revive
- staticcheck
- unparam
- unused
linters-settings:
gofmt:
simplify: true
dupl:
threshold: 600
settings:
dupl:
threshold: 600
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- goimports
settings:
gofmt:
simplify: true
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$

94
vendor/github.com/Masterminds/semver/v3/constraints.go generated vendored
View File

@@ -21,21 +21,43 @@ type Constraints struct {
IncludePrerelease bool
}
// MaxConstraintLen is the maximum allowed length of a constraint string.
const MaxConstraintLen = 512
// MaxConstraintGroups is the maximum number of OR groups allowed in a
// constraint string.
const MaxConstraintGroups = 32
// ErrConstraintTooLong is returned when a constraint string exceeds the
// maximum allowed length.
var ErrConstraintTooLong = fmt.Errorf("constraint string is too long (max %d bytes)", MaxConstraintLen)
// ErrTooManyConstraintGroups is returned when a constraint string contains
// too many OR groups.
var ErrTooManyConstraintGroups = fmt.Errorf("too many constraint groups (max %d)", MaxConstraintGroups)
// NewConstraint returns a Constraints instance that a Version instance can
// be checked against. If there is a parse error it will be returned.
func NewConstraint(c string) (*Constraints, error) {
if len(c) > MaxConstraintLen {
return nil, ErrConstraintTooLong
}
// Rewrite - ranges into a comparison operation.
c = rewriteRange(c)
ors := strings.Split(c, "||")
if len(ors) > MaxConstraintGroups {
return nil, ErrTooManyConstraintGroups
}
lenors := len(ors)
or := make([][]*constraint, lenors)
hasPre := make([]bool, lenors)
for k, v := range ors {
// Validate the segment
if !validConstraintRegex.MatchString(v) {
return nil, fmt.Errorf("improper constraint: %s", v)
return nil, fmt.Errorf("improper constraint: %q", v)
}
cs := findConstraintRegex.FindAllString(v, -1)
@@ -104,9 +126,9 @@ func (cs Constraints) Validate(v *Version) (bool, []error) {
for _, c := range o {
// Before running the check handle the case there the version is
// a prerelease and the check is not searching for prereleases.
if !(cs.IncludePrerelease || cs.containsPre[i]) && v.pre != "" {
if !cs.IncludePrerelease && !cs.containsPre[i] && v.pre != "" {
if !prerelesase {
em := fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
em := fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
e = append(e, em)
prerelesase = true
}
@@ -258,7 +280,7 @@ func parseConstraint(c string) (*constraint, error) {
if len(c) > 0 {
m := constraintRegex.FindStringSubmatch(c)
if m == nil {
return nil, fmt.Errorf("improper constraint: %s", c)
return nil, fmt.Errorf("improper constraint: %q", c)
}
cs := &constraint{
@@ -325,7 +347,7 @@ func constraintNotEqual(v *Version, c *constraint, includePre bool) (bool, error
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
if c.dirty {
@@ -335,7 +357,7 @@ func constraintNotEqual(v *Version, c *constraint, includePre bool) (bool, error
if c.con.Minor() != v.Minor() && !c.minorDirty {
return true, nil
} else if c.minorDirty {
return false, fmt.Errorf("%s is equal to %s", v, c.orig)
return false, fmt.Errorf("%q is equal to %q", v, c.orig)
} else if c.con.Patch() != v.Patch() && !c.patchDirty {
return true, nil
} else if c.patchDirty {
@@ -345,15 +367,15 @@ func constraintNotEqual(v *Version, c *constraint, includePre bool) (bool, error
if eq {
return true, nil
}
return false, fmt.Errorf("%s is equal to %s", v, c.orig)
return false, fmt.Errorf("%q is equal to %q", v, c.orig)
}
return false, fmt.Errorf("%s is equal to %s", v, c.orig)
return false, fmt.Errorf("%q is equal to %q", v, c.orig)
}
}
eq := v.Equal(c.con)
if eq {
return false, fmt.Errorf("%s is equal to %s", v, c.orig)
return false, fmt.Errorf("%q is equal to %q", v, c.orig)
}
return true, nil
@@ -364,7 +386,7 @@ func constraintGreaterThan(v *Version, c *constraint, includePre bool) (bool, er
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
var eq bool
@@ -374,17 +396,17 @@ func constraintGreaterThan(v *Version, c *constraint, includePre bool) (bool, er
if eq {
return true, nil
}
return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
return false, fmt.Errorf("%q is less than or equal to %q", v, c.orig)
}
if v.Major() > c.con.Major() {
return true, nil
} else if v.Major() < c.con.Major() {
return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
return false, fmt.Errorf("%q is less than or equal to %q", v, c.orig)
} else if c.minorDirty {
// This is a range case such as >11. When the version is something like
// 11.1.0 is it not > 11. For that we would need 12 or higher
return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
return false, fmt.Errorf("%q is less than or equal to %q", v, c.orig)
} else if c.patchDirty {
// This is for ranges such as >11.1. A version of 11.1.1 is not greater
// which one of 11.2.1 is greater
@@ -392,7 +414,7 @@ func constraintGreaterThan(v *Version, c *constraint, includePre bool) (bool, er
if eq {
return true, nil
}
return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
return false, fmt.Errorf("%q is less than or equal to %q", v, c.orig)
}
// If we have gotten here we are not comparing pre-preleases and can use the
@@ -401,21 +423,21 @@ func constraintGreaterThan(v *Version, c *constraint, includePre bool) (bool, er
if eq {
return true, nil
}
return false, fmt.Errorf("%s is less than or equal to %s", v, c.orig)
return false, fmt.Errorf("%q is less than or equal to %q", v, c.orig)
}
func constraintLessThan(v *Version, c *constraint, includePre bool) (bool, error) {
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
eq := v.Compare(c.con) < 0
if eq {
return true, nil
}
return false, fmt.Errorf("%s is greater than or equal to %s", v, c.orig)
return false, fmt.Errorf("%q is greater than or equal to %q", v, c.orig)
}
func constraintGreaterThanEqual(v *Version, c *constraint, includePre bool) (bool, error) {
@@ -423,21 +445,21 @@ func constraintGreaterThanEqual(v *Version, c *constraint, includePre bool) (boo
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
eq := v.Compare(c.con) >= 0
if eq {
return true, nil
}
return false, fmt.Errorf("%s is less than %s", v, c.orig)
return false, fmt.Errorf("%q is less than %q", v, c.orig)
}
func constraintLessThanEqual(v *Version, c *constraint, includePre bool) (bool, error) {
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
var eq bool
@@ -447,13 +469,13 @@ func constraintLessThanEqual(v *Version, c *constraint, includePre bool) (bool,
if eq {
return true, nil
}
return false, fmt.Errorf("%s is greater than %s", v, c.orig)
return false, fmt.Errorf("%q is greater than %q", v, c.orig)
}
if v.Major() > c.con.Major() {
return false, fmt.Errorf("%s is greater than %s", v, c.orig)
return false, fmt.Errorf("%q is greater than %q", v, c.orig)
} else if v.Major() == c.con.Major() && v.Minor() > c.con.Minor() && !c.minorDirty {
return false, fmt.Errorf("%s is greater than %s", v, c.orig)
return false, fmt.Errorf("%q is greater than %q", v, c.orig)
}
return true, nil
@@ -469,11 +491,11 @@ func constraintTilde(v *Version, c *constraint, includePre bool) (bool, error) {
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
if v.LessThan(c.con) {
return false, fmt.Errorf("%s is less than %s", v, c.orig)
return false, fmt.Errorf("%q is less than %q", v, c.orig)
}
// ~0.0.0 is a special case where all constraints are accepted. It's
@@ -484,11 +506,11 @@ func constraintTilde(v *Version, c *constraint, includePre bool) (bool, error) {
}
if v.Major() != c.con.Major() {
return false, fmt.Errorf("%s does not have same major version as %s", v, c.orig)
return false, fmt.Errorf("%q does not have same major version as %q", v, c.orig)
}
if v.Minor() != c.con.Minor() && !c.minorDirty {
return false, fmt.Errorf("%s does not have same major and minor version as %s", v, c.orig)
return false, fmt.Errorf("%q does not have same major and minor version as %q", v, c.orig)
}
return true, nil
@@ -500,7 +522,7 @@ func constraintTildeOrEqual(v *Version, c *constraint, includePre bool) (bool, e
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
if c.dirty {
@@ -512,7 +534,7 @@ func constraintTildeOrEqual(v *Version, c *constraint, includePre bool) (bool, e
return true, nil
}
return false, fmt.Errorf("%s is not equal to %s", v, c.orig)
return false, fmt.Errorf("%q is not equal to %q", v, c.orig)
}
// ^* --> (any)
@@ -528,12 +550,12 @@ func constraintCaret(v *Version, c *constraint, includePre bool) (bool, error) {
// The existence of prereleases is checked at the group level and passed in.
// Exit early if the version has a prerelease but those are to be ignored.
if v.Prerelease() != "" && !includePre {
return false, fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v)
return false, fmt.Errorf("%q is a prerelease version and the constraint is only looking for release versions", v)
}
// This less than handles prereleases
if v.LessThan(c.con) {
return false, fmt.Errorf("%s is less than %s", v, c.orig)
return false, fmt.Errorf("%q is less than %q", v, c.orig)
}
var eq bool
@@ -548,12 +570,12 @@ func constraintCaret(v *Version, c *constraint, includePre bool) (bool, error) {
if eq {
return true, nil
}
return false, fmt.Errorf("%s does not have same major version as %s", v, c.orig)
return false, fmt.Errorf("%q does not have same major version as %q", v, c.orig)
}
// ^ when the major is 0 and minor > 0 is >=0.y.z < 0.y+1
if c.con.Major() == 0 && v.Major() > 0 {
return false, fmt.Errorf("%s does not have same major version as %s", v, c.orig)
return false, fmt.Errorf("%q does not have same major version as %q", v, c.orig)
}
// If the con Minor is > 0 it is not dirty
if c.con.Minor() > 0 || c.patchDirty {
@@ -561,11 +583,11 @@ func constraintCaret(v *Version, c *constraint, includePre bool) (bool, error) {
if eq {
return true, nil
}
return false, fmt.Errorf("%s does not have same minor version as %s. Expected minor versions to match when constraint major version is 0", v, c.orig)
return false, fmt.Errorf("%q does not have same minor version as %q. Expected minor versions to match when constraint major version is 0", v, c.orig)
}
// ^ when the minor is 0 and minor > 0 is =0.0.z
if c.con.Minor() == 0 && v.Minor() > 0 {
return false, fmt.Errorf("%s does not have same minor version as %s", v, c.orig)
return false, fmt.Errorf("%q does not have same minor version as %q", v, c.orig)
}
// At this point the major is 0 and the minor is 0 and not dirty. The patch
@@ -574,7 +596,7 @@ func constraintCaret(v *Version, c *constraint, includePre bool) (bool, error) {
if eq {
return true, nil
}
return false, fmt.Errorf("%s does not equal %s. Expect version and constraint to equal when major and minor versions are 0", v, c.orig)
return false, fmt.Errorf("%q does not equal %q. Expect version and constraint to equal when major and minor versions are 0", v, c.orig)
}
func isX(x string) bool {

39
vendor/github.com/Masterminds/semver/v3/version.go generated vendored
View File

@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"math"
"regexp"
"strconv"
"strings"
@@ -48,8 +49,16 @@ var (
// ErrInvalidPrerelease is returned when the pre-release is an invalid format
ErrInvalidPrerelease = errors.New("invalid prerelease string")
// ErrVersionTooLong is returned when a version string exceeds the
// maximum allowed length.
ErrVersionTooLong = fmt.Errorf("version string is too long (max %d bytes)", MaxVersionLen)
)
// MaxVersionLen is the maximum allowed length of a version string. This guards
// against unbounded input causing excessive memory allocations during parsing.
const MaxVersionLen = 256
// semVerRegex is the regular expression used to parse a semantic version.
// This is not the official regex from the semver spec. It has been modified to allow for loose handling
// where versions like 2.1 are detected.
@@ -94,6 +103,10 @@ func StrictNewVersion(v string) (*Version, error) {
return nil, ErrEmptyString
}
if len(v) > MaxVersionLen {
return nil, ErrVersionTooLong
}
// Split the parts into [0]major, [1]minor, and [2]patch,prerelease,build
parts := strings.SplitN(v, ".", 3)
if len(parts) != 3 {
@@ -161,6 +174,9 @@ func StrictNewVersion(v string) (*Version, error) {
// attempts to convert it to SemVer. If you want to validate it was a strict
// semantic version at parse time see StrictNewVersion().
func NewVersion(v string) (*Version, error) {
if len(v) > MaxVersionLen {
return nil, ErrVersionTooLong
}
if CoerceNewVersion {
return coerceNewVersion(v)
}
@@ -289,6 +305,8 @@ func coerceNewVersion(v string) (*Version, error) {
// New creates a new instance of Version with each of the parts passed in as
// arguments instead of parsing a version string.
// Note, New does not validate prerelease or metadata. Incorrect information can
// be passed in.
func New(major, minor, patch uint64, pre, metadata string) *Version {
v := Version{
major: major,
@@ -301,6 +319,7 @@ func New(major, minor, patch uint64, pre, metadata string) *Version {
v.original = v.String()
// TODO: In the next semver major version validate the pre and metadata. Return error if there is one.
return &v
}
@@ -388,6 +407,9 @@ func (v Version) IncPatch() Version {
} else {
vNext.metadata = ""
vNext.pre = ""
if v.patch == math.MaxUint64 {
panic("patch version increment would overflow uint64")
}
vNext.patch = v.patch + 1
}
vNext.original = v.originalVPrefix() + "" + vNext.String()
@@ -404,6 +426,9 @@ func (v Version) IncMinor() Version {
vNext.metadata = ""
vNext.pre = ""
vNext.patch = 0
if v.minor == math.MaxUint64 {
panic("minor version increment would overflow uint64")
}
vNext.minor = v.minor + 1
vNext.original = v.originalVPrefix() + "" + vNext.String()
return vNext
@@ -421,6 +446,9 @@ func (v Version) IncMajor() Version {
vNext.pre = ""
vNext.patch = 0
vNext.minor = 0
if v.major == math.MaxUint64 {
panic("major version increment would overflow uint64")
}
vNext.major = v.major + 1
vNext.original = v.originalVPrefix() + "" + vNext.String()
return vNext
@@ -568,7 +596,16 @@ func (v Version) MarshalText() ([]byte, error) {
// Scan implements the SQL.Scanner interface.
func (v *Version) Scan(value interface{}) error {
var s string
s, _ = value.(string)
switch t := value.(type) {
case string:
s = t
case []byte:
s = string(t)
case nil:
return fmt.Errorf("cannot scan nil into Version")
default:
return fmt.Errorf("unsupported Scan type %T", value)
}
temp, err := NewVersion(s)
if err != nil {
return err

12
vendor/github.com/clipperhouse/displaywidth/AGENTS.md generated vendored
View File

@@ -28,6 +28,18 @@ Retrieve and consider the comments on the PR, which may have come from GitHub Co
Offer to optionally post a brief summary of the review to the PR, via the gh CLI tool.
## Tagged Go releases
If I ask you whether we are ready to release, this means a tagged Go release on the main branch. Go releases are git tagged with a version number.
Review the changes since the last release, i.e. the previous git tag. Ensure that the changes are complete and correct. Identify new features, bug fixes, and performance improvements.
Identify breaking changes, especially API changes.
Ensure good test coverage. Look for performance changes, especially performance regressions, by running benchmarks against the previous release.
Ensure that the documentation in READMEs and GoDocs are complete, correct and consistent.
## Comparisons to go-runewidth
We originally attempted to make this package compatible with go-runewidth.

14
vendor/github.com/clipperhouse/displaywidth/CHANGELOG.md generated vendored
View File

@@ -1,5 +1,19 @@
# Changelog
## [0.11.0]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.10.0...v0.11.0)
### Added
- New `ControlSequences8Bit` option to treat 8-bit ECMA-48 (C1) escape sequences as zero-width. (#22)
### Changed
- Upgraded uax29 dependency to v2.7.0 for 8-bit escape sequence support in the grapheme iterator.
- Truncation now validates that preserved trailing escape sequences are zero-width, preventing edge cases where non-zero-width sequences could leak into output.
### Note
- `ControlSequences8Bit` is deliberately ignored by `TruncateString` and `TruncateBytes`, because C1 byte values (0x80–0x9F) overlap with UTF-8 multi-byte encoding.
## [0.10.0]
[Compare](https://github.com/clipperhouse/displaywidth/compare/v0.9.0...v0.10.0)

22
vendor/github.com/clipperhouse/displaywidth/README.md generated vendored
View File

@@ -79,6 +79,16 @@ when calculating the display width. When `false` (default), ANSI escape
sequences are treated as just a series of characters. When `true`, they are
treated as a single zero-width unit.
#### ControlSequences8Bit
`ControlSequences8Bit` specifies whether to ignore 8-bit ECMA-48 escape sequences
when calculating the display width. When `false` (default), these are treated
as just a series of characters. When `true`, they are treated as a single
zero-width unit.
Note: this option is ignored by the `Truncate` methods, as the concatenation
can lead to unintended UTF-8 semantics.
#### EastAsianWidth
`EastAsianWidth` defines how
@@ -103,12 +113,22 @@ and [regional indicator pairs](https://en.wikipedia.org/wiki/Regional_indicator_
for emojis. We are keeping an eye on
[emerging standards](https://www.jeffquast.com/post/state-of-terminal-emulation-2025/).
For control sequences, we implement the [ECMA-48](https://ecma-international.org/publications-and-standards/standards/ecma-48/) standard for 7-bit ASCII control sequences.
For control sequences, we implement the [ECMA-48](https://ecma-international.org/publications-and-standards/standards/ecma-48/) standard for 7-bit and 8-bit control sequences.
`clipperhouse/displaywidth`, `mattn/go-runewidth`, and `rivo/uniseg` will
give the same outputs for most real-world text. Extensive details are in the
[compatibility analysis](comparison/COMPATIBILITY_ANALYSIS.md).
## Invalid UTF-8
This package does not validate UTF-8. If you pass invalid UTF-8, the results
are undefined. We fuzz against invalid UTF-8 to ensure we don't panic or
loop indefinitely.
The `ControlSequences8Bit` option means that we will segment valid 8-bit
control sequences, which are typically _not_ valid UTF-8. 8-bit control bytes
happen to also be UTF-8 continuation bytes. Use with caution.
## Prior Art
[mattn/go-runewidth](https://github.com/mattn/go-runewidth)

2
vendor/github.com/clipperhouse/displaywidth/graphemes.go generated vendored
View File

@@ -45,6 +45,7 @@ func StringGraphemes(s string) Graphemes[string] {
func (options Options) StringGraphemes(s string) Graphemes[string] {
g := graphemes.FromString(s)
g.AnsiEscapeSequences = options.ControlSequences
g.AnsiEscapeSequences8Bit = options.ControlSequences8Bit
return Graphemes[string]{iter: g, options: options}
}
@@ -66,6 +67,7 @@ func BytesGraphemes(s []byte) Graphemes[[]byte] {
func (options Options) BytesGraphemes(s []byte) Graphemes[[]byte] {
g := graphemes.FromBytes(s)
g.AnsiEscapeSequences = options.ControlSequences
g.AnsiEscapeSequences8Bit = options.ControlSequences8Bit
return Graphemes[[]byte]{iter: g, options: options}
}

30
vendor/github.com/clipperhouse/displaywidth/options.go generated vendored Normal file
View File

@@ -0,0 +1,30 @@
package displaywidth
// Options allows you to specify the treatment of ambiguous East Asian
// characters and ANSI escape sequences.
type Options struct {
// EastAsianWidth specifies whether to treat ambiguous East Asian characters
// as width 1 or 2. When false (default), ambiguous East Asian characters
// are treated as width 1. When true, they are width 2.
EastAsianWidth bool
// ControlSequences specifies whether to ignore 7-bit ECMA-48 escape sequences
// when calculating the display width. When false (default), ANSI escape
// sequences are treated as just a series of characters. When true, they are
// treated as a single zero-width unit.
ControlSequences bool
// ControlSequences8Bit specifies whether to ignore 8-bit ECMA-48 escape sequences
// when calculating the display width. When false (default), these are treated
// as just a series of characters. When true, they are treated as a single
// zero-width unit.
ControlSequences8Bit bool
}
// DefaultOptions is the default options for the display width
// calculation, which is EastAsianWidth false, ControlSequences false, and
// ControlSequences8Bit false.
var DefaultOptions = Options{
EastAsianWidth: false,
ControlSequences: false,
ControlSequences8Bit: false,
}

149
vendor/github.com/clipperhouse/displaywidth/truncate.go generated vendored Normal file
View File

@@ -0,0 +1,149 @@
package displaywidth
import (
"strings"
"github.com/clipperhouse/uax29/v2/graphemes"
)
// TruncateString truncates a string to the given maxWidth, and appends the
// given tail if the string is truncated.
//
// It ensures the visible width, including the width of the tail, is less than or
// equal to maxWidth.
//
// When [Options.ControlSequences] is true, 7-bit ANSI escape sequences that
// appear after the truncation point are preserved in the output. This ensures
// that escape sequences such as SGR resets are not lost, preventing color
// bleed in terminal output.
//
// [Options.ControlSequences8Bit] is ignored by truncation. 8-bit C1 byte values
// (0x80-0x9F) overlap with UTF-8 multi-byte encoding, so manipulating them
// during truncation can shift byte boundaries and form unintended visible
// characters. Use [Options.String] or [Options.Bytes] for 8-bit-aware width
// measurement.
func (options Options) TruncateString(s string, maxWidth int, tail string) string {
// We deliberately ignore ControlSequences8Bit for truncation, see above.
options.ControlSequences8Bit = false
maxWidthWithoutTail := maxWidth - options.String(tail)
var pos, total int
g := graphemes.FromString(s)
g.AnsiEscapeSequences = options.ControlSequences
for g.Next() {
gw := graphemeWidth(g.Value(), options)
if total+gw <= maxWidthWithoutTail {
pos = g.End()
}
total += gw
if total > maxWidth {
if options.ControlSequences {
// Build result with trailing 7-bit ANSI escape sequences preserved
var b strings.Builder
b.Grow(len(s) + len(tail)) // at most original + tail
b.WriteString(s[:pos])
b.WriteString(tail)
rem := graphemes.FromString(s[pos:])
rem.AnsiEscapeSequences = options.ControlSequences
for rem.Next() {
v := rem.Value()
// Only preserve 7-bit escapes (ESC = 0x1B) that measure
// as zero-width on their own; some sequences (e.g. SOS)
// are only valid in their original context.
if len(v) > 0 && v[0] == 0x1B && options.String(v) == 0 {
b.WriteString(v)
}
}
return b.String()
}
return s[:pos] + tail
}
}
// No truncation
return s
}
// TruncateString truncates a string to the given maxWidth, and appends the
// given tail if the string is truncated.
//
// It ensures the total width, including the width of the tail, is less than or
// equal to maxWidth.
func TruncateString(s string, maxWidth int, tail string) string {
return DefaultOptions.TruncateString(s, maxWidth, tail)
}
// TruncateBytes truncates a []byte to the given maxWidth, and appends the
// given tail if the []byte is truncated.
//
// It ensures the visible width, including the width of the tail, is less than or
// equal to maxWidth.
//
// When [Options.ControlSequences] is true, 7-bit ANSI escape sequences that
// appear after the truncation point are preserved in the output. This ensures
// that escape sequences such as SGR resets are not lost, preventing color
// bleed in terminal output.
//
// [Options.ControlSequences8Bit] is ignored by truncation. 8-bit C1 byte values
// (0x80-0x9F) overlap with UTF-8 multi-byte encoding, so manipulating them
// during truncation can shift byte boundaries and form unintended visible
// characters. Use [Options.String] or [Options.Bytes] for 8-bit-aware width
// measurement.
func (options Options) TruncateBytes(s []byte, maxWidth int, tail []byte) []byte {
// We deliberately ignore ControlSequences8Bit for truncation, see above.
options.ControlSequences8Bit = false
maxWidthWithoutTail := maxWidth - options.Bytes(tail)
var pos, total int
g := graphemes.FromBytes(s)
g.AnsiEscapeSequences = options.ControlSequences
for g.Next() {
gw := graphemeWidth(g.Value(), options)
if total+gw <= maxWidthWithoutTail {
pos = g.End()
}
total += gw
if total > maxWidth {
if options.ControlSequences {
// Build result with trailing 7-bit ANSI escape sequences preserved
result := make([]byte, 0, len(s)+len(tail)) // at most original + tail
result = append(result, s[:pos]...)
result = append(result, tail...)
rem := graphemes.FromBytes(s[pos:])
rem.AnsiEscapeSequences = options.ControlSequences
for rem.Next() {
v := rem.Value()
// Only preserve 7-bit escapes (ESC = 0x1B) that measure
// as zero-width on their own; some sequences (e.g. SOS)
// are only valid in their original context.
if len(v) > 0 && v[0] == 0x1B && options.Bytes(v) == 0 {
result = append(result, v...)
}
}
return result
}
result := make([]byte, 0, pos+len(tail))
result = append(result, s[:pos]...)
result = append(result, tail...)
return result
}
}
// No truncation
return s
}
// TruncateBytes truncates a []byte to the given maxWidth, and appends the
// given tail if the []byte is truncated.
//
// It ensures the total width, including the width of the tail, is less than or
// equal to maxWidth.
func TruncateBytes(s []byte, maxWidth int, tail []byte) []byte {
return DefaultOptions.TruncateBytes(s, maxWidth, tail)
}

156
vendor/github.com/clipperhouse/displaywidth/width.go generated vendored
View File

@@ -1,35 +1,11 @@
package displaywidth
import (
"strings"
"unicode/utf8"
"github.com/clipperhouse/uax29/v2/graphemes"
)
// Options allows you to specify the treatment of ambiguous East Asian
// characters and ANSI escape sequences.
type Options struct {
// EastAsianWidth specifies whether to treat ambiguous East Asian characters
// as width 1 or 2. When false (default), ambiguous East Asian characters
// are treated as width 1. When true, they are width 2.
EastAsianWidth bool
// ControlSequences specifies whether to ignore ECMA-48 escape sequences
// when calculating the display width. When false (default), ANSI escape
// sequences are treated as just a series of characters. When true, they are
// treated as a single zero-width unit.
//
// Note that this option is about *sequences*. Individual control characters
// are already treated as zero-width. With this option, ANSI sequences such as
// "\x1b[31m" and "\x1b[0m" do not count towards the width of a string.
ControlSequences bool
}
// DefaultOptions is the default options for the display width
// calculation, which is EastAsianWidth false and ControlSequences false.
var DefaultOptions = Options{EastAsianWidth: false, ControlSequences: false}
// String calculates the display width of a string,
// by iterating over grapheme clusters in the string
// and summing their widths.
@@ -55,6 +31,7 @@ func (options Options) String(s string) int {
// Not ASCII, use grapheme parsing
g := graphemes.FromString(s[pos:])
g.AnsiEscapeSequences = options.ControlSequences
g.AnsiEscapeSequences8Bit = options.ControlSequences8Bit
start := pos
@@ -105,6 +82,7 @@ func (options Options) Bytes(s []byte) int {
// Not ASCII, use grapheme parsing
g := graphemes.FromBytes(s[pos:])
g.AnsiEscapeSequences = options.ControlSequences
g.AnsiEscapeSequences8Bit = options.ControlSequences8Bit
start := pos
@@ -166,128 +144,22 @@ func (options Options) Rune(r rune) int {
const _Default property = 0
// TruncateString truncates a string to the given maxWidth, and appends the
// given tail if the string is truncated.
//
// It ensures the visible width, including the width of the tail, is less than or
// equal to maxWidth.
//
// When [Options.ControlSequences] is true, ANSI escape sequences that appear
// after the truncation point are preserved in the output. This ensures that
// escape sequences such as SGR resets are not lost, preventing color bleed
// in terminal output.
func (options Options) TruncateString(s string, maxWidth int, tail string) string {
maxWidthWithoutTail := maxWidth - options.String(tail)
var pos, total int
g := graphemes.FromString(s)
g.AnsiEscapeSequences = options.ControlSequences
for g.Next() {
gw := graphemeWidth(g.Value(), options)
if total+gw <= maxWidthWithoutTail {
pos = g.End()
}
total += gw
if total > maxWidth {
if options.ControlSequences {
// Build result with trailing ANSI escape sequences preserved
var b strings.Builder
b.Grow(len(s) + len(tail)) // at most original + tail
b.WriteString(s[:pos])
b.WriteString(tail)
rem := graphemes.FromString(s[pos:])
rem.AnsiEscapeSequences = true
for rem.Next() {
v := rem.Value()
if len(v) > 0 && v[0] == 0x1B {
b.WriteString(v)
}
}
return b.String()
}
return s[:pos] + tail
}
}
// No truncation
return s
}
// TruncateString truncates a string to the given maxWidth, and appends the
// given tail if the string is truncated.
//
// It ensures the total width, including the width of the tail, is less than or
// equal to maxWidth.
func TruncateString(s string, maxWidth int, tail string) string {
return DefaultOptions.TruncateString(s, maxWidth, tail)
}
// TruncateBytes truncates a []byte to the given maxWidth, and appends the
// given tail if the []byte is truncated.
//
// It ensures the visible width, including the width of the tail, is less than or
// equal to maxWidth.
//
// When [Options.ControlSequences] is true, ANSI escape sequences that appear
// after the truncation point are preserved in the output. This ensures that
// escape sequences such as SGR resets are not lost, preventing color bleed
// in terminal output.
func (options Options) TruncateBytes(s []byte, maxWidth int, tail []byte) []byte {
maxWidthWithoutTail := maxWidth - options.Bytes(tail)
var pos, total int
g := graphemes.FromBytes(s)
g.AnsiEscapeSequences = options.ControlSequences
for g.Next() {
gw := graphemeWidth(g.Value(), options)
if total+gw <= maxWidthWithoutTail {
pos = g.End()
}
total += gw
if total > maxWidth {
if options.ControlSequences {
// Build result with trailing ANSI escape sequences preserved
result := make([]byte, 0, len(s)+len(tail)) // at most original + tail
result = append(result, s[:pos]...)
result = append(result, tail...)
rem := graphemes.FromBytes(s[pos:])
rem.AnsiEscapeSequences = true
for rem.Next() {
v := rem.Value()
if len(v) > 0 && v[0] == 0x1B {
result = append(result, v...)
}
}
return result
}
result := make([]byte, 0, pos+len(tail))
result = append(result, s[:pos]...)
result = append(result, tail...)
return result
}
}
// No truncation
return s
}
// TruncateBytes truncates a []byte to the given maxWidth, and appends the
// given tail if the []byte is truncated.
//
// It ensures the total width, including the width of the tail, is less than or
// equal to maxWidth.
func TruncateBytes(s []byte, maxWidth int, tail []byte) []byte {
return DefaultOptions.TruncateBytes(s, maxWidth, tail)
}
// graphemeWidth returns the display width of a grapheme cluster.
// The passed string must be a single grapheme cluster.
func graphemeWidth[T ~string | []byte](s T, options Options) int {
// Optimization: no need to look up properties
switch len(s) {
case 0:
if len(s) == 0 {
return 0
case 1:
}
// C1 controls (0x80-0x9F) are zero-width when 8-bit control sequences
// are enabled. This must be checked before the single-byte optimization
// below, which would otherwise return width 1 for these bytes.
if options.ControlSequences8Bit && s[0] >= 0x80 && s[0] <= 0x9F {
return 0
}
// Optimization: single-byte graphemes need no property lookup
if len(s) == 1 {
return asciiWidth(s[0])
}

60
vendor/github.com/clipperhouse/uax29/v2/graphemes/README.md generated vendored
View File

@@ -7,18 +7,17 @@ An implementation of grapheme cluster boundaries from [Unicode text segmentation
## Quick start
```
go get "github.com/clipperhouse/uax29/v2/graphemes"
go get github.com/clipperhouse/uax29/v2/graphemes
```
```go
import "github.com/clipperhouse/uax29/v2/graphemes"
text := "Hello, 世界. Nice dog! 👍🐶"
g := graphemes.FromString(text)
tokens := graphemes.FromString(text)
for tokens.Next() { // Next() returns true until end of data
fmt.Println(tokens.Value()) // Do something with the current grapheme
for g.Next() { // Next() returns true until end of data
fmt.Println(g.Value()) // Do something with the current grapheme
}
```
@@ -37,11 +36,10 @@ We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-36.html#Te
```go
text := "Hello, 世界. Nice dog! 👍🐶"
g := graphemes.FromString(text)
tokens := graphemes.FromString(text)
for tokens.Next() { // Next() returns true until end of data
fmt.Println(tokens.Value()) // Do something with the current grapheme
for g.Next() { // Next() returns true until end of data
fmt.Println(g.Value()) // Do something with the current grapheme
}
```
@@ -50,15 +48,15 @@ for tokens.Next() { // Next() returns true until end of data
`FromReader` embeds a [`bufio.Scanner`](https://pkg.go.dev/bufio#Scanner), so just use those methods.
```go
r := getYourReader() // from a file or network maybe
tokens := graphemes.FromReader(r)
r := getYourReader() // from a file or network maybe
g := graphemes.FromReader(r)
for tokens.Scan() { // Scan() returns true until error or EOF
fmt.Println(tokens.Text()) // Do something with the current grapheme
for g.Scan() { // Scan() returns true until error or EOF
fmt.Println(g.Text()) // Do something with the current grapheme
}
if tokens.Err() != nil { // Check the error
log.Fatal(tokens.Err())
if g.Err() != nil { // Check the error
log.Fatal(g.Err())
}
```
@@ -67,13 +65,39 @@ if tokens.Err() != nil { // Check the error
```go
b := []byte("Hello, 世界. Nice dog! 👍🐶")
tokens := graphemes.FromBytes(b)
g := graphemes.FromBytes(b)
for tokens.Next() { // Next() returns true until end of data
fmt.Println(tokens.Value()) // Do something with the current grapheme
for g.Next() { // Next() returns true until end of data
fmt.Println(g.Value()) // Do something with the current grapheme
}
```
### ANSI escape sequences
By the UAX 29 specification, ANSI escape sequences are not grapheme clusters. To treat 7-bit ANSI escape sequences as a single cluster, set `AnsiEscapeSequences` to true.
```go
text := "Hello, \x1b[31mworld\x1b[0m!"
g := graphemes.FromString(text)
g.AnsiEscapeSequences = true
for g.Next() {
fmt.Println(g.Value())
}
```
To also parse 8-bit C1 controls (non-UTF-8 bytes), set `AnsiEscapeSequences8Bit` to true.
```go
g.AnsiEscapeSequences = true // 7-bit forms (ESC ...)
g.AnsiEscapeSequences8Bit = true // 8-bit C1 forms (0x80-0x9F), not valid UTF-8
```
For ESC-initiated (7-bit) control strings, only 7-bit terminators are recognized.
For C1-initiated (8-bit) control strings, only C1 ST (`0x9C`) is recognized as ST.
We implement [ECMA-48](https://ecma-international.org/publications-and-standards/standards/ecma-48/) control codes in both 7-bit and 8-bit representations. 8-bit control codes are not UTF-8 encoded and are not valid UTF-8, caveat emptor.
### Benchmarks
```

77
vendor/github.com/clipperhouse/uax29/v2/graphemes/ansi.go generated vendored
View File

@@ -1,46 +1,43 @@
package graphemes
// ansiEscapeLength returns the byte length of a valid ANSI escape sequence at the
// start of data, or 0 if none. Input is UTF-8; only 7-bit ESC sequences are
// recognized (C1 0x80–0x9F can be UTF-8 continuation bytes).
// ansiEscapeLength returns the byte length of a valid 7-bit ANSI escape
// sequence at the start of data, or 0 if none.
//
// Recognized forms (ECMA-48 / ISO 6429):
// - CSI: ESC [ then parameter bytes (0x30–0x3F), intermediate (0x20–0x2F), final (0x40–0x7E)
// - OSC: ESC ] then payload until ST (ESC \) or BEL (0x07)
// - DCS, SOS, PM, APC: ESC P / X / ^ / _ then payload until ST (ESC \)
// - Two-byte: ESC + Fe (0x40–0x5F excluding above), or Fp (0x30–0x3F), or nF (0x20–0x2F then final)
// - CSI: ESC [ then parameter bytes (0x30-0x3F), intermediate (0x20-0x2F), final (0x40-0x7E)
// - OSC: ESC ] then payload until BEL (0x07), 7-bit ST (ESC \), CAN (0x18), or SUB (0x1A)
// - DCS, SOS, PM, APC: ESC P/X/^/_ then payload until 7-bit ST (ESC \), CAN, or SUB
// - Two-byte: ESC + Fe/Fs (0x40-0x7E excluding above), or Fp (0x30-0x3F), or nF (0x20-0x2F then final)
func ansiEscapeLength[T ~string | ~[]byte](data T) int {
n := len(data)
if n < 2 {
return 0
}
if data[0] != esc {
if n < 2 || data[0] != esc {
return 0
}
b1 := data[1]
switch b1 {
case '[': // CSI
body := csiLength(data[2:])
body := csiBodyLength(data[2:])
if body == 0 {
return 0
}
return 2 + body
case ']': // OSC – allows BEL or ST as terminator
case ']': // OSC - allows BEL or 7-bit ST terminator
body := oscLength(data[2:])
if body == 0 {
if body < 0 {
return 0
}
return 2 + body
case 'P', 'X', '^', '_': // DCS, SOS, PM, APC – require ST (ESC \) only
case 'P', 'X', '^', '_': // DCS, SOS, PM, APC
body := stSequenceLength(data[2:])
if body == 0 {
if body < 0 {
return 0
}
return 2 + body
}
if b1 >= 0x40 && b1 <= 0x5F {
// Fe (C1) two-byte; [ ] P X ^ _ handled above
if b1 >= 0x40 && b1 <= 0x7E {
// Fe/Fs two-byte; [ ] P X ^ _ handled above
return 2
}
if b1 >= 0x30 && b1 <= 0x3F {
@@ -48,7 +45,7 @@ func ansiEscapeLength[T ~string | ~[]byte](data T) int {
return 2
}
if b1 >= 0x20 && b1 <= 0x2F {
// nF: intermediates then one final (0x30–0x7E)
// nF: intermediates then one final (0x30-0x7E)
i := 2
for i < n && data[i] >= 0x20 && data[i] <= 0x2F {
i++
@@ -58,17 +55,18 @@ func ansiEscapeLength[T ~string | ~[]byte](data T) int {
}
return 0
}
return 0
}
// csiLength returns the length of the CSI body (param/intermediate/final bytes).
// csiBodyLength returns the length of the CSI body (param/intermediate/final bytes).
// data is the slice after "ESC [".
// Per ECMA-48, the CSI body has the form:
//
// parameters (0x30–0x3F)*, intermediates (0x20–0x2F)*, final (0x40–0x7E)
//
// Once an intermediate byte is seen, subsequent parameter bytes are invalid.
func csiLength[T ~string | ~[]byte](data T) int {
func csiBodyLength[T ~string | ~[]byte](data T) int {
seenIntermediate := false
for i := 0; i < len(data); i++ {
b := data[i]
@@ -90,30 +88,51 @@ func csiLength[T ~string | ~[]byte](data T) int {
return 0
}
// oscLength returns the length of the OSC body up to and including
// the terminator. OSC accepts either BEL (0x07) or ST (ESC \) per
// widespread terminal convention. data is the slice after "ESC ]".
// oscLength returns the length of the OSC body.
// data is the slice after "ESC ]".
//
// Returns:
// - n >= 0: consumed body length (includes BEL/ST terminator when present)
// - -1: not terminated in the provided data
//
// OSC accepts BEL (0x07) or 7-bit ST (ESC \) as terminators by widespread convention.
// Per ECMA-48, CAN (0x18) and SUB (0x1A) cancel the control string; in that
// case they are not part of the OSC sequence length.
func oscLength[T ~string | ~[]byte](data T) int {
for i := 0; i < len(data); i++ {
b := data[i]
if b == bel {
return i + 1
}
if b == can || b == sub {
return i
}
if b == esc && i+1 < len(data) && data[i+1] == '\\' {
return i + 2
}
}
return 0
return -1
}
// stSequenceLength returns the length of a control-string body up to and
// including the ST (ESC \) terminator. Used for DCS, SOS, PM, and APC, which
// per ECMA-48 require ST and do not accept BEL. data is the slice after "ESC x".
// stSequenceLength returns the length of a control-string body.
// data is the slice after "ESC x".
//
// Returns:
// - n >= 0: consumed body length (includes ST terminator when present)
// - -1: not terminated in the provided data
//
// Used for DCS, SOS, PM, and APC, which per ECMA-48 terminate with ST.
// ST here is the 7-bit form (ESC \).
// CAN (0x18) and SUB (0x1A) cancel the control string; in that case they are
// not part of the sequence length.
func stSequenceLength[T ~string | ~[]byte](data T) int {
for i := 0; i < len(data); i++ {
if data[i] == can || data[i] == sub {
return i
}
if data[i] == esc && i+1 < len(data) && data[i+1] == '\\' {
return i + 2
}
}
return 0
return -1
}

79
vendor/github.com/clipperhouse/uax29/v2/graphemes/ansi8.go generated vendored Normal file
View File

@@ -0,0 +1,79 @@
package graphemes
// ansiEscapeLength8Bit returns the byte length of a valid 8-bit C1 ANSI
// sequence at the start of data, or 0 if none.
//
// Recognized forms (ECMA-48 / ISO 6429):
// - C1 CSI (0x9B) body as parameter/intermediate/final bytes
// - C1 OSC (0x9D) body terminated by BEL, C1 ST, CAN, or SUB
// - C1 DCS/SOS/PM/APC (0x90/0x98/0x9E/0x9F) body terminated by C1 ST, CAN, or SUB
// - Standalone C1 controls (0x80..0x9F not listed above): single byte
func ansiEscapeLength8Bit[T ~string | ~[]byte](data T) int {
if len(data) == 0 {
return 0
}
switch data[0] {
case 0x9B: // C1 CSI
body := csiBodyLength(data[1:])
if body == 0 {
return 0
}
return 1 + body
case 0x9D: // C1 OSC
body := oscLengthC1(data[1:])
if body < 0 {
return 0
}
return 1 + body
case 0x90, 0x98, 0x9E, 0x9F: // C1 DCS, SOS, PM, APC
body := stSequenceLengthC1(data[1:])
if body < 0 {
return 0
}
return 1 + body
default:
if data[0] >= 0x80 && data[0] <= 0x9F {
return 1
}
}
return 0
}
// oscLengthC1 returns the length of a C1 OSC body.
// data is the slice after the C1 OSC initiator (0x9D).
//
// Returns:
// - n >= 0: consumed body length (includes BEL/ST terminator when present)
// - -1: not terminated in the provided data
//
// Terminators: BEL (0x07) or C1 ST (0x9C).
// CAN (0x18) and SUB (0x1A) cancel the control string.
func oscLengthC1[T ~string | ~[]byte](data T) int {
for i := 0; i < len(data); i++ {
b := data[i]
if b == bel || b == st {
return i + 1
}
if b == can || b == sub {
return i
}
}
return -1
}
// stSequenceLengthC1 parses DCS/SOS/PM/APC bodies that terminate with C1 ST
// (0x9C), or are canceled by CAN/SUB.
func stSequenceLengthC1[T ~string | ~[]byte](data T) int {
for i := 0; i < len(data); i++ {
b := data[i]
if b == can || b == sub {
return i
}
if b == st {
return i + 1
}
}
return -1
}

29
vendor/github.com/clipperhouse/uax29/v2/graphemes/iterator.go generated vendored
View File

@@ -27,9 +27,18 @@ type Iterator[T ~string | ~[]byte] struct {
data T
pos int
start int
// AnsiEscapeSequences treats ANSI escape sequences (ECMA-48) as single grapheme
// clusters when true. Default is false.
// AnsiEscapeSequences treats 7-bit ANSI escape sequences (ECMA-48) as
// single grapheme clusters when true. The default is false.
//
// 8-bit controls are not enabled by this option. See [AnsiEscapeSequences8Bit].
AnsiEscapeSequences bool
// AnsiEscapeSequences8Bit treats 8-bit C1 ANSI escape sequences (ECMA-48) as single
// grapheme clusters when true. The default is false.
//
// 8-bit control bytes are not UTF-8 encoded, i.e. not valid UTF-8. If you
// choose this option, you are choosing to interpret non-UTF-8 data, caveat
// emptor.
AnsiEscapeSequences8Bit bool
}
var (
@@ -41,6 +50,9 @@ const (
esc = 0x1B
cr = 0x0D
bel = 0x07
can = 0x18
sub = 0x1A
st = 0x9C
)
// Next advances the iterator to the next grapheme cluster.
@@ -51,16 +63,21 @@ func (iter *Iterator[T]) Next() bool {
}
iter.start = iter.pos
if iter.AnsiEscapeSequences && iter.data[iter.pos] == esc {
b := iter.data[iter.pos]
if iter.AnsiEscapeSequences && b == esc {
if a := ansiEscapeLength(iter.data[iter.pos:]); a > 0 {
iter.pos += a
return true
}
}
if iter.AnsiEscapeSequences8Bit && b >= 0x80 && b <= 0x9F {
if a := ansiEscapeLength8Bit(iter.data[iter.pos:]); a > 0 {
iter.pos += a
return true
}
}
// ASCII hot path: any ASCII is one grapheme when next byte is ASCII or end.
// Fall through on CR so splitfunc can handle CR+LF as a single cluster.
b := iter.data[iter.pos]
if b < utf8.RuneSelf && b != cr {
if iter.pos+1 >= len(iter.data) || iter.data[iter.pos+1] < utf8.RuneSelf {
iter.pos++
@@ -68,7 +85,7 @@ func (iter *Iterator[T]) Next() bool {
}
}
// Fall back to actual grapheme parsing
// Fall back to UAX29 grapheme parsing
remaining := iter.data[iter.pos:]
advance, _, err := iter.split(remaining, true)
if err != nil {

6
vendor/github.com/dlclark/regexp2/.travis.yml generated vendored
View File

@@ -1,5 +1,7 @@
language: go
arch:
- AMD64
- ppc64le
go:
- 1.9
- tip
- tip

95
vendor/github.com/dlclark/regexp2/README.md generated vendored
View File

@@ -4,10 +4,17 @@ Regexp2 is a feature-rich RegExp engine for Go. It doesn't have constant time g
## Basis of the engine
The engine is ported from the .NET framework's System.Text.RegularExpressions.Regex engine. That engine was open sourced in 2015 under the MIT license. There are some fundamental differences between .NET strings and Go strings that required a bit of borrowing from the Go framework regex engine as well. I cleaned up a couple of the dirtier bits during the port (regexcharclass.cs was terrible), but the parse tree, code emmitted, and therefore patterns matched should be identical.
## New Code Generation
For extra performance use `regexp2` with [`regexp2cg`](https://github.com/dlclark/regexp2cg). It is a code generation utility for `regexp2` and you can likely improve your regexp runtime performance by 3-10x in hot code paths. As always you should benchmark your specifics to confirm the results. Give it a try!
## Installing
This is a go-gettable library, so install is easy:
go get github.com/dlclark/regexp2/...
go get github.com/dlclark/regexp2
To use the new Code Generation (while it's in beta) you'll need to use the `code_gen` branch:
go get github.com/dlclark/regexp2@code_gen
## Usage
Usage is similar to the Go `regexp` package. Just like in `regexp`, you start by converting a regex into a state machine via the `Compile` or `MustCompile` methods. They ultimately do the same thing, but `MustCompile` will panic if the regex is invalid. You can then use the provided `Regexp` struct to find matches repeatedly. A `Regexp` struct is safe to use across goroutines.
@@ -39,12 +46,30 @@ Group 0 is embedded in the Match. Group 0 is an automatically-assigned group th
The __last__ capture is embedded in each group, so `g.String()` will return the same thing as `g.Capture.String()` and `g.Captures[len(g.Captures)-1].String()`.
If you want to find multiple matches from a single input string you should use the `FindNextMatch` method. For example, to implement a function similar to `regexp.FindAllString`:
```go
func regexp2FindAllString(re *regexp2.Regexp, s string) []string {
var matches []string
m, _ := re.FindStringMatch(s)
for m != nil {
matches = append(matches, m.String())
m, _ = re.FindNextMatch(m)
}
return matches
}
```
`FindNextMatch` is optmized so that it re-uses the underlying string/rune slice.
The internals of `regexp2` always operate on `[]rune` so `Index` and `Length` data in a `Match` always reference a position in `rune`s rather than `byte`s (even if the input was given as a string). This is a dramatic difference between `regexp` and `regexp2`. It's advisable to use the provided `String()` methods to avoid having to work with indices.
## Compare `regexp` and `regexp2`
| Category | regexp | regexp2 |
| --- | --- | --- |
| Catastrophic backtracking possible | no, constant execution time guarantees | yes, if your pattern is at risk you can use the `re.MatchTimeout` field |
| Python-style capture groups `(?P<name>re)` | yes | no (yes in RE2 compat mode) |
| .NET-style capture groups `(?<name>re)` or `(?'name're)` | no | yes |
| .NET-style capture groups `(?<name>re)` or `(?'name're)` | yes | yes |
| comments `(?#comment)` | no | yes |
| branch numbering reset `(?\|a\|b)` | no | no |
| possessive match `(?>re)` | no | yes |
@@ -62,6 +87,8 @@ The default behavior of `regexp2` is to match the .NET regexp engine, however th
* add support for named ascii character classes (e.g. `[[:foo:]]`)
* add support for python-style capture groups (e.g. `(P<name>re)`)
* change singleline behavior for `$` to only match end of string (like RE2) (see [#24](https://github.com/dlclark/regexp2/issues/24))
* change the character classes `\d` `\s` and `\w` to match the same characters as RE2. NOTE: if you also use the `ECMAScript` option then this will change the `\s` character class to match ECMAScript instead of RE2. ECMAScript allows more whitespace characters in `\s` than RE2 (but still fewer than the the default behavior).
* allow character escape sequences to have defaults. For example, by default `\_` isn't a known character escape and will fail to compile, but in RE2 mode it will match the literal character `_`
```go
re := regexp2.MustCompile(`Your RE2-compatible pattern`, regexp2.RE2)
@@ -72,6 +99,70 @@ if isMatch, _ := re.MatchString(`Something to match`); isMatch {
This feature is a work in progress and I'm open to ideas for more things to put here (maybe more relaxed character escaping rules?).
## Catastrophic Backtracking and Timeouts
`regexp2` supports features that can lead to catastrophic backtracking.
`Regexp.MatchTimeout` can be set to to limit the impact of such behavior; the
match will fail with an error after approximately MatchTimeout. No timeout
checks are done by default.
Timeout checking is not free. The current timeout checking implementation starts
a background worker that updates a clock value approximately once every 100
milliseconds. The matching code compares this value against the precomputed
deadline for the match. The performance impact is as follows.
1. A match with a timeout runs almost as fast as a match without a timeout.
2. If any live matches have a timeout, there will be a background CPU load
(`~0.15%` currently on a modern machine). This load will remain constant
regardless of the number of matches done including matches done in parallel.
3. If no live matches are using a timeout, the background load will remain
until the longest deadline (match timeout + the time when the match started)
is reached. E.g., if you set a timeout of one minute the load will persist
for approximately a minute even if the match finishes quickly.
See [PR #58](https://github.com/dlclark/regexp2/pull/58) for more details and
alternatives considered.
## Goroutine leak error
If you're using a library during unit tests (e.g. https://github.com/uber-go/goleak) that validates all goroutines are exited then you'll likely get an error if you or any of your dependencies use regex's with a MatchTimeout.
To remedy the problem you'll need to tell the unit test to wait until the backgroup timeout goroutine is exited.
```go
func TestSomething(t *testing.T) {
defer goleak.VerifyNone(t)
defer regexp2.StopTimeoutClock()
// ... test
}
//or
func TestMain(m *testing.M) {
// setup
// ...
// run
m.Run()
//tear down
regexp2.StopTimeoutClock()
goleak.VerifyNone(t)
}
```
This will add ~100ms runtime to each test (or TestMain). If that's too much time you can set the clock cycle rate of the timeout goroutine in an init function in a test file. `regexp2.SetTimeoutCheckPeriod` isn't threadsafe so it must be setup before starting any regex's with Timeouts.
```go
func init() {
//speed up testing by making the timeout clock 1ms
regexp2.SetTimeoutCheckPeriod(time.Millisecond)
}
```
## ECMAScript compatibility mode
In this mode the engine provides compatibility with the [regex engine](https://tc39.es/ecma262/multipage/text-processing.html#sec-regexp-regular-expression-objects) described in the ECMAScript specification.
Additionally a Unicode mode is provided which allows parsing of `\u{CodePoint}` syntax that is only when both are provided.
## Library features that I'm still working on
- Regex split

141
vendor/github.com/dlclark/regexp2/fastclock.go generated vendored Normal file
View File

@@ -0,0 +1,141 @@
package regexp2
import (
"sync"
"sync/atomic"
"time"
)
// fasttime holds a time value (ticks since clock initialization)
type fasttime int64
// fastclock provides a fast clock implementation.
//
// A background goroutine periodically stores the current time
// into an atomic variable.
//
// A deadline can be quickly checked for expiration by comparing
// its value to the clock stored in the atomic variable.
//
// The goroutine automatically stops once clockEnd is reached.
// (clockEnd covers the largest deadline seen so far + some
// extra time). This ensures that if regexp2 with timeouts
// stops being used we will stop background work.
type fastclock struct {
// instances of atomicTime must be at the start of the struct (or at least 64-bit aligned)
// otherwise 32-bit architectures will panic
current atomicTime // Current time (approximate)
clockEnd atomicTime // When clock updater is supposed to stop (>= any existing deadline)
// current and clockEnd can be read via atomic loads.
// Reads and writes of other fields require mu to be held.
mu sync.Mutex
start time.Time // Time corresponding to fasttime(0)
running bool // Is a clock updater running?
}
var fast fastclock
// reached returns true if current time is at or past t.
func (t fasttime) reached() bool {
return fast.current.read() >= t
}
// makeDeadline returns a time that is approximately time.Now().Add(d)
func makeDeadline(d time.Duration) fasttime {
// Increase the deadline since the clock we are reading may be
// just about to tick forwards.
end := fast.current.read() + durationToTicks(d+clockPeriod)
// Start or extend clock if necessary.
if end > fast.clockEnd.read() {
// If time.Since(last use) > timeout, there's a chance that
// fast.current will no longer be updated, which can lead to
// incorrect 'end' calculations that can trigger a false timeout
fast.mu.Lock()
if !fast.running && !fast.start.IsZero() {
// update fast.current
fast.current.write(durationToTicks(time.Since(fast.start)))
// recalculate our end value
end = fast.current.read() + durationToTicks(d+clockPeriod)
}
fast.mu.Unlock()
extendClock(end)
}
return end
}
// extendClock ensures that clock is live and will run until at least end.
func extendClock(end fasttime) {
fast.mu.Lock()
defer fast.mu.Unlock()
if fast.start.IsZero() {
fast.start = time.Now()
}
// Extend the running time to cover end as well as a bit of slop.
if shutdown := end + durationToTicks(time.Second); shutdown > fast.clockEnd.read() {
fast.clockEnd.write(shutdown)
}
// Start clock if necessary
if !fast.running {
fast.running = true
go runClock()
}
}
// stop the timeout clock in the background
// should only used for unit tests to abandon the background goroutine
func stopClock() {
fast.mu.Lock()
if fast.running {
fast.clockEnd.write(fasttime(0))
}
fast.mu.Unlock()
// pause until not running
// get and release the lock
isRunning := true
for isRunning {
time.Sleep(clockPeriod / 2)
fast.mu.Lock()
isRunning = fast.running
fast.mu.Unlock()
}
}
func durationToTicks(d time.Duration) fasttime {
// Downscale nanoseconds to approximately a millisecond so that we can avoid
// overflow even if the caller passes in math.MaxInt64.
return fasttime(d) >> 20
}
const DefaultClockPeriod = 100 * time.Millisecond
// clockPeriod is the approximate interval between updates of approximateClock.
var clockPeriod = DefaultClockPeriod
func runClock() {
fast.mu.Lock()
defer fast.mu.Unlock()
for fast.current.read() <= fast.clockEnd.read() {
// Unlock while sleeping.
fast.mu.Unlock()
time.Sleep(clockPeriod)
fast.mu.Lock()
newTime := durationToTicks(time.Since(fast.start))
fast.current.write(newTime)
}
fast.running = false
}
type atomicTime struct{ v int64 } // Should change to atomic.Int64 when we can use go 1.19
func (t *atomicTime) read() fasttime { return fasttime(atomic.LoadInt64(&t.v)) }
func (t *atomicTime) write(v fasttime) { atomic.StoreInt64(&t.v, int64(v)) }

14
vendor/github.com/dlclark/regexp2/match.go generated vendored
View File

@@ -6,8 +6,9 @@ import (
)
// Match is a single regex result match that contains groups and repeated captures
// -Groups
// -Capture
//
// -Groups
// -Capture
type Match struct {
Group //embeded group 0
@@ -43,10 +44,10 @@ type Group struct {
type Capture struct {
// the original string
text []rune
// the position in the original string where the first character of
// captured substring was found.
// Index is the position in the underlying rune slice where the first character of
// captured substring was found. Even if you pass in a string this will be in Runes.
Index int
// the length of the captured substring.
// Length is the number of runes in the captured substring.
Length int
}
@@ -187,7 +188,8 @@ func (m *Match) addMatch(c, start, l int) {
}
// Nonpublic builder: Add a capture to balance the specified group. This is used by the
// balanced match construct. (?<foo-foo2>...)
//
// balanced match construct. (?<foo-foo2>...)
//
// If there were no such thing as backtracking, this would be as simple as calling RemoveMatch(c).
// However, since we have backtracking, we need to keep track of everything.

48
vendor/github.com/dlclark/regexp2/regexp.go generated vendored
View File

@@ -18,13 +18,21 @@ import (
"github.com/dlclark/regexp2/syntax"
)
// Default timeout used when running regexp matches -- "forever"
var DefaultMatchTimeout = time.Duration(math.MaxInt64)
var (
// DefaultMatchTimeout used when running regexp matches -- "forever"
DefaultMatchTimeout = time.Duration(math.MaxInt64)
// DefaultUnmarshalOptions used when unmarshaling a regex from text
DefaultUnmarshalOptions = None
)
// Regexp is the representation of a compiled regular expression.
// A Regexp is safe for concurrent use by multiple goroutines.
type Regexp struct {
//timeout when trying to find matches
// A match will time out if it takes (approximately) more than
// MatchTimeout. This is a safety check in case the match
// encounters catastrophic backtracking. The default value
// (DefaultMatchTimeout) causes all time out checking to be
// suppressed.
MatchTimeout time.Duration
// read-only after Compile
@@ -39,7 +47,7 @@ type Regexp struct {
code *syntax.Code // compiled program
// cache of machines for running regexp
muRun sync.Mutex
muRun *sync.Mutex
runner []*runner
}
@@ -68,6 +76,7 @@ func Compile(expr string, opt RegexOptions) (*Regexp, error) {
capsize: code.Capsize,
code: code,
MatchTimeout: DefaultMatchTimeout,
muRun: &sync.Mutex{},
}, nil
}
@@ -92,6 +101,19 @@ func Unescape(input string) (string, error) {
return syntax.Unescape(input)
}
// SetTimeoutPeriod is a debug function that sets the frequency of the timeout goroutine's sleep cycle.
// Defaults to 100ms. The only benefit of setting this lower is that the 1 background goroutine that manages
// timeouts may exit slightly sooner after all the timeouts have expired. See Github issue #63
func SetTimeoutCheckPeriod(d time.Duration) {
clockPeriod = d
}
// StopTimeoutClock should only be used in unit tests to prevent the timeout clock goroutine
// from appearing like a leaking goroutine
func StopTimeoutClock() {
stopClock()
}
// String returns the source text used to compile the regular expression.
func (re *Regexp) String() string {
return re.pattern
@@ -121,6 +143,7 @@ const (
Debug = 0x0080 // "d"
ECMAScript = 0x0100 // "e"
RE2 = 0x0200 // RE2 (regexp package) compatibility mode
Unicode = 0x0400 // "u"
)
func (re *Regexp) RightToLeft() bool {
@@ -353,3 +376,20 @@ func (re *Regexp) GroupNumberFromName(name string) int {
return -1
}
// MarshalText implements [encoding.TextMarshaler]. The output
// matches that of calling the [Regexp.String] method.
func (re *Regexp) MarshalText() ([]byte, error) {
return []byte(re.String()), nil
}
// UnmarshalText implements [encoding.TextUnmarshaler] by calling
// [Compile] on the encoded value.
func (re *Regexp) UnmarshalText(text []byte) error {
newRE, err := Compile(string(text), DefaultUnmarshalOptions)
if err != nil {
return err
}
*re = *newRE
return nil
}

66
vendor/github.com/dlclark/regexp2/runner.go generated vendored
View File

@@ -58,10 +58,9 @@ type runner struct {
runmatch *Match // result object
ignoreTimeout bool
timeout time.Duration // timeout in milliseconds (needed for actual)
timeoutChecksToSkip int
timeoutAt time.Time
ignoreTimeout bool
timeout time.Duration // timeout in milliseconds (needed for actual)
deadline fasttime
operator syntax.InstOp
codepos int
@@ -314,12 +313,9 @@ func (r *runner) execute() error {
}
} else {
// The inner expression found an empty match, so we'll go directly to 'back2' if we
// backtrack. In this case, we need to push something on the stack, since back2 pops.
// However, in the case of ()+? or similar, this empty match may be legitimate, so push the text
// position associated with that empty match.
r.stackPush(oldMarkPos)
r.trackPushNeg1(r.stackPeek()) // Save old mark
// backtrack. Don't touch the grouping stack here; instead, record the old mark and
// a flag indicating that backtracking doesn't need to pop a grouping stack frame.
r.trackPushNeg2(oldMarkPos, 0)
}
r.advance(1)
continue
@@ -335,18 +331,22 @@ func (r *runner) execute() error {
r.trackPopN(2)
pos := r.trackPeekN(1)
r.trackPushNeg1(r.trackPeek()) // Save old mark
r.stackPush(pos) // Make new mark
r.textto(pos) // Recall position
r.goTo(r.operand(0)) // Loop
r.trackPushNeg2(r.trackPeek(), 1) // Save old mark, note that we pushed a new mark
r.stackPush(pos) // Make new mark
r.textto(pos) // Recall position
r.goTo(r.operand(0)) // Loop
continue
case syntax.Lazybranchmark | syntax.Back2:
// The lazy loop has failed. We'll do a true backtrack and
// start over before the lazy loop.
r.stackPop()
r.trackPop()
r.stackPush(r.trackPeek()) // Recall old mark
r.trackPopN(2)
oldMark := r.trackPeek()
needsPop := r.trackPeekN(1)
if needsPop != 0 {
r.stackPop()
}
r.stackPush(oldMark) // Recall old mark
break
case syntax.Setcount:
@@ -1551,39 +1551,15 @@ func (r *runner) isECMABoundary(index, startpos, endpos int) bool {
(index < endpos && syntax.IsECMAWordChar(r.runtext[index]))
}
// this seems like a comment to justify randomly picking 1000 :-P
// We have determined this value in a series of experiments where x86 retail
// builds (ono-lab-optimized) were run on different pattern/input pairs. Larger values
// of TimeoutCheckFrequency did not tend to increase performance; smaller values
// of TimeoutCheckFrequency tended to slow down the execution.
const timeoutCheckFrequency int = 1000
func (r *runner) startTimeoutWatch() {
if r.ignoreTimeout {
return
}
r.timeoutChecksToSkip = timeoutCheckFrequency
r.timeoutAt = time.Now().Add(r.timeout)
r.deadline = makeDeadline(r.timeout)
}
func (r *runner) checkTimeout() error {
if r.ignoreTimeout {
return nil
}
r.timeoutChecksToSkip--
if r.timeoutChecksToSkip != 0 {
return nil
}
r.timeoutChecksToSkip = timeoutCheckFrequency
return r.doCheckTimeout()
}
func (r *runner) doCheckTimeout() error {
current := time.Now()
if current.Before(r.timeoutAt) {
if r.ignoreTimeout || !r.deadline.reached() {
return nil
}
@@ -1629,6 +1605,10 @@ func (re *Regexp) getRunner() *runner {
// run using re. (The cache empties when re gets garbage collected.)
func (re *Regexp) putRunner(r *runner) {
re.muRun.Lock()
r.runtext = nil
if r.runmatch != nil {
r.runmatch.text = nil
}
re.runner = append(re.runner, r)
re.muRun.Unlock()
}

15
vendor/github.com/dlclark/regexp2/syntax/charclass.go generated vendored
View File

@@ -37,6 +37,8 @@ var (
ecmaSpace = []rune{0x0009, 0x000e, 0x0020, 0x0021, 0x00a0, 0x00a1, 0x1680, 0x1681, 0x2000, 0x200b, 0x2028, 0x202a, 0x202f, 0x2030, 0x205f, 0x2060, 0x3000, 0x3001, 0xfeff, 0xff00}
ecmaWord = []rune{0x0030, 0x003a, 0x0041, 0x005b, 0x005f, 0x0060, 0x0061, 0x007b}
ecmaDigit = []rune{0x0030, 0x003a}
re2Space = []rune{0x0009, 0x000b, 0x000c, 0x000e, 0x0020, 0x0021}
)
var (
@@ -56,6 +58,9 @@ var (
NotSpaceClass = getCharSetFromCategoryString(true, false, spaceCategoryText)
DigitClass = getCharSetFromCategoryString(false, false, "Nd")
NotDigitClass = getCharSetFromCategoryString(false, true, "Nd")
RE2SpaceClass = getCharSetFromOldString(re2Space, false)
NotRE2SpaceClass = getCharSetFromOldString(re2Space, true)
)
var unicodeCategories = func() map[string]*unicode.RangeTable {
@@ -401,13 +406,19 @@ func (c *CharSet) addChar(ch rune) {
c.addRange(ch, ch)
}
func (c *CharSet) addSpace(ecma, negate bool) {
func (c *CharSet) addSpace(ecma, re2, negate bool) {
if ecma {
if negate {
c.addRanges(NotECMASpaceClass().ranges)
} else {
c.addRanges(ECMASpaceClass().ranges)
}
} else if re2 {
if negate {
c.addRanges(NotRE2SpaceClass().ranges)
} else {
c.addRanges(RE2SpaceClass().ranges)
}
} else {
c.addCategories(category{cat: spaceCategoryText, negate: negate})
}
@@ -563,7 +574,7 @@ func (c *CharSet) addNamedASCII(name string, negate bool) bool {
case "punct": //[!-/:-@[-`{-~]
rs = []singleRange{singleRange{'!', '/'}, singleRange{':', '@'}, singleRange{'[', '`'}, singleRange{'{', '~'}}
case "space":
c.addSpace(true, negate)
c.addSpace(true, false, negate)
case "upper":
rs = []singleRange{singleRange{'A', 'Z'}}
case "word":

93
vendor/github.com/dlclark/regexp2/syntax/parser.go generated vendored
View File

@@ -22,6 +22,7 @@ const (
Debug = 0x0080 // "d"
ECMAScript = 0x0100 // "e"
RE2 = 0x0200 // RE2 compat mode
Unicode = 0x0400 // "u"
)
func optionFromCode(ch rune) RegexOptions {
@@ -43,6 +44,8 @@ func optionFromCode(ch rune) RegexOptions {
return Debug
case 'e', 'E':
return ECMAScript
case 'u', 'U':
return Unicode
default:
return 0
}
@@ -104,7 +107,7 @@ const (
ErrBadClassInCharRange = "cannot include class \\%v in character range"
ErrUnterminatedBracket = "unterminated [] set"
ErrSubtractionMustBeLast = "a subtraction must be the last element in a character class"
ErrReversedCharRange = "[x-y] range in reverse order"
ErrReversedCharRange = "[%c-%c] range in reverse order"
)
func (e ErrorCode) String() string {
@@ -550,10 +553,10 @@ func (p *parser) scanRegex() (*regexNode, error) {
}
case '.':
if p.useOptionE() {
p.addUnitSet(ECMAAnyClass())
} else if p.useOptionS() {
if p.useOptionS() {
p.addUnitSet(AnyClass())
} else if p.useOptionE() {
p.addUnitSet(ECMAAnyClass())
} else {
p.addUnitNotone('\n')
}
@@ -1121,14 +1124,14 @@ func (p *parser) scanBackslash(scanOnly bool) (*regexNode, error) {
case 'w':
p.moveRight(1)
if p.useOptionE() {
if p.useOptionE() || p.useRE2() {
return newRegexNodeSet(ntSet, p.options, ECMAWordClass()), nil
}
return newRegexNodeSet(ntSet, p.options, WordClass()), nil
case 'W':
p.moveRight(1)
if p.useOptionE() {
if p.useOptionE() || p.useRE2() {
return newRegexNodeSet(ntSet, p.options, NotECMAWordClass()), nil
}
return newRegexNodeSet(ntSet, p.options, NotWordClass()), nil
@@ -1137,6 +1140,8 @@ func (p *parser) scanBackslash(scanOnly bool) (*regexNode, error) {
p.moveRight(1)
if p.useOptionE() {
return newRegexNodeSet(ntSet, p.options, ECMASpaceClass()), nil
} else if p.useRE2() {
return newRegexNodeSet(ntSet, p.options, RE2SpaceClass()), nil
}
return newRegexNodeSet(ntSet, p.options, SpaceClass()), nil
@@ -1144,19 +1149,21 @@ func (p *parser) scanBackslash(scanOnly bool) (*regexNode, error) {
p.moveRight(1)
if p.useOptionE() {
return newRegexNodeSet(ntSet, p.options, NotECMASpaceClass()), nil
} else if p.useRE2() {
return newRegexNodeSet(ntSet, p.options, NotRE2SpaceClass()), nil
}
return newRegexNodeSet(ntSet, p.options, NotSpaceClass()), nil
case 'd':
p.moveRight(1)
if p.useOptionE() {
if p.useOptionE() || p.useRE2() {
return newRegexNodeSet(ntSet, p.options, ECMADigitClass()), nil
}
return newRegexNodeSet(ntSet, p.options, DigitClass()), nil
case 'D':
p.moveRight(1)
if p.useOptionE() {
if p.useOptionE() || p.useRE2() {
return newRegexNodeSet(ntSet, p.options, NotECMADigitClass()), nil
}
return newRegexNodeSet(ntSet, p.options, NotDigitClass()), nil
@@ -1186,19 +1193,24 @@ func (p *parser) scanBasicBackslash(scanOnly bool) (*regexNode, error) {
return nil, p.getErr(ErrIllegalEndEscape)
}
angled := false
k := false
close := '\x00'
backpos := p.textpos()
ch := p.rightChar(0)
// allow \k<foo> instead of \<foo>, which is now deprecated
// Allow \k<foo> instead of \<foo>, which is now deprecated.
if ch == 'k' {
// According to ECMAScript specification, \k<name> is only parsed as a named group reference if
// there is at least one group name in the regexp.
// See https://www.ecma-international.org/ecma-262/#sec-isvalidregularexpressionliteral, step 7.
// Note, during the first (scanOnly) run we may not have all group names scanned, but that's ok.
if ch == 'k' && (!p.useOptionE() || len(p.capnames) > 0) {
if p.charsRight() >= 2 {
p.moveRight(1)
ch = p.moveRightGetChar()
if ch == '<' || ch == '\'' {
if ch == '<' || (!p.useOptionE() && ch == '\'') { // No support for \k'name' in ECMAScript
angled = true
if ch == '\'' {
close = '\''
@@ -1213,8 +1225,9 @@ func (p *parser) scanBasicBackslash(scanOnly bool) (*regexNode, error) {
}
ch = p.rightChar(0)
k = true
} else if (ch == '<' || ch == '\'') && p.charsRight() > 1 { // Note angle without \g
} else if !p.useOptionE() && (ch == '<' || ch == '\'') && p.charsRight() > 1 { // Note angle without \g
angled = true
if ch == '\'' {
close = '\''
@@ -1257,14 +1270,23 @@ func (p *parser) scanBasicBackslash(scanOnly bool) (*regexNode, error) {
return nil, p.getErr(ErrUndefinedBackRef, capnum)
}
} else if angled && IsWordChar(ch) {
} else if angled {
capname := p.scanCapname()
if p.charsRight() > 0 && p.moveRightGetChar() == close {
if capname != "" && p.charsRight() > 0 && p.moveRightGetChar() == close {
if scanOnly {
return nil, nil
}
if p.isCaptureName(capname) {
return newRegexNodeM(ntRef, p.options, p.captureSlotFromName(capname)), nil
}
return nil, p.getErr(ErrUndefinedNameRef, capname)
} else {
if k {
return nil, p.getErr(ErrMalformedNameRef)
}
}
}
@@ -1276,6 +1298,10 @@ func (p *parser) scanBasicBackslash(scanOnly bool) (*regexNode, error) {
return nil, err
}
if scanOnly {
return nil, nil
}
if p.useOptionI() {
ch = unicode.ToLower(ch)
}
@@ -1285,6 +1311,17 @@ func (p *parser) scanBasicBackslash(scanOnly bool) (*regexNode, error) {
// Scans X for \p{X} or \P{X}
func (p *parser) parseProperty() (string, error) {
// RE2 and PCRE supports \pX syntax (no {} and only 1 letter unicode cats supported)
// since this is purely additive syntax it's not behind a flag
if p.charsRight() >= 1 && p.rightChar(0) != '{' {
ch := string(p.moveRightGetChar())
// check if it's a valid cat
if !isValidUnicodeCat(ch) {
return "", p.getErr(ErrUnknownSlashP, ch)
}
return ch, nil
}
if p.charsRight() < 3 {
return "", p.getErr(ErrIncompleteSlashP)
}
@@ -1401,7 +1438,7 @@ func (p *parser) scanCapname() string {
return string(p.pattern[startpos:p.textpos()])
}
//Scans contents of [] (not including []'s), and converts to a set.
// Scans contents of [] (not including []'s), and converts to a set.
func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
ch := '\x00'
chPrev := '\x00'
@@ -1443,7 +1480,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
if inRange {
return nil, p.getErr(ErrBadClassInCharRange, ch)
}
cc.addDigit(p.useOptionE(), ch == 'D', p.patternRaw)
cc.addDigit(p.useOptionE() || p.useRE2(), ch == 'D', p.patternRaw)
}
continue
@@ -1452,7 +1489,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
if inRange {
return nil, p.getErr(ErrBadClassInCharRange, ch)
}
cc.addSpace(p.useOptionE(), ch == 'S')
cc.addSpace(p.useOptionE(), p.useRE2(), ch == 'S')
}
continue
@@ -1462,7 +1499,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
return nil, p.getErr(ErrBadClassInCharRange, ch)
}
cc.addWord(p.useOptionE(), ch == 'W')
cc.addWord(p.useOptionE() || p.useRE2(), ch == 'W')
}
continue
@@ -1548,7 +1585,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
} else {
// a regular range, like a-z
if chPrev > ch {
return nil, p.getErr(ErrReversedCharRange)
return nil, p.getErr(ErrReversedCharRange, chPrev, ch)
}
cc.addRange(chPrev, ch)
}
@@ -1672,7 +1709,13 @@ func (p *parser) scanCharEscape() (r rune, err error) {
r, err = p.scanHex(2)
}
case 'u':
r, err = p.scanHex(4)
// ECMAscript suppot \u{HEX} only if `u` is also set
if p.useOptionE() && p.useOptionU() && p.charsRight() > 0 && p.rightChar(0) == '{' {
p.moveRight(1)
return p.scanHexUntilBrace()
} else {
r, err = p.scanHex(4)
}
case 'a':
return '\u0007', nil
case 'b':
@@ -1692,7 +1735,7 @@ func (p *parser) scanCharEscape() (r rune, err error) {
case 'c':
r, err = p.scanControl()
default:
if !p.useOptionE() && IsWordChar(ch) {
if !p.useOptionE() && !p.useRE2() && IsWordChar(ch) {
return 0, p.getErr(ErrUnrecognizedEscape, string(ch))
}
return ch, nil
@@ -1949,6 +1992,11 @@ func (p *parser) useRE2() bool {
return (p.options & RE2) != 0
}
// True if U option enabling ECMAScript's Unicode behavior on.
func (p *parser) useOptionU() bool {
return (p.options & Unicode) != 0
}
// True if options stack is empty.
func (p *parser) emptyOptionsStack() bool {
return len(p.optionsStack) == 0
@@ -2044,7 +2092,8 @@ func (p *parser) addToConcatenate(pos, cch int, isReplacement bool) {
}
if cch > 1 {
str := p.pattern[pos : pos+cch]
str := make([]rune, cch)
copy(str, p.pattern[pos:pos+cch])
if p.useOptionI() && !isReplacement {
// We do the ToLower character by character for consistency. With surrogate chars, doing

2
vendor/github.com/dlclark/regexp2/syntax/prefix.go generated vendored
View File

@@ -712,7 +712,7 @@ func (b *BmPrefix) Scan(text []rune, index, beglimit, endlimit int) int {
if chTest != b.pattern[match] {
advance = b.positive[match]
if (chTest & 0xFF80) == 0 {
if chTest < 128 {
test2 = (match - startmatch) + b.negativeASCII[chTest]
} else if chTest < 0xffff && len(b.negativeUnicode) > 0 {
unicodeLookup = b.negativeUnicode[chTest>>8]

112
vendor/github.com/fatih/color/color.go generated vendored
View File

@@ -19,15 +19,15 @@ var (
// set (regardless of its value). This is a global option and affects all
// colors. For more control over each color block use the methods
// DisableColor() individually.
NoColor = noColorIsSet() || os.Getenv("TERM") == "dumb" ||
(!isatty.IsTerminal(os.Stdout.Fd()) && !isatty.IsCygwinTerminal(os.Stdout.Fd()))
NoColor = noColorIsSet() || os.Getenv("TERM") == "dumb" || !stdoutIsTerminal()
// Output defines the standard output of the print functions. By default,
// os.Stdout is used.
Output = colorable.NewColorableStdout()
// stdOut() is used.
Output = stdOut()
// Error defines a color supporting writer for os.Stderr.
Error = colorable.NewColorableStderr()
// Error defines the standard error of the print functions. By default,
// stdErr() is used.
Error = stdErr()
// colorsCache is used to reduce the count of created Color objects and
// allows to reuse already created objects with required Attribute.
@@ -40,6 +40,33 @@ func noColorIsSet() bool {
return os.Getenv("NO_COLOR") != ""
}
// stdoutIsTerminal returns true if os.Stdout is a terminal.
// Returns false if os.Stdout is nil (e.g., when running as a Windows service).
func stdoutIsTerminal() bool {
if os.Stdout == nil {
return false
}
return isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())
}
// stdOut returns a writer for color output.
// Returns io.Discard if os.Stdout is nil (e.g., when running as a Windows service).
func stdOut() io.Writer {
if os.Stdout == nil {
return io.Discard
}
return colorable.NewColorableStdout()
}
// stdErr returns a writer for color error output.
// Returns io.Discard if os.Stderr is nil (e.g., when running as a Windows service).
func stdErr() io.Writer {
if os.Stderr == nil {
return io.Discard
}
return colorable.NewColorableStderr()
}
// Color defines a custom color object which is defined by SGR parameters.
type Color struct {
params []Attribute
@@ -220,26 +247,30 @@ func (c *Color) unset() {
// a low-level function, and users should use the higher-level functions, such
// as color.Fprint, color.Print, etc.
func (c *Color) SetWriter(w io.Writer) *Color {
_, _ = c.setWriter(w)
return c
}
func (c *Color) setWriter(w io.Writer) (int, error) {
if c.isNoColorSet() {
return c
return 0, nil
}
fmt.Fprint(w, c.format())
return c
return fmt.Fprint(w, c.format())
}
// UnsetWriter resets all escape attributes and clears the output with the give
// io.Writer. Usually should be called after SetWriter().
func (c *Color) UnsetWriter(w io.Writer) {
_, _ = c.unsetWriter(w)
}
func (c *Color) unsetWriter(w io.Writer) (int, error) {
if c.isNoColorSet() {
return
return 0, nil
}
if NoColor {
return
}
fmt.Fprintf(w, "%s[%dm", escape, Reset)
return fmt.Fprintf(w, "%s[%dm", escape, Reset)
}
// Add is used to chain SGR parameters. Use as many as parameters to combine
@@ -255,10 +286,20 @@ func (c *Color) Add(value ...Attribute) *Color {
// On Windows, users should wrap w with colorable.NewColorable() if w is of
// type *os.File.
func (c *Color) Fprint(w io.Writer, a ...interface{}) (n int, err error) {
c.SetWriter(w)
defer c.UnsetWriter(w)
n, err = c.setWriter(w)
if err != nil {
return n, err
}
return fmt.Fprint(w, a...)
nn, err := fmt.Fprint(w, a...)
n += nn
if err != nil {
return
}
nn, err = c.unsetWriter(w)
n += nn
return n, err
}
// Print formats using the default formats for its operands and writes to
@@ -278,10 +319,20 @@ func (c *Color) Print(a ...interface{}) (n int, err error) {
// On Windows, users should wrap w with colorable.NewColorable() if w is of
// type *os.File.
func (c *Color) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
c.SetWriter(w)
defer c.UnsetWriter(w)
n, err = c.setWriter(w)
if err != nil {
return n, err
}
return fmt.Fprintf(w, format, a...)
nn, err := fmt.Fprintf(w, format, a...)
n += nn
if err != nil {
return
}
nn, err = c.unsetWriter(w)
n += nn
return n, err
}
// Printf formats according to a format specifier and writes to standard output.
@@ -475,29 +526,26 @@ func (c *Color) Equals(c2 *Color) bool {
if c == nil || c2 == nil {
return false
}
if len(c.params) != len(c2.params) {
return false
}
counts := make(map[Attribute]int, len(c.params))
for _, attr := range c.params {
if !c2.attrExists(attr) {
counts[attr]++
}
for _, attr := range c2.params {
if counts[attr] == 0 {
return false
}
counts[attr]--
}
return true
}
func (c *Color) attrExists(a Attribute) bool {
for _, attr := range c.params {
if attr == a {
return true
}
}
return false
}
func boolPtr(v bool) *bool {
return &v
}

3
vendor/github.com/fatih/color/color_windows.go generated vendored
View File

@@ -9,6 +9,9 @@ import (
func init() {
// Opt-in for ansi color support for current process.
// https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#output-sequences
if os.Stdout == nil {
return
}
var outMode uint32
out := windows.Handle(os.Stdout.Fd())
if err := windows.GetConsoleMode(out, &outMode); err != nil {

12
vendor/github.com/go-ini/ini/.editorconfig generated vendored
View File

@@ -1,12 +0,0 @@
# http://editorconfig.org
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[*_test.go]
trim_trailing_whitespace = false

7
vendor/github.com/go-ini/ini/.gitignore generated vendored
View File

@@ -1,7 +0,0 @@
testdata/conf_out.ini
ini.sublime-project
ini.sublime-workspace
testdata/conf_reflect.ini
.idea
/.vscode
.DS_Store

27
vendor/github.com/go-ini/ini/.golangci.yml generated vendored
View File

@@ -1,27 +0,0 @@
linters-settings:
staticcheck:
checks: [
"all",
"-SA1019" # There are valid use cases of strings.Title
]
nakedret:
max-func-lines: 0 # Disallow any unnamed return statement
linters:
enable:
- deadcode
- errcheck
- gosimple
- govet
- ineffassign
- staticcheck
- structcheck
- typecheck
- unused
- varcheck
- nakedret
- gofmt
- rowserrcheck
- unconvert
- goimports
- unparam

191
vendor/github.com/go-ini/ini/LICENSE generated vendored
View File

@@ -1,191 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct or
indirect, to cause the direction or management of such entity, whether by
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.
"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.
"Object" form shall mean any form resulting from mechanical transformation or
translation of a Source form, including but not limited to compiled object code,
generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form, made
available under the License, as indicated by a copyright notice that is included
in or attached to the work (an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object form, that
is based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative Works
shall not include works that remain separable from, or merely link (or bind by
name) to the interfaces of, the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative Works
thereof, that is intentionally submitted to Licensor for inclusion in the Work
by the copyright owner or by an individual or Legal Entity authorized to submit
on behalf of the copyright owner. For the purposes of this definition,
"submitted" means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems, and
issue tracking systems that are managed by, or on behalf of, the Licensor for
the purpose of discussing and improving the Work, but excluding communication
that is conspicuously marked or otherwise designated in writing by the copyright
owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
of whom a Contribution has been received by Licensor and subsequently
incorporated within the Work.
2. Grant of Copyright License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the Work and such
Derivative Works in Source or Object form.
3. Grant of Patent License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to make, have
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
such license applies only to those patent claims licensable by such Contributor
that are necessarily infringed by their Contribution(s) alone or by combination
of their Contribution(s) with the Work to which such Contribution(s) was
submitted. If You institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
Contribution incorporated within the Work constitutes direct or contributory
patent infringement, then any patent licenses granted to You under this License
for that Work shall terminate as of the date such litigation is filed.
4. Redistribution.
You may reproduce and distribute copies of the Work or Derivative Works thereof
in any medium, with or without modifications, and in Source or Object form,
provided that You meet the following conditions:
You must give any other recipients of the Work or Derivative Works a copy of
this License; and
You must cause any modified files to carry prominent notices stating that You
changed the files; and
You must retain, in the Source form of any Derivative Works that You distribute,
all copyright, patent, trademark, and attribution notices from the Source form
of the Work, excluding those notices that do not pertain to any part of the
Derivative Works; and
If the Work includes a "NOTICE" text file as part of its distribution, then any
Derivative Works that You distribute must include a readable copy of the
attribution notices contained within such NOTICE file, excluding those notices
that do not pertain to any part of the Derivative Works, in at least one of the
following places: within a NOTICE text file distributed as part of the
Derivative Works; within the Source form or documentation, if provided along
with the Derivative Works; or, within a display generated by the Derivative
Works, if and wherever such third-party notices normally appear. The contents of
the NOTICE file are for informational purposes only and do not modify the
License. You may add Your own attribution notices within Derivative Works that
You distribute, alongside or as an addendum to the NOTICE text from the Work,
provided that such additional attribution notices cannot be construed as
modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction, or
distribution of Your modifications, or for any such Derivative Works as a whole,
provided Your use, reproduction, and distribution of the Work otherwise complies
with the conditions stated in this License.
5. Submission of Contributions.
Unless You explicitly state otherwise, any Contribution intentionally submitted
for inclusion in the Work by You to the Licensor shall be under the terms and
conditions of this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify the terms of
any separate license agreement you may have executed with Licensor regarding
such Contributions.
6. Trademarks.
This License does not grant permission to use the trade names, trademarks,
service marks, or product names of the Licensor, except as required for
reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.
7. Disclaimer of Warranty.
Unless required by applicable law or agreed to in writing, Licensor provides the
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
including, without limitation, any warranties or conditions of TITLE,
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
solely responsible for determining the appropriateness of using or
redistributing the Work and assume any risks associated with Your exercise of
permissions under this License.
8. Limitation of Liability.
In no event and under no legal theory, whether in tort (including negligence),
contract, or otherwise, unless required by applicable law (such as deliberate
and grossly negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special, incidental,
or consequential damages of any character arising as a result of this License or
out of the use or inability to use the Work (including but not limited to
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
any and all other commercial damages or losses), even if such Contributor has
been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability.
While redistributing the Work or Derivative Works thereof, You may choose to
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
other liability obligations and/or rights consistent with this License. However,
in accepting such obligations, You may act only on Your own behalf and on Your
sole responsibility, not on behalf of any other Contributor, and only if You
agree to indemnify, defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason of your
accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work
To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own
identifying information. (Don't include the brackets!) The text should be
enclosed in the appropriate comment syntax for the file format. We also
recommend that a file or class name and description of purpose be included on
the same "printed page" as the copyright notice for easier identification within
third-party archives.
Copyright 2014 Unknwon
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

15
vendor/github.com/go-ini/ini/Makefile generated vendored
View File

@@ -1,15 +0,0 @@
.PHONY: build test bench vet coverage
build: vet bench
test:
go test -v -cover -race
bench:
go test -v -cover -test.bench=. -test.benchmem
vet:
go vet
coverage:
go test -coverprofile=c.out && go tool cover -html=c.out && rm c.out

43
vendor/github.com/go-ini/ini/README.md generated vendored
View File

@@ -1,43 +0,0 @@
# INI
[![GitHub Workflow Status](https://img.shields.io/github/checks-status/go-ini/ini/main?logo=github&style=for-the-badge)](https://github.com/go-ini/ini/actions?query=branch%3Amain)
[![codecov](https://img.shields.io/codecov/c/github/go-ini/ini/master?logo=codecov&style=for-the-badge)](https://codecov.io/gh/go-ini/ini)
[![GoDoc](https://img.shields.io/badge/GoDoc-Reference-blue?style=for-the-badge&logo=go)](https://pkg.go.dev/github.com/go-ini/ini?tab=doc)
[![Sourcegraph](https://img.shields.io/badge/view%20on-Sourcegraph-brightgreen.svg?style=for-the-badge&logo=sourcegraph)](https://sourcegraph.com/github.com/go-ini/ini)
![](https://avatars0.githubusercontent.com/u/10216035?v=3&s=200)
Package ini provides INI file read and write functionality in Go.
## Features
- Load from multiple data sources(file, `[]byte`, `io.Reader` and `io.ReadCloser`) with overwrites.
- Read with recursion values.
- Read with parent-child sections.
- Read with auto-increment key names.
- Read with multiple-line values.
- Read with tons of helper methods.
- Read and convert values to Go types.
- Read and **WRITE** comments of sections and keys.
- Manipulate sections, keys and comments with ease.
- Keep sections and keys in order as you parse and save.
## Installation
The minimum requirement of Go is **1.13**.
```sh
$ go get gopkg.in/ini.v1
```
Please add `-u` flag to update in the future.
## Getting Help
- [Getting Started](https://ini.unknwon.io/docs/intro/getting_started)
- [API Documentation](https://gowalker.org/gopkg.in/ini.v1)
- 中国大陆镜像：https://ini.unknwon.cn
## License
This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text.

16
vendor/github.com/go-ini/ini/codecov.yml generated vendored
View File

@@ -1,16 +0,0 @@
coverage:
range: "60...95"
status:
project:
default:
threshold: 1%
informational: true
patch:
defualt:
only_pulls: true
informational: true
comment:
layout: 'diff'
github_checks: false

76
vendor/github.com/go-ini/ini/data_source.go generated vendored
View File

@@ -1,76 +0,0 @@
// Copyright 2019 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
)
var (
_ dataSource = (*sourceFile)(nil)
_ dataSource = (*sourceData)(nil)
_ dataSource = (*sourceReadCloser)(nil)
)
// dataSource is an interface that returns object which can be read and closed.
type dataSource interface {
ReadCloser() (io.ReadCloser, error)
}
// sourceFile represents an object that contains content on the local file system.
type sourceFile struct {
name string
}
func (s sourceFile) ReadCloser() (_ io.ReadCloser, err error) {
return os.Open(s.name)
}
// sourceData represents an object that contains content in memory.
type sourceData struct {
data []byte
}
func (s *sourceData) ReadCloser() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewReader(s.data)), nil
}
// sourceReadCloser represents an input stream with Close method.
type sourceReadCloser struct {
reader io.ReadCloser
}
func (s *sourceReadCloser) ReadCloser() (io.ReadCloser, error) {
return s.reader, nil
}
func parseDataSource(source interface{}) (dataSource, error) {
switch s := source.(type) {
case string:
return sourceFile{s}, nil
case []byte:
return &sourceData{s}, nil
case io.ReadCloser:
return &sourceReadCloser{s}, nil
case io.Reader:
return &sourceReadCloser{ioutil.NopCloser(s)}, nil
default:
return nil, fmt.Errorf("error parsing data source: unknown type %q", s)
}
}

22
vendor/github.com/go-ini/ini/deprecated.go generated vendored
View File

@@ -1,22 +0,0 @@
// Copyright 2019 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
var (
// Deprecated: Use "DefaultSection" instead.
DEFAULT_SECTION = DefaultSection
// Deprecated: AllCapsUnderscore converts to format ALL_CAPS_UNDERSCORE.
AllCapsUnderscore = SnackCase
)

49
vendor/github.com/go-ini/ini/error.go generated vendored
View File

@@ -1,49 +0,0 @@
// Copyright 2016 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
import (
"fmt"
)
// ErrDelimiterNotFound indicates the error type of no delimiter is found which there should be one.
type ErrDelimiterNotFound struct {
Line string
}
// IsErrDelimiterNotFound returns true if the given error is an instance of ErrDelimiterNotFound.
func IsErrDelimiterNotFound(err error) bool {
_, ok := err.(ErrDelimiterNotFound)
return ok
}
func (err ErrDelimiterNotFound) Error() string {
return fmt.Sprintf("key-value delimiter not found: %s", err.Line)
}
// ErrEmptyKeyName indicates the error type of no key name is found which there should be one.
type ErrEmptyKeyName struct {
Line string
}
// IsErrEmptyKeyName returns true if the given error is an instance of ErrEmptyKeyName.
func IsErrEmptyKeyName(err error) bool {
_, ok := err.(ErrEmptyKeyName)
return ok
}
func (err ErrEmptyKeyName) Error() string {
return fmt.Sprintf("empty key name: %s", err.Line)
}

541
vendor/github.com/go-ini/ini/file.go generated vendored
View File

@@ -1,541 +0,0 @@
// Copyright 2017 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"sync"
)
// File represents a combination of one or more INI files in memory.
type File struct {
options LoadOptions
dataSources []dataSource
// Should make things safe, but sometimes doesn't matter.
BlockMode bool
lock sync.RWMutex
// To keep data in order.
sectionList []string
// To keep track of the index of a section with same name.
// This meta list is only used with non-unique section names are allowed.
sectionIndexes []int
// Actual data is stored here.
sections map[string][]*Section
NameMapper
ValueMapper
}
// newFile initializes File object with given data sources.
func newFile(dataSources []dataSource, opts LoadOptions) *File {
if len(opts.KeyValueDelimiters) == 0 {
opts.KeyValueDelimiters = "=:"
}
if len(opts.KeyValueDelimiterOnWrite) == 0 {
opts.KeyValueDelimiterOnWrite = "="
}
if len(opts.ChildSectionDelimiter) == 0 {
opts.ChildSectionDelimiter = "."
}
return &File{
BlockMode: true,
dataSources: dataSources,
sections: make(map[string][]*Section),
options: opts,
}
}
// Empty returns an empty file object.
func Empty(opts ...LoadOptions) *File {
var opt LoadOptions
if len(opts) > 0 {
opt = opts[0]
}
// Ignore error here, we are sure our data is good.
f, _ := LoadSources(opt, []byte(""))
return f
}
// NewSection creates a new section.
func (f *File) NewSection(name string) (*Section, error) {
if len(name) == 0 {
return nil, errors.New("empty section name")
}
if (f.options.Insensitive || f.options.InsensitiveSections) && name != DefaultSection {
name = strings.ToLower(name)
}
if f.BlockMode {
f.lock.Lock()
defer f.lock.Unlock()
}
if !f.options.AllowNonUniqueSections && inSlice(name, f.sectionList) {
return f.sections[name][0], nil
}
f.sectionList = append(f.sectionList, name)
// NOTE: Append to indexes must happen before appending to sections,
// otherwise index will have off-by-one problem.
f.sectionIndexes = append(f.sectionIndexes, len(f.sections[name]))
sec := newSection(f, name)
f.sections[name] = append(f.sections[name], sec)
return sec, nil
}
// NewRawSection creates a new section with an unparseable body.
func (f *File) NewRawSection(name, body string) (*Section, error) {
section, err := f.NewSection(name)
if err != nil {
return nil, err
}
section.isRawSection = true
section.rawBody = body
return section, nil
}
// NewSections creates a list of sections.
func (f *File) NewSections(names ...string) (err error) {
for _, name := range names {
if _, err = f.NewSection(name); err != nil {
return err
}
}
return nil
}
// GetSection returns section by given name.
func (f *File) GetSection(name string) (*Section, error) {
secs, err := f.SectionsByName(name)
if err != nil {
return nil, err
}
return secs[0], err
}
// HasSection returns true if the file contains a section with given name.
func (f *File) HasSection(name string) bool {
section, _ := f.GetSection(name)
return section != nil
}
// SectionsByName returns all sections with given name.
func (f *File) SectionsByName(name string) ([]*Section, error) {
if len(name) == 0 {
name = DefaultSection
}
if f.options.Insensitive || f.options.InsensitiveSections {
name = strings.ToLower(name)
}
if f.BlockMode {
f.lock.RLock()
defer f.lock.RUnlock()
}
secs := f.sections[name]
if len(secs) == 0 {
return nil, fmt.Errorf("section %q does not exist", name)
}
return secs, nil
}
// Section assumes named section exists and returns a zero-value when not.
func (f *File) Section(name string) *Section {
sec, err := f.GetSection(name)
if err != nil {
if name == "" {
name = DefaultSection
}
sec, _ = f.NewSection(name)
return sec
}
return sec
}
// SectionWithIndex assumes named section exists and returns a new section when not.
func (f *File) SectionWithIndex(name string, index int) *Section {
secs, err := f.SectionsByName(name)
if err != nil || len(secs) <= index {
// NOTE: It's OK here because the only possible error is empty section name,
// but if it's empty, this piece of code won't be executed.
newSec, _ := f.NewSection(name)
return newSec
}
return secs[index]
}
// Sections returns a list of Section stored in the current instance.
func (f *File) Sections() []*Section {
if f.BlockMode {
f.lock.RLock()
defer f.lock.RUnlock()
}
sections := make([]*Section, len(f.sectionList))
for i, name := range f.sectionList {
sections[i] = f.sections[name][f.sectionIndexes[i]]
}
return sections
}
// ChildSections returns a list of child sections of given section name.
func (f *File) ChildSections(name string) []*Section {
return f.Section(name).ChildSections()
}
// SectionStrings returns list of section names.
func (f *File) SectionStrings() []string {
list := make([]string, len(f.sectionList))
copy(list, f.sectionList)
return list
}
// DeleteSection deletes a section or all sections with given name.
func (f *File) DeleteSection(name string) {
secs, err := f.SectionsByName(name)
if err != nil {
return
}
for i := 0; i < len(secs); i++ {
// For non-unique sections, it is always needed to remove the first one so
// in the next iteration, the subsequent section continue having index 0.
// Ignoring the error as index 0 never returns an error.
_ = f.DeleteSectionWithIndex(name, 0)
}
}
// DeleteSectionWithIndex deletes a section with given name and index.
func (f *File) DeleteSectionWithIndex(name string, index int) error {
if !f.options.AllowNonUniqueSections && index != 0 {
return fmt.Errorf("delete section with non-zero index is only allowed when non-unique sections is enabled")
}
if len(name) == 0 {
name = DefaultSection
}
if f.options.Insensitive || f.options.InsensitiveSections {
name = strings.ToLower(name)
}
if f.BlockMode {
f.lock.Lock()
defer f.lock.Unlock()
}
// Count occurrences of the sections
occurrences := 0
sectionListCopy := make([]string, len(f.sectionList))
copy(sectionListCopy, f.sectionList)
for i, s := range sectionListCopy {
if s != name {
continue
}
if occurrences == index {
if len(f.sections[name]) <= 1 {
delete(f.sections, name) // The last one in the map
} else {
f.sections[name] = append(f.sections[name][:index], f.sections[name][index+1:]...)
}
// Fix section lists
f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...)
f.sectionIndexes = append(f.sectionIndexes[:i], f.sectionIndexes[i+1:]...)
} else if occurrences > index {
// Fix the indices of all following sections with this name.
f.sectionIndexes[i-1]--
}
occurrences++
}
return nil
}
func (f *File) reload(s dataSource) error {
r, err := s.ReadCloser()
if err != nil {
return err
}
defer r.Close()
return f.parse(r)
}
// Reload reloads and parses all data sources.
func (f *File) Reload() (err error) {
for _, s := range f.dataSources {
if err = f.reload(s); err != nil {
// In loose mode, we create an empty default section for nonexistent files.
if os.IsNotExist(err) && f.options.Loose {
_ = f.parse(bytes.NewBuffer(nil))
continue
}
return err
}
if f.options.ShortCircuit {
return nil
}
}
return nil
}
// Append appends one or more data sources and reloads automatically.
func (f *File) Append(source interface{}, others ...interface{}) error {
ds, err := parseDataSource(source)
if err != nil {
return err
}
f.dataSources = append(f.dataSources, ds)
for _, s := range others {
ds, err = parseDataSource(s)
if err != nil {
return err
}
f.dataSources = append(f.dataSources, ds)
}
return f.Reload()
}
func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) {
equalSign := DefaultFormatLeft + f.options.KeyValueDelimiterOnWrite + DefaultFormatRight
if PrettyFormat || PrettyEqual {
equalSign = fmt.Sprintf(" %s ", f.options.KeyValueDelimiterOnWrite)
}
// Use buffer to make sure target is safe until finish encoding.
buf := bytes.NewBuffer(nil)
lastSectionIdx := len(f.sectionList) - 1
for i, sname := range f.sectionList {
sec := f.SectionWithIndex(sname, f.sectionIndexes[i])
if len(sec.Comment) > 0 {
// Support multiline comments
lines := strings.Split(sec.Comment, LineBreak)
for i := range lines {
if lines[i][0] != '#' && lines[i][0] != ';' {
lines[i] = "; " + lines[i]
} else {
lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:])
}
if _, err := buf.WriteString(lines[i] + LineBreak); err != nil {
return nil, err
}
}
}
if i > 0 || DefaultHeader || (i == 0 && strings.ToUpper(sec.name) != DefaultSection) {
if _, err := buf.WriteString("[" + sname + "]" + LineBreak); err != nil {
return nil, err
}
} else {
// Write nothing if default section is empty
if len(sec.keyList) == 0 {
continue
}
}
isLastSection := i == lastSectionIdx
if sec.isRawSection {
if _, err := buf.WriteString(sec.rawBody); err != nil {
return nil, err
}
if PrettySection && !isLastSection {
// Put a line between sections
if _, err := buf.WriteString(LineBreak); err != nil {
return nil, err
}
}
continue
}
// Count and generate alignment length and buffer spaces using the
// longest key. Keys may be modified if they contain certain characters so
// we need to take that into account in our calculation.
alignLength := 0
if PrettyFormat {
for _, kname := range sec.keyList {
keyLength := len(kname)
// First case will surround key by ` and second by """
if strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters) {
keyLength += 2
} else if strings.Contains(kname, "`") {
keyLength += 6
}
if keyLength > alignLength {
alignLength = keyLength
}
}
}
alignSpaces := bytes.Repeat([]byte(" "), alignLength)
KeyList:
for _, kname := range sec.keyList {
key := sec.Key(kname)
if len(key.Comment) > 0 {
if len(indent) > 0 && sname != DefaultSection {
buf.WriteString(indent)
}
// Support multiline comments
lines := strings.Split(key.Comment, LineBreak)
for i := range lines {
if lines[i][0] != '#' && lines[i][0] != ';' {
lines[i] = "; " + strings.TrimSpace(lines[i])
} else {
lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:])
}
if _, err := buf.WriteString(lines[i] + LineBreak); err != nil {
return nil, err
}
}
}
if len(indent) > 0 && sname != DefaultSection {
buf.WriteString(indent)
}
switch {
case key.isAutoIncrement:
kname = "-"
case strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters):
kname = "`" + kname + "`"
case strings.Contains(kname, "`"):
kname = `"""` + kname + `"""`
}
writeKeyValue := func(val string) (bool, error) {
if _, err := buf.WriteString(kname); err != nil {
return false, err
}
if key.isBooleanType {
buf.WriteString(LineBreak)
return true, nil
}
// Write out alignment spaces before "=" sign
if PrettyFormat {
buf.Write(alignSpaces[:alignLength-len(kname)])
}
// In case key value contains "\n", "`", "\"", "#" or ";"
if strings.ContainsAny(val, "\n`") {
val = `"""` + val + `"""`
} else if !f.options.IgnoreInlineComment && strings.ContainsAny(val, "#;") {
val = "`" + val + "`"
} else if len(strings.TrimSpace(val)) != len(val) {
val = `"` + val + `"`
}
if _, err := buf.WriteString(equalSign + val + LineBreak); err != nil {
return false, err
}
return false, nil
}
shadows := key.ValueWithShadows()
if len(shadows) == 0 {
if _, err := writeKeyValue(""); err != nil {
return nil, err
}
}
for _, val := range shadows {
exitLoop, err := writeKeyValue(val)
if err != nil {
return nil, err
} else if exitLoop {
continue KeyList
}
}
for _, val := range key.nestedValues {
if _, err := buf.WriteString(indent + " " + val + LineBreak); err != nil {
return nil, err
}
}
}
if PrettySection && !isLastSection {
// Put a line between sections
if _, err := buf.WriteString(LineBreak); err != nil {
return nil, err
}
}
}
return buf, nil
}
// WriteToIndent writes content into io.Writer with given indention.
// If PrettyFormat has been set to be true,
// it will align "=" sign with spaces under each section.
func (f *File) WriteToIndent(w io.Writer, indent string) (int64, error) {
buf, err := f.writeToBuffer(indent)
if err != nil {
return 0, err
}
return buf.WriteTo(w)
}
// WriteTo writes file content into io.Writer.
func (f *File) WriteTo(w io.Writer) (int64, error) {
return f.WriteToIndent(w, "")
}
// SaveToIndent writes content to file system with given value indention.
func (f *File) SaveToIndent(filename, indent string) error {
// Note: Because we are truncating with os.Create,
// so it's safer to save to a temporary file location and rename after done.
buf, err := f.writeToBuffer(indent)
if err != nil {
return err
}
return ioutil.WriteFile(filename, buf.Bytes(), 0666)
}
// SaveTo writes content to file system.
func (f *File) SaveTo(filename string) error {
return f.SaveToIndent(filename, "")
}

24
vendor/github.com/go-ini/ini/helper.go generated vendored
View File

@@ -1,24 +0,0 @@
// Copyright 2019 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
func inSlice(str string, s []string) bool {
for _, v := range s {
if str == v {
return true
}
}
return false
}

176
vendor/github.com/go-ini/ini/ini.go generated vendored
View File

@@ -1,176 +0,0 @@
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// Package ini provides INI file read and write functionality in Go.
package ini
import (
"os"
"regexp"
"runtime"
"strings"
)
const (
// Maximum allowed depth when recursively substituing variable names.
depthValues = 99
)
var (
// DefaultSection is the name of default section. You can use this var or the string literal.
// In most of cases, an empty string is all you need to access the section.
DefaultSection = "DEFAULT"
// LineBreak is the delimiter to determine or compose a new line.
// This variable will be changed to "\r\n" automatically on Windows at package init time.
LineBreak = "\n"
// Variable regexp pattern: %(variable)s
varPattern = regexp.MustCompile(`%\(([^)]+)\)s`)
// DefaultHeader explicitly writes default section header.
DefaultHeader = false
// PrettySection indicates whether to put a line between sections.
PrettySection = true
// PrettyFormat indicates whether to align "=" sign with spaces to produce pretty output
// or reduce all possible spaces for compact format.
PrettyFormat = true
// PrettyEqual places spaces around "=" sign even when PrettyFormat is false.
PrettyEqual = false
// DefaultFormatLeft places custom spaces on the left when PrettyFormat and PrettyEqual are both disabled.
DefaultFormatLeft = ""
// DefaultFormatRight places custom spaces on the right when PrettyFormat and PrettyEqual are both disabled.
DefaultFormatRight = ""
)
var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test")
func init() {
if runtime.GOOS == "windows" && !inTest {
LineBreak = "\r\n"
}
}
// LoadOptions contains all customized options used for load data source(s).
type LoadOptions struct {
// Loose indicates whether the parser should ignore nonexistent files or return error.
Loose bool
// Insensitive indicates whether the parser forces all section and key names to lowercase.
Insensitive bool
// InsensitiveSections indicates whether the parser forces all section to lowercase.
InsensitiveSections bool
// InsensitiveKeys indicates whether the parser forces all key names to lowercase.
InsensitiveKeys bool
// IgnoreContinuation indicates whether to ignore continuation lines while parsing.
IgnoreContinuation bool
// IgnoreInlineComment indicates whether to ignore comments at the end of value and treat it as part of value.
IgnoreInlineComment bool
// SkipUnrecognizableLines indicates whether to skip unrecognizable lines that do not conform to key/value pairs.
SkipUnrecognizableLines bool
// ShortCircuit indicates whether to ignore other configuration sources after loaded the first available configuration source.
ShortCircuit bool
// AllowBooleanKeys indicates whether to allow boolean type keys or treat as value is missing.
// This type of keys are mostly used in my.cnf.
AllowBooleanKeys bool
// AllowShadows indicates whether to keep track of keys with same name under same section.
AllowShadows bool
// AllowNestedValues indicates whether to allow AWS-like nested values.
// Docs: http://docs.aws.amazon.com/cli/latest/topic/config-vars.html#nested-values
AllowNestedValues bool
// AllowPythonMultilineValues indicates whether to allow Python-like multi-line values.
// Docs: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure
// Relevant quote: Values can also span multiple lines, as long as they are indented deeper
// than the first line of the value.
AllowPythonMultilineValues bool
// SpaceBeforeInlineComment indicates whether to allow comment symbols (\# and \;) inside value.
// Docs: https://docs.python.org/2/library/configparser.html
// Quote: Comments may appear on their own in an otherwise empty line, or may be entered in lines holding values or section names.
// In the latter case, they need to be preceded by a whitespace character to be recognized as a comment.
SpaceBeforeInlineComment bool
// UnescapeValueDoubleQuotes indicates whether to unescape double quotes inside value to regular format
// when value is surrounded by double quotes, e.g. key="a \"value\"" => key=a "value"
UnescapeValueDoubleQuotes bool
// UnescapeValueCommentSymbols indicates to unescape comment symbols (\# and \;) inside value to regular format
// when value is NOT surrounded by any quotes.
// Note: UNSTABLE, behavior might change to only unescape inside double quotes but may noy necessary at all.
UnescapeValueCommentSymbols bool
// UnparseableSections stores a list of blocks that are allowed with raw content which do not otherwise
// conform to key/value pairs. Specify the names of those blocks here.
UnparseableSections []string
// KeyValueDelimiters is the sequence of delimiters that are used to separate key and value. By default, it is "=:".
KeyValueDelimiters string
// KeyValueDelimiterOnWrite is the delimiter that are used to separate key and value output. By default, it is "=".
KeyValueDelimiterOnWrite string
// ChildSectionDelimiter is the delimiter that is used to separate child sections. By default, it is ".".
ChildSectionDelimiter string
// PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes).
PreserveSurroundedQuote bool
// DebugFunc is called to collect debug information (currently only useful to debug parsing Python-style multiline values).
DebugFunc DebugFunc
// ReaderBufferSize is the buffer size of the reader in bytes.
ReaderBufferSize int
// AllowNonUniqueSections indicates whether to allow sections with the same name multiple times.
AllowNonUniqueSections bool
// AllowDuplicateShadowValues indicates whether values for shadowed keys should be deduplicated.
AllowDuplicateShadowValues bool
}
// DebugFunc is the type of function called to log parse events.
type DebugFunc func(message string)
// LoadSources allows caller to apply customized options for loading from data source(s).
func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) {
sources := make([]dataSource, len(others)+1)
sources[0], err = parseDataSource(source)
if err != nil {
return nil, err
}
for i := range others {
sources[i+1], err = parseDataSource(others[i])
if err != nil {
return nil, err
}
}
f := newFile(sources, opts)
if err = f.Reload(); err != nil {
return nil, err
}
return f, nil
}
// Load loads and parses from INI data sources.
// Arguments can be mixed of file name with string type, or raw data in []byte.
// It will return error if list contains nonexistent files.
func Load(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{}, source, others...)
}
// LooseLoad has exactly same functionality as Load function
// except it ignores nonexistent files instead of returning error.
func LooseLoad(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{Loose: true}, source, others...)
}
// InsensitiveLoad has exactly same functionality as Load function
// except it forces all section and key names to be lowercased.
func InsensitiveLoad(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{Insensitive: true}, source, others...)
}
// ShadowLoad has exactly same functionality as Load function
// except it allows have shadow keys.
func ShadowLoad(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{AllowShadows: true}, source, others...)
}

837
vendor/github.com/go-ini/ini/key.go generated vendored
View File

@@ -1,837 +0,0 @@
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
import (
"bytes"
"errors"
"fmt"
"strconv"
"strings"
"time"
)
// Key represents a key under a section.
type Key struct {
s *Section
Comment string
name string
value string
isAutoIncrement bool
isBooleanType bool
isShadow bool
shadows []*Key
nestedValues []string
}
// newKey simply return a key object with given values.
func newKey(s *Section, name, val string) *Key {
return &Key{
s: s,
name: name,
value: val,
}
}
func (k *Key) addShadow(val string) error {
if k.isShadow {
return errors.New("cannot add shadow to another shadow key")
} else if k.isAutoIncrement || k.isBooleanType {
return errors.New("cannot add shadow to auto-increment or boolean key")
}
if !k.s.f.options.AllowDuplicateShadowValues {
// Deduplicate shadows based on their values.
if k.value == val {
return nil
}
for i := range k.shadows {
if k.shadows[i].value == val {
return nil
}
}
}
shadow := newKey(k.s, k.name, val)
shadow.isShadow = true
k.shadows = append(k.shadows, shadow)
return nil
}
// AddShadow adds a new shadow key to itself.
func (k *Key) AddShadow(val string) error {
if !k.s.f.options.AllowShadows {
return errors.New("shadow key is not allowed")
}
return k.addShadow(val)
}
func (k *Key) addNestedValue(val string) error {
if k.isAutoIncrement || k.isBooleanType {
return errors.New("cannot add nested value to auto-increment or boolean key")
}
k.nestedValues = append(k.nestedValues, val)
return nil
}
// AddNestedValue adds a nested value to the key.
func (k *Key) AddNestedValue(val string) error {
if !k.s.f.options.AllowNestedValues {
return errors.New("nested value is not allowed")
}
return k.addNestedValue(val)
}
// ValueMapper represents a mapping function for values, e.g. os.ExpandEnv
type ValueMapper func(string) string
// Name returns name of key.
func (k *Key) Name() string {
return k.name
}
// Value returns raw value of key for performance purpose.
func (k *Key) Value() string {
return k.value
}
// ValueWithShadows returns raw values of key and its shadows if any. Shadow
// keys with empty values are ignored from the returned list.
func (k *Key) ValueWithShadows() []string {
if len(k.shadows) == 0 {
if k.value == "" {
return []string{}
}
return []string{k.value}
}
vals := make([]string, 0, len(k.shadows)+1)
if k.value != "" {
vals = append(vals, k.value)
}
for _, s := range k.shadows {
if s.value != "" {
vals = append(vals, s.value)
}
}
return vals
}
// NestedValues returns nested values stored in the key.
// It is possible returned value is nil if no nested values stored in the key.
func (k *Key) NestedValues() []string {
return k.nestedValues
}
// transformValue takes a raw value and transforms to its final string.
func (k *Key) transformValue(val string) string {
if k.s.f.ValueMapper != nil {
val = k.s.f.ValueMapper(val)
}
// Fail-fast if no indicate char found for recursive value
if !strings.Contains(val, "%") {
return val
}
for i := 0; i < depthValues; i++ {
vr := varPattern.FindString(val)
if len(vr) == 0 {
break
}
// Take off leading '%(' and trailing ')s'.
noption := vr[2 : len(vr)-2]
// Search in the same section.
// If not found or found the key itself, then search again in default section.
nk, err := k.s.GetKey(noption)
if err != nil || k == nk {
nk, _ = k.s.f.Section("").GetKey(noption)
if nk == nil {
// Stop when no results found in the default section,
// and returns the value as-is.
break
}
}
// Substitute by new value and take off leading '%(' and trailing ')s'.
val = strings.Replace(val, vr, nk.value, -1)
}
return val
}
// String returns string representation of value.
func (k *Key) String() string {
return k.transformValue(k.value)
}
// Validate accepts a validate function which can
// return modifed result as key value.
func (k *Key) Validate(fn func(string) string) string {
return fn(k.String())
}
// parseBool returns the boolean value represented by the string.
//
// It accepts 1, t, T, TRUE, true, True, YES, yes, Yes, y, ON, on, On,
// 0, f, F, FALSE, false, False, NO, no, No, n, OFF, off, Off.
// Any other value returns an error.
func parseBool(str string) (value bool, err error) {
switch str {
case "1", "t", "T", "true", "TRUE", "True", "YES", "yes", "Yes", "y", "ON", "on", "On":
return true, nil
case "0", "f", "F", "false", "FALSE", "False", "NO", "no", "No", "n", "OFF", "off", "Off":
return false, nil
}
return false, fmt.Errorf("parsing \"%s\": invalid syntax", str)
}
// Bool returns bool type value.
func (k *Key) Bool() (bool, error) {
return parseBool(k.String())
}
// Float64 returns float64 type value.
func (k *Key) Float64() (float64, error) {
return strconv.ParseFloat(k.String(), 64)
}
// Int returns int type value.
func (k *Key) Int() (int, error) {
v, err := strconv.ParseInt(k.String(), 0, 64)
return int(v), err
}
// Int64 returns int64 type value.
func (k *Key) Int64() (int64, error) {
return strconv.ParseInt(k.String(), 0, 64)
}
// Uint returns uint type valued.
func (k *Key) Uint() (uint, error) {
u, e := strconv.ParseUint(k.String(), 0, 64)
return uint(u), e
}
// Uint64 returns uint64 type value.
func (k *Key) Uint64() (uint64, error) {
return strconv.ParseUint(k.String(), 0, 64)
}
// Duration returns time.Duration type value.
func (k *Key) Duration() (time.Duration, error) {
return time.ParseDuration(k.String())
}
// TimeFormat parses with given format and returns time.Time type value.
func (k *Key) TimeFormat(format string) (time.Time, error) {
return time.Parse(format, k.String())
}
// Time parses with RFC3339 format and returns time.Time type value.
func (k *Key) Time() (time.Time, error) {
return k.TimeFormat(time.RFC3339)
}
// MustString returns default value if key value is empty.
func (k *Key) MustString(defaultVal string) string {
val := k.String()
if len(val) == 0 {
k.value = defaultVal
return defaultVal
}
return val
}
// MustBool always returns value without error,
// it returns false if error occurs.
func (k *Key) MustBool(defaultVal ...bool) bool {
val, err := k.Bool()
if len(defaultVal) > 0 && err != nil {
k.value = strconv.FormatBool(defaultVal[0])
return defaultVal[0]
}
return val
}
// MustFloat64 always returns value without error,
// it returns 0.0 if error occurs.
func (k *Key) MustFloat64(defaultVal ...float64) float64 {
val, err := k.Float64()
if len(defaultVal) > 0 && err != nil {
k.value = strconv.FormatFloat(defaultVal[0], 'f', -1, 64)
return defaultVal[0]
}
return val
}
// MustInt always returns value without error,
// it returns 0 if error occurs.
func (k *Key) MustInt(defaultVal ...int) int {
val, err := k.Int()
if len(defaultVal) > 0 && err != nil {
k.value = strconv.FormatInt(int64(defaultVal[0]), 10)
return defaultVal[0]
}
return val
}
// MustInt64 always returns value without error,
// it returns 0 if error occurs.
func (k *Key) MustInt64(defaultVal ...int64) int64 {
val, err := k.Int64()
if len(defaultVal) > 0 && err != nil {
k.value = strconv.FormatInt(defaultVal[0], 10)
return defaultVal[0]
}
return val
}
// MustUint always returns value without error,
// it returns 0 if error occurs.
func (k *Key) MustUint(defaultVal ...uint) uint {
val, err := k.Uint()
if len(defaultVal) > 0 && err != nil {
k.value = strconv.FormatUint(uint64(defaultVal[0]), 10)
return defaultVal[0]
}
return val
}
// MustUint64 always returns value without error,
// it returns 0 if error occurs.
func (k *Key) MustUint64(defaultVal ...uint64) uint64 {
val, err := k.Uint64()
if len(defaultVal) > 0 && err != nil {
k.value = strconv.FormatUint(defaultVal[0], 10)
return defaultVal[0]
}
return val
}
// MustDuration always returns value without error,
// it returns zero value if error occurs.
func (k *Key) MustDuration(defaultVal ...time.Duration) time.Duration {
val, err := k.Duration()
if len(defaultVal) > 0 && err != nil {
k.value = defaultVal[0].String()
return defaultVal[0]
}
return val
}
// MustTimeFormat always parses with given format and returns value without error,
// it returns zero value if error occurs.
func (k *Key) MustTimeFormat(format string, defaultVal ...time.Time) time.Time {
val, err := k.TimeFormat(format)
if len(defaultVal) > 0 && err != nil {
k.value = defaultVal[0].Format(format)
return defaultVal[0]
}
return val
}
// MustTime always parses with RFC3339 format and returns value without error,
// it returns zero value if error occurs.
func (k *Key) MustTime(defaultVal ...time.Time) time.Time {
return k.MustTimeFormat(time.RFC3339, defaultVal...)
}
// In always returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) In(defaultVal string, candidates []string) string {
val := k.String()
for _, cand := range candidates {
if val == cand {
return val
}
}
return defaultVal
}
// InFloat64 always returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) InFloat64(defaultVal float64, candidates []float64) float64 {
val := k.MustFloat64()
for _, cand := range candidates {
if val == cand {
return val
}
}
return defaultVal
}
// InInt always returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) InInt(defaultVal int, candidates []int) int {
val := k.MustInt()
for _, cand := range candidates {
if val == cand {
return val
}
}
return defaultVal
}
// InInt64 always returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) InInt64(defaultVal int64, candidates []int64) int64 {
val := k.MustInt64()
for _, cand := range candidates {
if val == cand {
return val
}
}
return defaultVal
}
// InUint always returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) InUint(defaultVal uint, candidates []uint) uint {
val := k.MustUint()
for _, cand := range candidates {
if val == cand {
return val
}
}
return defaultVal
}
// InUint64 always returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) InUint64(defaultVal uint64, candidates []uint64) uint64 {
val := k.MustUint64()
for _, cand := range candidates {
if val == cand {
return val
}
}
return defaultVal
}
// InTimeFormat always parses with given format and returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) InTimeFormat(format string, defaultVal time.Time, candidates []time.Time) time.Time {
val := k.MustTimeFormat(format)
for _, cand := range candidates {
if val == cand {
return val
}
}
return defaultVal
}
// InTime always parses with RFC3339 format and returns value without error,
// it returns default value if error occurs or doesn't fit into candidates.
func (k *Key) InTime(defaultVal time.Time, candidates []time.Time) time.Time {
return k.InTimeFormat(time.RFC3339, defaultVal, candidates)
}
// RangeFloat64 checks if value is in given range inclusively,
// and returns default value if it's not.
func (k *Key) RangeFloat64(defaultVal, min, max float64) float64 {
val := k.MustFloat64()
if val < min || val > max {
return defaultVal
}
return val
}
// RangeInt checks if value is in given range inclusively,
// and returns default value if it's not.
func (k *Key) RangeInt(defaultVal, min, max int) int {
val := k.MustInt()
if val < min || val > max {
return defaultVal
}
return val
}
// RangeInt64 checks if value is in given range inclusively,
// and returns default value if it's not.
func (k *Key) RangeInt64(defaultVal, min, max int64) int64 {
val := k.MustInt64()
if val < min || val > max {
return defaultVal
}
return val
}
// RangeTimeFormat checks if value with given format is in given range inclusively,
// and returns default value if it's not.
func (k *Key) RangeTimeFormat(format string, defaultVal, min, max time.Time) time.Time {
val := k.MustTimeFormat(format)
if val.Unix() < min.Unix() || val.Unix() > max.Unix() {
return defaultVal
}
return val
}
// RangeTime checks if value with RFC3339 format is in given range inclusively,
// and returns default value if it's not.
func (k *Key) RangeTime(defaultVal, min, max time.Time) time.Time {
return k.RangeTimeFormat(time.RFC3339, defaultVal, min, max)
}
// Strings returns list of string divided by given delimiter.
func (k *Key) Strings(delim string) []string {
str := k.String()
if len(str) == 0 {
return []string{}
}
runes := []rune(str)
vals := make([]string, 0, 2)
var buf bytes.Buffer
escape := false
idx := 0
for {
if escape {
escape = false
if runes[idx] != '\\' && !strings.HasPrefix(string(runes[idx:]), delim) {
buf.WriteRune('\\')
}
buf.WriteRune(runes[idx])
} else {
if runes[idx] == '\\' {
escape = true
} else if strings.HasPrefix(string(runes[idx:]), delim) {
idx += len(delim) - 1
vals = append(vals, strings.TrimSpace(buf.String()))
buf.Reset()
} else {
buf.WriteRune(runes[idx])
}
}
idx++
if idx == len(runes) {
break
}
}
if buf.Len() > 0 {
vals = append(vals, strings.TrimSpace(buf.String()))
}
return vals
}
// StringsWithShadows returns list of string divided by given delimiter.
// Shadows will also be appended if any.
func (k *Key) StringsWithShadows(delim string) []string {
vals := k.ValueWithShadows()
results := make([]string, 0, len(vals)*2)
for i := range vals {
if len(vals) == 0 {
continue
}
results = append(results, strings.Split(vals[i], delim)...)
}
for i := range results {
results[i] = k.transformValue(strings.TrimSpace(results[i]))
}
return results
}
// Float64s returns list of float64 divided by given delimiter. Any invalid input will be treated as zero value.
func (k *Key) Float64s(delim string) []float64 {
vals, _ := k.parseFloat64s(k.Strings(delim), true, false)
return vals
}
// Ints returns list of int divided by given delimiter. Any invalid input will be treated as zero value.
func (k *Key) Ints(delim string) []int {
vals, _ := k.parseInts(k.Strings(delim), true, false)
return vals
}
// Int64s returns list of int64 divided by given delimiter. Any invalid input will be treated as zero value.
func (k *Key) Int64s(delim string) []int64 {
vals, _ := k.parseInt64s(k.Strings(delim), true, false)
return vals
}
// Uints returns list of uint divided by given delimiter. Any invalid input will be treated as zero value.
func (k *Key) Uints(delim string) []uint {
vals, _ := k.parseUints(k.Strings(delim), true, false)
return vals
}
// Uint64s returns list of uint64 divided by given delimiter. Any invalid input will be treated as zero value.
func (k *Key) Uint64s(delim string) []uint64 {
vals, _ := k.parseUint64s(k.Strings(delim), true, false)
return vals
}
// Bools returns list of bool divided by given delimiter. Any invalid input will be treated as zero value.
func (k *Key) Bools(delim string) []bool {
vals, _ := k.parseBools(k.Strings(delim), true, false)
return vals
}
// TimesFormat parses with given format and returns list of time.Time divided by given delimiter.
// Any invalid input will be treated as zero value (0001-01-01 00:00:00 +0000 UTC).
func (k *Key) TimesFormat(format, delim string) []time.Time {
vals, _ := k.parseTimesFormat(format, k.Strings(delim), true, false)
return vals
}
// Times parses with RFC3339 format and returns list of time.Time divided by given delimiter.
// Any invalid input will be treated as zero value (0001-01-01 00:00:00 +0000 UTC).
func (k *Key) Times(delim string) []time.Time {
return k.TimesFormat(time.RFC3339, delim)
}
// ValidFloat64s returns list of float64 divided by given delimiter. If some value is not float, then
// it will not be included to result list.
func (k *Key) ValidFloat64s(delim string) []float64 {
vals, _ := k.parseFloat64s(k.Strings(delim), false, false)
return vals
}
// ValidInts returns list of int divided by given delimiter. If some value is not integer, then it will
// not be included to result list.
func (k *Key) ValidInts(delim string) []int {
vals, _ := k.parseInts(k.Strings(delim), false, false)
return vals
}
// ValidInt64s returns list of int64 divided by given delimiter. If some value is not 64-bit integer,
// then it will not be included to result list.
func (k *Key) ValidInt64s(delim string) []int64 {
vals, _ := k.parseInt64s(k.Strings(delim), false, false)
return vals
}
// ValidUints returns list of uint divided by given delimiter. If some value is not unsigned integer,
// then it will not be included to result list.
func (k *Key) ValidUints(delim string) []uint {
vals, _ := k.parseUints(k.Strings(delim), false, false)
return vals
}
// ValidUint64s returns list of uint64 divided by given delimiter. If some value is not 64-bit unsigned
// integer, then it will not be included to result list.
func (k *Key) ValidUint64s(delim string) []uint64 {
vals, _ := k.parseUint64s(k.Strings(delim), false, false)
return vals
}
// ValidBools returns list of bool divided by given delimiter. If some value is not 64-bit unsigned
// integer, then it will not be included to result list.
func (k *Key) ValidBools(delim string) []bool {
vals, _ := k.parseBools(k.Strings(delim), false, false)
return vals
}
// ValidTimesFormat parses with given format and returns list of time.Time divided by given delimiter.
func (k *Key) ValidTimesFormat(format, delim string) []time.Time {
vals, _ := k.parseTimesFormat(format, k.Strings(delim), false, false)
return vals
}
// ValidTimes parses with RFC3339 format and returns list of time.Time divided by given delimiter.
func (k *Key) ValidTimes(delim string) []time.Time {
return k.ValidTimesFormat(time.RFC3339, delim)
}
// StrictFloat64s returns list of float64 divided by given delimiter or error on first invalid input.
func (k *Key) StrictFloat64s(delim string) ([]float64, error) {
return k.parseFloat64s(k.Strings(delim), false, true)
}
// StrictInts returns list of int divided by given delimiter or error on first invalid input.
func (k *Key) StrictInts(delim string) ([]int, error) {
return k.parseInts(k.Strings(delim), false, true)
}
// StrictInt64s returns list of int64 divided by given delimiter or error on first invalid input.
func (k *Key) StrictInt64s(delim string) ([]int64, error) {
return k.parseInt64s(k.Strings(delim), false, true)
}
// StrictUints returns list of uint divided by given delimiter or error on first invalid input.
func (k *Key) StrictUints(delim string) ([]uint, error) {
return k.parseUints(k.Strings(delim), false, true)
}
// StrictUint64s returns list of uint64 divided by given delimiter or error on first invalid input.
func (k *Key) StrictUint64s(delim string) ([]uint64, error) {
return k.parseUint64s(k.Strings(delim), false, true)
}
// StrictBools returns list of bool divided by given delimiter or error on first invalid input.
func (k *Key) StrictBools(delim string) ([]bool, error) {
return k.parseBools(k.Strings(delim), false, true)
}
// StrictTimesFormat parses with given format and returns list of time.Time divided by given delimiter
// or error on first invalid input.
func (k *Key) StrictTimesFormat(format, delim string) ([]time.Time, error) {
return k.parseTimesFormat(format, k.Strings(delim), false, true)
}
// StrictTimes parses with RFC3339 format and returns list of time.Time divided by given delimiter
// or error on first invalid input.
func (k *Key) StrictTimes(delim string) ([]time.Time, error) {
return k.StrictTimesFormat(time.RFC3339, delim)
}
// parseBools transforms strings to bools.
func (k *Key) parseBools(strs []string, addInvalid, returnOnInvalid bool) ([]bool, error) {
vals := make([]bool, 0, len(strs))
parser := func(str string) (interface{}, error) {
val, err := parseBool(str)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(bool))
}
}
return vals, err
}
// parseFloat64s transforms strings to float64s.
func (k *Key) parseFloat64s(strs []string, addInvalid, returnOnInvalid bool) ([]float64, error) {
vals := make([]float64, 0, len(strs))
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseFloat(str, 64)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(float64))
}
}
return vals, err
}
// parseInts transforms strings to ints.
func (k *Key) parseInts(strs []string, addInvalid, returnOnInvalid bool) ([]int, error) {
vals := make([]int, 0, len(strs))
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseInt(str, 0, 64)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, int(val.(int64)))
}
}
return vals, err
}
// parseInt64s transforms strings to int64s.
func (k *Key) parseInt64s(strs []string, addInvalid, returnOnInvalid bool) ([]int64, error) {
vals := make([]int64, 0, len(strs))
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseInt(str, 0, 64)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(int64))
}
}
return vals, err
}
// parseUints transforms strings to uints.
func (k *Key) parseUints(strs []string, addInvalid, returnOnInvalid bool) ([]uint, error) {
vals := make([]uint, 0, len(strs))
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseUint(str, 0, 64)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, uint(val.(uint64)))
}
}
return vals, err
}
// parseUint64s transforms strings to uint64s.
func (k *Key) parseUint64s(strs []string, addInvalid, returnOnInvalid bool) ([]uint64, error) {
vals := make([]uint64, 0, len(strs))
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseUint(str, 0, 64)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(uint64))
}
}
return vals, err
}
type Parser func(str string) (interface{}, error)
// parseTimesFormat transforms strings to times in given format.
func (k *Key) parseTimesFormat(format string, strs []string, addInvalid, returnOnInvalid bool) ([]time.Time, error) {
vals := make([]time.Time, 0, len(strs))
parser := func(str string) (interface{}, error) {
val, err := time.Parse(format, str)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(time.Time))
}
}
return vals, err
}
// doParse transforms strings to different types
func (k *Key) doParse(strs []string, addInvalid, returnOnInvalid bool, parser Parser) ([]interface{}, error) {
vals := make([]interface{}, 0, len(strs))
for _, str := range strs {
val, err := parser(str)
if err != nil && returnOnInvalid {
return nil, err
}
if err == nil || addInvalid {
vals = append(vals, val)
}
}
return vals, nil
}
// SetValue changes key value.
func (k *Key) SetValue(v string) {
if k.s.f.BlockMode {
k.s.f.lock.Lock()
defer k.s.f.lock.Unlock()
}
k.value = v
k.s.keysHash[k.name] = v
}

520
vendor/github.com/go-ini/ini/parser.go generated vendored
View File

@@ -1,520 +0,0 @@
// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
import (
"bufio"
"bytes"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"unicode"
)
const minReaderBufferSize = 4096
var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`)
type parserOptions struct {
IgnoreContinuation bool
IgnoreInlineComment bool
AllowPythonMultilineValues bool
SpaceBeforeInlineComment bool
UnescapeValueDoubleQuotes bool
UnescapeValueCommentSymbols bool
PreserveSurroundedQuote bool
DebugFunc DebugFunc
ReaderBufferSize int
}
type parser struct {
buf *bufio.Reader
options parserOptions
isEOF bool
count int
comment *bytes.Buffer
}
func (p *parser) debug(format string, args ...interface{}) {
if p.options.DebugFunc != nil {
p.options.DebugFunc(fmt.Sprintf(format, args...))
}
}
func newParser(r io.Reader, opts parserOptions) *parser {
size := opts.ReaderBufferSize
if size < minReaderBufferSize {
size = minReaderBufferSize
}
return &parser{
buf: bufio.NewReaderSize(r, size),
options: opts,
count: 1,
comment: &bytes.Buffer{},
}
}
// BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format.
// http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding
func (p *parser) BOM() error {
mask, err := p.buf.Peek(2)
if err != nil && err != io.EOF {
return err
} else if len(mask) < 2 {
return nil
}
switch {
case mask[0] == 254 && mask[1] == 255:
fallthrough
case mask[0] == 255 && mask[1] == 254:
_, err = p.buf.Read(mask)
if err != nil {
return err
}
case mask[0] == 239 && mask[1] == 187:
mask, err := p.buf.Peek(3)
if err != nil && err != io.EOF {
return err
} else if len(mask) < 3 {
return nil
}
if mask[2] == 191 {
_, err = p.buf.Read(mask)
if err != nil {
return err
}
}
}
return nil
}
func (p *parser) readUntil(delim byte) ([]byte, error) {
data, err := p.buf.ReadBytes(delim)
if err != nil {
if err == io.EOF {
p.isEOF = true
} else {
return nil, err
}
}
return data, nil
}
func cleanComment(in []byte) ([]byte, bool) {
i := bytes.IndexAny(in, "#;")
if i == -1 {
return nil, false
}
return in[i:], true
}
func readKeyName(delimiters string, in []byte) (string, int, error) {
line := string(in)
// Check if key name surrounded by quotes.
var keyQuote string
if line[0] == '"' {
if len(line) > 6 && line[0:3] == `"""` {
keyQuote = `"""`
} else {
keyQuote = `"`
}
} else if line[0] == '`' {
keyQuote = "`"
}
// Get out key name
var endIdx int
if len(keyQuote) > 0 {
startIdx := len(keyQuote)
// FIXME: fail case -> """"""name"""=value
pos := strings.Index(line[startIdx:], keyQuote)
if pos == -1 {
return "", -1, fmt.Errorf("missing closing key quote: %s", line)
}
pos += startIdx
// Find key-value delimiter
i := strings.IndexAny(line[pos+startIdx:], delimiters)
if i < 0 {
return "", -1, ErrDelimiterNotFound{line}
}
endIdx = pos + i
return strings.TrimSpace(line[startIdx:pos]), endIdx + startIdx + 1, nil
}
endIdx = strings.IndexAny(line, delimiters)
if endIdx < 0 {
return "", -1, ErrDelimiterNotFound{line}
}
if endIdx == 0 {
return "", -1, ErrEmptyKeyName{line}
}
return strings.TrimSpace(line[0:endIdx]), endIdx + 1, nil
}
func (p *parser) readMultilines(line, val, valQuote string) (string, error) {
for {
data, err := p.readUntil('\n')
if err != nil {
return "", err
}
next := string(data)
pos := strings.LastIndex(next, valQuote)
if pos > -1 {
val += next[:pos]
comment, has := cleanComment([]byte(next[pos:]))
if has {
p.comment.Write(bytes.TrimSpace(comment))
}
break
}
val += next
if p.isEOF {
return "", fmt.Errorf("missing closing key quote from %q to %q", line, next)
}
}
return val, nil
}
func (p *parser) readContinuationLines(val string) (string, error) {
for {
data, err := p.readUntil('\n')
if err != nil {
return "", err
}
next := strings.TrimSpace(string(data))
if len(next) == 0 {
break
}
val += next
if val[len(val)-1] != '\\' {
break
}
val = val[:len(val)-1]
}
return val, nil
}
// hasSurroundedQuote check if and only if the first and last characters
// are quotes \" or \'.
// It returns false if any other parts also contain same kind of quotes.
func hasSurroundedQuote(in string, quote byte) bool {
return len(in) >= 2 && in[0] == quote && in[len(in)-1] == quote &&
strings.IndexByte(in[1:], quote) == len(in)-2
}
func (p *parser) readValue(in []byte, bufferSize int) (string, error) {
line := strings.TrimLeftFunc(string(in), unicode.IsSpace)
if len(line) == 0 {
if p.options.AllowPythonMultilineValues && len(in) > 0 && in[len(in)-1] == '\n' {
return p.readPythonMultilines(line, bufferSize)
}
return "", nil
}
var valQuote string
if len(line) > 3 && line[0:3] == `"""` {
valQuote = `"""`
} else if line[0] == '`' {
valQuote = "`"
} else if p.options.UnescapeValueDoubleQuotes && line[0] == '"' {
valQuote = `"`
}
if len(valQuote) > 0 {
startIdx := len(valQuote)
pos := strings.LastIndex(line[startIdx:], valQuote)
// Check for multi-line value
if pos == -1 {
return p.readMultilines(line, line[startIdx:], valQuote)
}
if p.options.UnescapeValueDoubleQuotes && valQuote == `"` {
return strings.Replace(line[startIdx:pos+startIdx], `\"`, `"`, -1), nil
}
return line[startIdx : pos+startIdx], nil
}
lastChar := line[len(line)-1]
// Won't be able to reach here if value only contains whitespace
line = strings.TrimSpace(line)
trimmedLastChar := line[len(line)-1]
// Check continuation lines when desired
if !p.options.IgnoreContinuation && trimmedLastChar == '\\' {
return p.readContinuationLines(line[:len(line)-1])
}
// Check if ignore inline comment
if !p.options.IgnoreInlineComment {
var i int
if p.options.SpaceBeforeInlineComment {
i = strings.Index(line, " #")
if i == -1 {
i = strings.Index(line, " ;")
}
} else {
i = strings.IndexAny(line, "#;")
}
if i > -1 {
p.comment.WriteString(line[i:])
line = strings.TrimSpace(line[:i])
}
}
// Trim single and double quotes
if (hasSurroundedQuote(line, '\'') ||
hasSurroundedQuote(line, '"')) && !p.options.PreserveSurroundedQuote {
line = line[1 : len(line)-1]
} else if len(valQuote) == 0 && p.options.UnescapeValueCommentSymbols {
line = strings.ReplaceAll(line, `\;`, ";")
line = strings.ReplaceAll(line, `\#`, "#")
} else if p.options.AllowPythonMultilineValues && lastChar == '\n' {
return p.readPythonMultilines(line, bufferSize)
}
return line, nil
}
func (p *parser) readPythonMultilines(line string, bufferSize int) (string, error) {
parserBufferPeekResult, _ := p.buf.Peek(bufferSize)
peekBuffer := bytes.NewBuffer(parserBufferPeekResult)
for {
peekData, peekErr := peekBuffer.ReadBytes('\n')
if peekErr != nil && peekErr != io.EOF {
p.debug("readPythonMultilines: failed to peek with error: %v", peekErr)
return "", peekErr
}
p.debug("readPythonMultilines: parsing %q", string(peekData))
peekMatches := pythonMultiline.FindStringSubmatch(string(peekData))
p.debug("readPythonMultilines: matched %d parts", len(peekMatches))
for n, v := range peekMatches {
p.debug(" %d: %q", n, v)
}
// Return if not a Python multiline value.
if len(peekMatches) != 3 {
p.debug("readPythonMultilines: end of value, got: %q", line)
return line, nil
}
// Advance the parser reader (buffer) in-sync with the peek buffer.
_, err := p.buf.Discard(len(peekData))
if err != nil {
p.debug("readPythonMultilines: failed to skip to the end, returning error")
return "", err
}
line += "\n" + peekMatches[0]
}
}
// parse parses data through an io.Reader.
func (f *File) parse(reader io.Reader) (err error) {
p := newParser(reader, parserOptions{
IgnoreContinuation: f.options.IgnoreContinuation,
IgnoreInlineComment: f.options.IgnoreInlineComment,
AllowPythonMultilineValues: f.options.AllowPythonMultilineValues,
SpaceBeforeInlineComment: f.options.SpaceBeforeInlineComment,
UnescapeValueDoubleQuotes: f.options.UnescapeValueDoubleQuotes,
UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols,
PreserveSurroundedQuote: f.options.PreserveSurroundedQuote,
DebugFunc: f.options.DebugFunc,
ReaderBufferSize: f.options.ReaderBufferSize,
})
if err = p.BOM(); err != nil {
return fmt.Errorf("BOM: %v", err)
}
// Ignore error because default section name is never empty string.
name := DefaultSection
if f.options.Insensitive || f.options.InsensitiveSections {
name = strings.ToLower(DefaultSection)
}
section, _ := f.NewSection(name)
// This "last" is not strictly equivalent to "previous one" if current key is not the first nested key
var isLastValueEmpty bool
var lastRegularKey *Key
var line []byte
var inUnparseableSection bool
// NOTE: Iterate and increase `currentPeekSize` until
// the size of the parser buffer is found.
// TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`.
parserBufferSize := 0
// NOTE: Peek 4kb at a time.
currentPeekSize := minReaderBufferSize
if f.options.AllowPythonMultilineValues {
for {
peekBytes, _ := p.buf.Peek(currentPeekSize)
peekBytesLength := len(peekBytes)
if parserBufferSize >= peekBytesLength {
break
}
currentPeekSize *= 2
parserBufferSize = peekBytesLength
}
}
for !p.isEOF {
line, err = p.readUntil('\n')
if err != nil {
return err
}
if f.options.AllowNestedValues &&
isLastValueEmpty && len(line) > 0 {
if line[0] == ' ' || line[0] == '\t' {
err = lastRegularKey.addNestedValue(string(bytes.TrimSpace(line)))
if err != nil {
return err
}
continue
}
}
line = bytes.TrimLeftFunc(line, unicode.IsSpace)
if len(line) == 0 {
continue
}
// Comments
if line[0] == '#' || line[0] == ';' {
// Note: we do not care ending line break,
// it is needed for adding second line,
// so just clean it once at the end when set to value.
p.comment.Write(line)
continue
}
// Section
if line[0] == '[' {
// Read to the next ']' (TODO: support quoted strings)
closeIdx := bytes.LastIndexByte(line, ']')
if closeIdx == -1 {
return fmt.Errorf("unclosed section: %s", line)
}
name := string(line[1:closeIdx])
section, err = f.NewSection(name)
if err != nil {
return err
}
comment, has := cleanComment(line[closeIdx+1:])
if has {
p.comment.Write(comment)
}
section.Comment = strings.TrimSpace(p.comment.String())
// Reset auto-counter and comments
p.comment.Reset()
p.count = 1
// Nested values can't span sections
isLastValueEmpty = false
inUnparseableSection = false
for i := range f.options.UnparseableSections {
if f.options.UnparseableSections[i] == name ||
((f.options.Insensitive || f.options.InsensitiveSections) && strings.EqualFold(f.options.UnparseableSections[i], name)) {
inUnparseableSection = true
continue
}
}
continue
}
if inUnparseableSection {
section.isRawSection = true
section.rawBody += string(line)
continue
}
kname, offset, err := readKeyName(f.options.KeyValueDelimiters, line)
if err != nil {
switch {
// Treat as boolean key when desired, and whole line is key name.
case IsErrDelimiterNotFound(err):
switch {
case f.options.AllowBooleanKeys:
kname, err := p.readValue(line, parserBufferSize)
if err != nil {
return err
}
key, err := section.NewBooleanKey(kname)
if err != nil {
return err
}
key.Comment = strings.TrimSpace(p.comment.String())
p.comment.Reset()
continue
case f.options.SkipUnrecognizableLines:
continue
}
case IsErrEmptyKeyName(err) && f.options.SkipUnrecognizableLines:
continue
}
return err
}
// Auto increment.
isAutoIncr := false
if kname == "-" {
isAutoIncr = true
kname = "#" + strconv.Itoa(p.count)
p.count++
}
value, err := p.readValue(line[offset:], parserBufferSize)
if err != nil {
return err
}
isLastValueEmpty = len(value) == 0
key, err := section.NewKey(kname, value)
if err != nil {
return err
}
key.isAutoIncrement = isAutoIncr
key.Comment = strings.TrimSpace(p.comment.String())
p.comment.Reset()
lastRegularKey = key
}
return nil
}

256
vendor/github.com/go-ini/ini/section.go generated vendored
View File

@@ -1,256 +0,0 @@
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
import (
"errors"
"fmt"
"strings"
)
// Section represents a config section.
type Section struct {
f *File
Comment string
name string
keys map[string]*Key
keyList []string
keysHash map[string]string
isRawSection bool
rawBody string
}
func newSection(f *File, name string) *Section {
return &Section{
f: f,
name: name,
keys: make(map[string]*Key),
keyList: make([]string, 0, 10),
keysHash: make(map[string]string),
}
}
// Name returns name of Section.
func (s *Section) Name() string {
return s.name
}
// Body returns rawBody of Section if the section was marked as unparseable.
// It still follows the other rules of the INI format surrounding leading/trailing whitespace.
func (s *Section) Body() string {
return strings.TrimSpace(s.rawBody)
}
// SetBody updates body content only if section is raw.
func (s *Section) SetBody(body string) {
if !s.isRawSection {
return
}
s.rawBody = body
}
// NewKey creates a new key to given section.
func (s *Section) NewKey(name, val string) (*Key, error) {
if len(name) == 0 {
return nil, errors.New("error creating new key: empty key name")
} else if s.f.options.Insensitive || s.f.options.InsensitiveKeys {
name = strings.ToLower(name)
}
if s.f.BlockMode {
s.f.lock.Lock()
defer s.f.lock.Unlock()
}
if inSlice(name, s.keyList) {
if s.f.options.AllowShadows {
if err := s.keys[name].addShadow(val); err != nil {
return nil, err
}
} else {
s.keys[name].value = val
s.keysHash[name] = val
}
return s.keys[name], nil
}
s.keyList = append(s.keyList, name)
s.keys[name] = newKey(s, name, val)
s.keysHash[name] = val
return s.keys[name], nil
}
// NewBooleanKey creates a new boolean type key to given section.
func (s *Section) NewBooleanKey(name string) (*Key, error) {
key, err := s.NewKey(name, "true")
if err != nil {
return nil, err
}
key.isBooleanType = true
return key, nil
}
// GetKey returns key in section by given name.
func (s *Section) GetKey(name string) (*Key, error) {
if s.f.BlockMode {
s.f.lock.RLock()
}
if s.f.options.Insensitive || s.f.options.InsensitiveKeys {
name = strings.ToLower(name)
}
key := s.keys[name]
if s.f.BlockMode {
s.f.lock.RUnlock()
}
if key == nil {
// Check if it is a child-section.
sname := s.name
for {
if i := strings.LastIndex(sname, s.f.options.ChildSectionDelimiter); i > -1 {
sname = sname[:i]
sec, err := s.f.GetSection(sname)
if err != nil {
continue
}
return sec.GetKey(name)
}
break
}
return nil, fmt.Errorf("error when getting key of section %q: key %q not exists", s.name, name)
}
return key, nil
}
// HasKey returns true if section contains a key with given name.
func (s *Section) HasKey(name string) bool {
key, _ := s.GetKey(name)
return key != nil
}
// Deprecated: Use "HasKey" instead.
func (s *Section) Haskey(name string) bool {
return s.HasKey(name)
}
// HasValue returns true if section contains given raw value.
func (s *Section) HasValue(value string) bool {
if s.f.BlockMode {
s.f.lock.RLock()
defer s.f.lock.RUnlock()
}
for _, k := range s.keys {
if value == k.value {
return true
}
}
return false
}
// Key assumes named Key exists in section and returns a zero-value when not.
func (s *Section) Key(name string) *Key {
key, err := s.GetKey(name)
if err != nil {
// It's OK here because the only possible error is empty key name,
// but if it's empty, this piece of code won't be executed.
key, _ = s.NewKey(name, "")
return key
}
return key
}
// Keys returns list of keys of section.
func (s *Section) Keys() []*Key {
keys := make([]*Key, len(s.keyList))
for i := range s.keyList {
keys[i] = s.Key(s.keyList[i])
}
return keys
}
// ParentKeys returns list of keys of parent section.
func (s *Section) ParentKeys() []*Key {
var parentKeys []*Key
sname := s.name
for {
if i := strings.LastIndex(sname, s.f.options.ChildSectionDelimiter); i > -1 {
sname = sname[:i]
sec, err := s.f.GetSection(sname)
if err != nil {
continue
}
parentKeys = append(parentKeys, sec.Keys()...)
} else {
break
}
}
return parentKeys
}
// KeyStrings returns list of key names of section.
func (s *Section) KeyStrings() []string {
list := make([]string, len(s.keyList))
copy(list, s.keyList)
return list
}
// KeysHash returns keys hash consisting of names and values.
func (s *Section) KeysHash() map[string]string {
if s.f.BlockMode {
s.f.lock.RLock()
defer s.f.lock.RUnlock()
}
hash := make(map[string]string, len(s.keysHash))
for key, value := range s.keysHash {
hash[key] = value
}
return hash
}
// DeleteKey deletes a key from section.
func (s *Section) DeleteKey(name string) {
if s.f.BlockMode {
s.f.lock.Lock()
defer s.f.lock.Unlock()
}
for i, k := range s.keyList {
if k == name {
s.keyList = append(s.keyList[:i], s.keyList[i+1:]...)
delete(s.keys, name)
delete(s.keysHash, name)
return
}
}
}
// ChildSections returns a list of child sections of current section.
// For example, "[parent.child1]" and "[parent.child12]" are child sections
// of section "[parent]".
func (s *Section) ChildSections() []*Section {
prefix := s.name + s.f.options.ChildSectionDelimiter
children := make([]*Section, 0, 3)
for _, name := range s.f.sectionList {
if strings.HasPrefix(name, prefix) {
children = append(children, s.f.sections[name]...)
}
}
return children
}

747
vendor/github.com/go-ini/ini/struct.go generated vendored
View File

@@ -1,747 +0,0 @@
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package ini
import (
"bytes"
"errors"
"fmt"
"reflect"
"strings"
"time"
"unicode"
)
// NameMapper represents a ini tag name mapper.
type NameMapper func(string) string
// Built-in name getters.
var (
// SnackCase converts to format SNACK_CASE.
SnackCase NameMapper = func(raw string) string {
newstr := make([]rune, 0, len(raw))
for i, chr := range raw {
if isUpper := 'A' <= chr && chr <= 'Z'; isUpper {
if i > 0 {
newstr = append(newstr, '_')
}
}
newstr = append(newstr, unicode.ToUpper(chr))
}
return string(newstr)
}
// TitleUnderscore converts to format title_underscore.
TitleUnderscore NameMapper = func(raw string) string {
newstr := make([]rune, 0, len(raw))
for i, chr := range raw {
if isUpper := 'A' <= chr && chr <= 'Z'; isUpper {
if i > 0 {
newstr = append(newstr, '_')
}
chr -= 'A' - 'a'
}
newstr = append(newstr, chr)
}
return string(newstr)
}
)
func (s *Section) parseFieldName(raw, actual string) string {
if len(actual) > 0 {
return actual
}
if s.f.NameMapper != nil {
return s.f.NameMapper(raw)
}
return raw
}
func parseDelim(actual string) string {
if len(actual) > 0 {
return actual
}
return ","
}
var reflectTime = reflect.TypeOf(time.Now()).Kind()
// setSliceWithProperType sets proper values to slice based on its type.
func setSliceWithProperType(key *Key, field reflect.Value, delim string, allowShadow, isStrict bool) error {
var strs []string
if allowShadow {
strs = key.StringsWithShadows(delim)
} else {
strs = key.Strings(delim)
}
numVals := len(strs)
if numVals == 0 {
return nil
}
var vals interface{}
var err error
sliceOf := field.Type().Elem().Kind()
switch sliceOf {
case reflect.String:
vals = strs
case reflect.Int:
vals, err = key.parseInts(strs, true, false)
case reflect.Int64:
vals, err = key.parseInt64s(strs, true, false)
case reflect.Uint:
vals, err = key.parseUints(strs, true, false)
case reflect.Uint64:
vals, err = key.parseUint64s(strs, true, false)
case reflect.Float64:
vals, err = key.parseFloat64s(strs, true, false)
case reflect.Bool:
vals, err = key.parseBools(strs, true, false)
case reflectTime:
vals, err = key.parseTimesFormat(time.RFC3339, strs, true, false)
default:
return fmt.Errorf("unsupported type '[]%s'", sliceOf)
}
if err != nil && isStrict {
return err
}
slice := reflect.MakeSlice(field.Type(), numVals, numVals)
for i := 0; i < numVals; i++ {
switch sliceOf {
case reflect.String:
slice.Index(i).Set(reflect.ValueOf(vals.([]string)[i]))
case reflect.Int:
slice.Index(i).Set(reflect.ValueOf(vals.([]int)[i]))
case reflect.Int64:
slice.Index(i).Set(reflect.ValueOf(vals.([]int64)[i]))
case reflect.Uint:
slice.Index(i).Set(reflect.ValueOf(vals.([]uint)[i]))
case reflect.Uint64:
slice.Index(i).Set(reflect.ValueOf(vals.([]uint64)[i]))
case reflect.Float64:
slice.Index(i).Set(reflect.ValueOf(vals.([]float64)[i]))
case reflect.Bool:
slice.Index(i).Set(reflect.ValueOf(vals.([]bool)[i]))
case reflectTime:
slice.Index(i).Set(reflect.ValueOf(vals.([]time.Time)[i]))
}
}
field.Set(slice)
return nil
}
func wrapStrictError(err error, isStrict bool) error {
if isStrict {
return err
}
return nil
}
// setWithProperType sets proper value to field based on its type,
// but it does not return error for failing parsing,
// because we want to use default value that is already assigned to struct.
func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim string, allowShadow, isStrict bool) error {
vt := t
isPtr := t.Kind() == reflect.Ptr
if isPtr {
vt = t.Elem()
}
switch vt.Kind() {
case reflect.String:
stringVal := key.String()
if isPtr {
field.Set(reflect.ValueOf(&stringVal))
} else if len(stringVal) > 0 {
field.SetString(key.String())
}
case reflect.Bool:
boolVal, err := key.Bool()
if err != nil {
return wrapStrictError(err, isStrict)
}
if isPtr {
field.Set(reflect.ValueOf(&boolVal))
} else {
field.SetBool(boolVal)
}
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
// ParseDuration will not return err for `0`, so check the type name
if vt.Name() == "Duration" {
durationVal, err := key.Duration()
if err != nil {
if intVal, err := key.Int64(); err == nil {
field.SetInt(intVal)
return nil
}
return wrapStrictError(err, isStrict)
}
if isPtr {
field.Set(reflect.ValueOf(&durationVal))
} else if int64(durationVal) > 0 {
field.Set(reflect.ValueOf(durationVal))
}
return nil
}
intVal, err := key.Int64()
if err != nil {
return wrapStrictError(err, isStrict)
}
if isPtr {
pv := reflect.New(t.Elem())
pv.Elem().SetInt(intVal)
field.Set(pv)
} else {
field.SetInt(intVal)
}
// byte is an alias for uint8, so supporting uint8 breaks support for byte
case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
durationVal, err := key.Duration()
// Skip zero value
if err == nil && uint64(durationVal) > 0 {
if isPtr {
field.Set(reflect.ValueOf(&durationVal))
} else {
field.Set(reflect.ValueOf(durationVal))
}
return nil
}
uintVal, err := key.Uint64()
if err != nil {
return wrapStrictError(err, isStrict)
}
if isPtr {
pv := reflect.New(t.Elem())
pv.Elem().SetUint(uintVal)
field.Set(pv)
} else {
field.SetUint(uintVal)
}
case reflect.Float32, reflect.Float64:
floatVal, err := key.Float64()
if err != nil {
return wrapStrictError(err, isStrict)
}
if isPtr {
pv := reflect.New(t.Elem())
pv.Elem().SetFloat(floatVal)
field.Set(pv)
} else {
field.SetFloat(floatVal)
}
case reflectTime:
timeVal, err := key.Time()
if err != nil {
return wrapStrictError(err, isStrict)
}
if isPtr {
field.Set(reflect.ValueOf(&timeVal))
} else {
field.Set(reflect.ValueOf(timeVal))
}
case reflect.Slice:
return setSliceWithProperType(key, field, delim, allowShadow, isStrict)
default:
return fmt.Errorf("unsupported type %q", t)
}
return nil
}
func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool, allowNonUnique bool, extends bool) {
opts := strings.SplitN(tag, ",", 5)
rawName = opts[0]
for _, opt := range opts[1:] {
omitEmpty = omitEmpty || (opt == "omitempty")
allowShadow = allowShadow || (opt == "allowshadow")
allowNonUnique = allowNonUnique || (opt == "nonunique")
extends = extends || (opt == "extends")
}
return rawName, omitEmpty, allowShadow, allowNonUnique, extends
}
// mapToField maps the given value to the matching field of the given section.
// The sectionIndex is the index (if non unique sections are enabled) to which the value should be added.
func (s *Section) mapToField(val reflect.Value, isStrict bool, sectionIndex int, sectionName string) error {
if val.Kind() == reflect.Ptr {
val = val.Elem()
}
typ := val.Type()
for i := 0; i < typ.NumField(); i++ {
field := val.Field(i)
tpField := typ.Field(i)
tag := tpField.Tag.Get("ini")
if tag == "-" {
continue
}
rawName, _, allowShadow, allowNonUnique, extends := parseTagOptions(tag)
fieldName := s.parseFieldName(tpField.Name, rawName)
if len(fieldName) == 0 || !field.CanSet() {
continue
}
isStruct := tpField.Type.Kind() == reflect.Struct
isStructPtr := tpField.Type.Kind() == reflect.Ptr && tpField.Type.Elem().Kind() == reflect.Struct
isAnonymousPtr := tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous
if isAnonymousPtr {
field.Set(reflect.New(tpField.Type.Elem()))
}
if extends && (isAnonymousPtr || (isStruct && tpField.Anonymous)) {
if isStructPtr && field.IsNil() {
field.Set(reflect.New(tpField.Type.Elem()))
}
fieldSection := s
if rawName != "" {
sectionName = s.name + s.f.options.ChildSectionDelimiter + rawName
if secs, err := s.f.SectionsByName(sectionName); err == nil && sectionIndex < len(secs) {
fieldSection = secs[sectionIndex]
}
}
if err := fieldSection.mapToField(field, isStrict, sectionIndex, sectionName); err != nil {
return fmt.Errorf("map to field %q: %v", fieldName, err)
}
} else if isAnonymousPtr || isStruct || isStructPtr {
if secs, err := s.f.SectionsByName(fieldName); err == nil {
if len(secs) <= sectionIndex {
return fmt.Errorf("there are not enough sections (%d <= %d) for the field %q", len(secs), sectionIndex, fieldName)
}
// Only set the field to non-nil struct value if we have a section for it.
// Otherwise, we end up with a non-nil struct ptr even though there is no data.
if isStructPtr && field.IsNil() {
field.Set(reflect.New(tpField.Type.Elem()))
}
if err = secs[sectionIndex].mapToField(field, isStrict, sectionIndex, fieldName); err != nil {
return fmt.Errorf("map to field %q: %v", fieldName, err)
}
continue
}
}
// Map non-unique sections
if allowNonUnique && tpField.Type.Kind() == reflect.Slice {
newField, err := s.mapToSlice(fieldName, field, isStrict)
if err != nil {
return fmt.Errorf("map to slice %q: %v", fieldName, err)
}
field.Set(newField)
continue
}
if key, err := s.GetKey(fieldName); err == nil {
delim := parseDelim(tpField.Tag.Get("delim"))
if err = setWithProperType(tpField.Type, key, field, delim, allowShadow, isStrict); err != nil {
return fmt.Errorf("set field %q: %v", fieldName, err)
}
}
}
return nil
}
// mapToSlice maps all sections with the same name and returns the new value.
// The type of the Value must be a slice.
func (s *Section) mapToSlice(secName string, val reflect.Value, isStrict bool) (reflect.Value, error) {
secs, err := s.f.SectionsByName(secName)
if err != nil {
return reflect.Value{}, err
}
typ := val.Type().Elem()
for i, sec := range secs {
elem := reflect.New(typ)
if err = sec.mapToField(elem, isStrict, i, sec.name); err != nil {
return reflect.Value{}, fmt.Errorf("map to field from section %q: %v", secName, err)
}
val = reflect.Append(val, elem.Elem())
}
return val, nil
}
// mapTo maps a section to object v.
func (s *Section) mapTo(v interface{}, isStrict bool) error {
typ := reflect.TypeOf(v)
val := reflect.ValueOf(v)
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
val = val.Elem()
} else {
return errors.New("not a pointer to a struct")
}
if typ.Kind() == reflect.Slice {
newField, err := s.mapToSlice(s.name, val, isStrict)
if err != nil {
return err
}
val.Set(newField)
return nil
}
return s.mapToField(val, isStrict, 0, s.name)
}
// MapTo maps section to given struct.
func (s *Section) MapTo(v interface{}) error {
return s.mapTo(v, false)
}
// StrictMapTo maps section to given struct in strict mode,
// which returns all possible error including value parsing error.
func (s *Section) StrictMapTo(v interface{}) error {
return s.mapTo(v, true)
}
// MapTo maps file to given struct.
func (f *File) MapTo(v interface{}) error {
return f.Section("").MapTo(v)
}
// StrictMapTo maps file to given struct in strict mode,
// which returns all possible error including value parsing error.
func (f *File) StrictMapTo(v interface{}) error {
return f.Section("").StrictMapTo(v)
}
// MapToWithMapper maps data sources to given struct with name mapper.
func MapToWithMapper(v interface{}, mapper NameMapper, source interface{}, others ...interface{}) error {
cfg, err := Load(source, others...)
if err != nil {
return err
}
cfg.NameMapper = mapper
return cfg.MapTo(v)
}
// StrictMapToWithMapper maps data sources to given struct with name mapper in strict mode,
// which returns all possible error including value parsing error.
func StrictMapToWithMapper(v interface{}, mapper NameMapper, source interface{}, others ...interface{}) error {
cfg, err := Load(source, others...)
if err != nil {
return err
}
cfg.NameMapper = mapper
return cfg.StrictMapTo(v)
}
// MapTo maps data sources to given struct.
func MapTo(v, source interface{}, others ...interface{}) error {
return MapToWithMapper(v, nil, source, others...)
}
// StrictMapTo maps data sources to given struct in strict mode,
// which returns all possible error including value parsing error.
func StrictMapTo(v, source interface{}, others ...interface{}) error {
return StrictMapToWithMapper(v, nil, source, others...)
}
// reflectSliceWithProperType does the opposite thing as setSliceWithProperType.
func reflectSliceWithProperType(key *Key, field reflect.Value, delim string, allowShadow bool) error {
slice := field.Slice(0, field.Len())
if field.Len() == 0 {
return nil
}
sliceOf := field.Type().Elem().Kind()
if allowShadow {
var keyWithShadows *Key
for i := 0; i < field.Len(); i++ {
var val string
switch sliceOf {
case reflect.String:
val = slice.Index(i).String()
case reflect.Int, reflect.Int64:
val = fmt.Sprint(slice.Index(i).Int())
case reflect.Uint, reflect.Uint64:
val = fmt.Sprint(slice.Index(i).Uint())
case reflect.Float64:
val = fmt.Sprint(slice.Index(i).Float())
case reflect.Bool:
val = fmt.Sprint(slice.Index(i).Bool())
case reflectTime:
val = slice.Index(i).Interface().(time.Time).Format(time.RFC3339)
default:
return fmt.Errorf("unsupported type '[]%s'", sliceOf)
}
if i == 0 {
keyWithShadows = newKey(key.s, key.name, val)
} else {
_ = keyWithShadows.AddShadow(val)
}
}
*key = *keyWithShadows
return nil
}
var buf bytes.Buffer
for i := 0; i < field.Len(); i++ {
switch sliceOf {
case reflect.String:
buf.WriteString(slice.Index(i).String())
case reflect.Int, reflect.Int64:
buf.WriteString(fmt.Sprint(slice.Index(i).Int()))
case reflect.Uint, reflect.Uint64:
buf.WriteString(fmt.Sprint(slice.Index(i).Uint()))
case reflect.Float64:
buf.WriteString(fmt.Sprint(slice.Index(i).Float()))
case reflect.Bool:
buf.WriteString(fmt.Sprint(slice.Index(i).Bool()))
case reflectTime:
buf.WriteString(slice.Index(i).Interface().(time.Time).Format(time.RFC3339))
default:
return fmt.Errorf("unsupported type '[]%s'", sliceOf)
}
buf.WriteString(delim)
}
key.SetValue(buf.String()[:buf.Len()-len(delim)])
return nil
}
// reflectWithProperType does the opposite thing as setWithProperType.
func reflectWithProperType(t reflect.Type, key *Key, field reflect.Value, delim string, allowShadow bool) error {
switch t.Kind() {
case reflect.String:
key.SetValue(field.String())
case reflect.Bool:
key.SetValue(fmt.Sprint(field.Bool()))
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
key.SetValue(fmt.Sprint(field.Int()))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
key.SetValue(fmt.Sprint(field.Uint()))
case reflect.Float32, reflect.Float64:
key.SetValue(fmt.Sprint(field.Float()))
case reflectTime:
key.SetValue(fmt.Sprint(field.Interface().(time.Time).Format(time.RFC3339)))
case reflect.Slice:
return reflectSliceWithProperType(key, field, delim, allowShadow)
case reflect.Ptr:
if !field.IsNil() {
return reflectWithProperType(t.Elem(), key, field.Elem(), delim, allowShadow)
}
default:
return fmt.Errorf("unsupported type %q", t)
}
return nil
}
// CR: copied from encoding/json/encode.go with modifications of time.Time support.
// TODO: add more test coverage.
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
case reflectTime:
t, ok := v.Interface().(time.Time)
return ok && t.IsZero()
}
return false
}
// StructReflector is the interface implemented by struct types that can extract themselves into INI objects.
type StructReflector interface {
ReflectINIStruct(*File) error
}
func (s *Section) reflectFrom(val reflect.Value) error {
if val.Kind() == reflect.Ptr {
val = val.Elem()
}
typ := val.Type()
for i := 0; i < typ.NumField(); i++ {
if !val.Field(i).CanInterface() {
continue
}
field := val.Field(i)
tpField := typ.Field(i)
tag := tpField.Tag.Get("ini")
if tag == "-" {
continue
}
rawName, omitEmpty, allowShadow, allowNonUnique, extends := parseTagOptions(tag)
if omitEmpty && isEmptyValue(field) {
continue
}
if r, ok := field.Interface().(StructReflector); ok {
return r.ReflectINIStruct(s.f)
}
fieldName := s.parseFieldName(tpField.Name, rawName)
if len(fieldName) == 0 || !field.CanSet() {
continue
}
if extends && tpField.Anonymous && (tpField.Type.Kind() == reflect.Ptr || tpField.Type.Kind() == reflect.Struct) {
if err := s.reflectFrom(field); err != nil {
return fmt.Errorf("reflect from field %q: %v", fieldName, err)
}
continue
}
if (tpField.Type.Kind() == reflect.Ptr && tpField.Type.Elem().Kind() == reflect.Struct) ||
(tpField.Type.Kind() == reflect.Struct && tpField.Type.Name() != "Time") {
// Note: The only error here is section doesn't exist.
sec, err := s.f.GetSection(fieldName)
if err != nil {
// Note: fieldName can never be empty here, ignore error.
sec, _ = s.f.NewSection(fieldName)
}
// Add comment from comment tag
if len(sec.Comment) == 0 {
sec.Comment = tpField.Tag.Get("comment")
}
if err = sec.reflectFrom(field); err != nil {
return fmt.Errorf("reflect from field %q: %v", fieldName, err)
}
continue
}
if allowNonUnique && tpField.Type.Kind() == reflect.Slice {
slice := field.Slice(0, field.Len())
if field.Len() == 0 {
return nil
}
sliceOf := field.Type().Elem().Kind()
for i := 0; i < field.Len(); i++ {
if sliceOf != reflect.Struct && sliceOf != reflect.Ptr {
return fmt.Errorf("field %q is not a slice of pointer or struct", fieldName)
}
sec, err := s.f.NewSection(fieldName)
if err != nil {
return err
}
// Add comment from comment tag
if len(sec.Comment) == 0 {
sec.Comment = tpField.Tag.Get("comment")
}
if err := sec.reflectFrom(slice.Index(i)); err != nil {
return fmt.Errorf("reflect from field %q: %v", fieldName, err)
}
}
continue
}
// Note: Same reason as section.
key, err := s.GetKey(fieldName)
if err != nil {
key, _ = s.NewKey(fieldName, "")
}
// Add comment from comment tag
if len(key.Comment) == 0 {
key.Comment = tpField.Tag.Get("comment")
}
delim := parseDelim(tpField.Tag.Get("delim"))
if err = reflectWithProperType(tpField.Type, key, field, delim, allowShadow); err != nil {
return fmt.Errorf("reflect field %q: %v", fieldName, err)
}
}
return nil
}
// ReflectFrom reflects section from given struct. It overwrites existing ones.
func (s *Section) ReflectFrom(v interface{}) error {
typ := reflect.TypeOf(v)
val := reflect.ValueOf(v)
if s.name != DefaultSection && s.f.options.AllowNonUniqueSections &&
(typ.Kind() == reflect.Slice || typ.Kind() == reflect.Ptr) {
// Clear sections to make sure none exists before adding the new ones
s.f.DeleteSection(s.name)
if typ.Kind() == reflect.Ptr {
sec, err := s.f.NewSection(s.name)
if err != nil {
return err
}
return sec.reflectFrom(val.Elem())
}
slice := val.Slice(0, val.Len())
sliceOf := val.Type().Elem().Kind()
if sliceOf != reflect.Ptr {
return fmt.Errorf("not a slice of pointers")
}
for i := 0; i < slice.Len(); i++ {
sec, err := s.f.NewSection(s.name)
if err != nil {
return err
}
err = sec.reflectFrom(slice.Index(i))
if err != nil {
return fmt.Errorf("reflect from %dth field: %v", i, err)
}
}
return nil
}
if typ.Kind() == reflect.Ptr {
val = val.Elem()
} else {
return errors.New("not a pointer to a struct")
}
return s.reflectFrom(val)
}
// ReflectFrom reflects file from given struct.
func (f *File) ReflectFrom(v interface{}) error {
return f.Section("").ReflectFrom(v)
}
// ReflectFromWithMapper reflects data sources from given struct with name mapper.
func ReflectFromWithMapper(cfg *File, v interface{}, mapper NameMapper) error {
cfg.NameMapper = mapper
return cfg.ReflectFrom(v)
}
// ReflectFrom reflects data sources from given struct.
func ReflectFrom(cfg *File, v interface{}) error {
return ReflectFromWithMapper(cfg, v, nil)
}

103
vendor/github.com/mattn/go-runewidth/runewidth.go generated vendored
View File

@@ -3,6 +3,7 @@ package runewidth
import (
"os"
"strings"
"unicode/utf8"
"github.com/clipperhouse/uax29/v2/graphemes"
)
@@ -23,10 +24,48 @@ var (
}
)
var (
zerowidth table // combining + nonprint merged for faster zero-width lookup
widewidth table // ambiguous + doublewidth merged for EA path
)
func init() {
zerowidth = mergeIntervals(combining, nonprint)
widewidth = mergeIntervals(ambiguous, doublewidth)
handleEnv()
}
func mergeIntervals(t1, t2 table) table {
merged := make(table, 0, len(t1)+len(t2))
i, j := 0, 0
for i < len(t1) && j < len(t2) {
if t1[i].first <= t2[j].first {
merged = append(merged, t1[i])
i++
} else {
merged = append(merged, t2[j])
j++
}
}
merged = append(merged, t1[i:]...)
merged = append(merged, t2[j:]...)
if len(merged) == 0 {
return merged
}
result := merged[:1]
for _, iv := range merged[1:] {
last := &result[len(result)-1]
if iv.first <= last.last+1 {
if iv.last > last.last {
last.last = iv.last
}
} else {
result = append(result, iv)
}
}
return result
}
func handleEnv() {
env := os.Getenv("RUNEWIDTH_EASTASIAN")
if env == "" {
@@ -51,15 +90,6 @@ type interval struct {
type table []interval
func inTables(r rune, ts ...table) bool {
for _, t := range ts {
if inTable(r, t) {
return true
}
}
return false
}
func inTable(r rune, t table) bool {
if r < t[0].first {
return false
@@ -130,9 +160,7 @@ func (c *Condition) RuneWidth(r rune) int {
return 0
case r < 0x300:
return 1
case inTable(r, narrow):
return 1
case inTables(r, nonprint, combining):
case inTable(r, zerowidth):
return 0
case inTable(r, doublewidth):
return 2
@@ -141,13 +169,13 @@ func (c *Condition) RuneWidth(r rune) int {
}
} else {
switch {
case inTables(r, nonprint, combining):
case inTable(r, zerowidth):
return 0
case inTable(r, narrow):
return 1
case inTables(r, ambiguous, doublewidth):
case inTable(r, widewidth):
return 2
case !c.StrictEmojiNeutral && inTables(r, ambiguous, emoji, narrow):
case !c.StrictEmojiNeutral && inTable(r, emoji):
return 2
default:
return 1
@@ -178,6 +206,22 @@ func (c *Condition) CreateLUT() {
// StringWidth return width as you can see
func (c *Condition) StringWidth(s string) (width int) {
if len(s) > 0 && len(s) <= utf8.UTFMax {
r, size := utf8.DecodeRuneInString(s)
if size == len(s) {
return c.RuneWidth(r)
}
}
// ASCII fast path: no grapheme clustering needed for pure ASCII
if isAllASCII(s) {
for i := 0; i < len(s); i++ {
b := s[i]
if b >= 0x20 && b != 0x7F {
width++
}
}
return
}
g := graphemes.FromString(s)
for g.Next() {
var chWidth int
@@ -192,6 +236,15 @@ func (c *Condition) StringWidth(s string) (width int) {
return
}
func isAllASCII(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= 0x80 {
return false
}
}
return true
}
// Truncate return string truncated with w cells
func (c *Condition) Truncate(s string, w int, tail string) string {
if c.StringWidth(s) <= w {
@@ -257,24 +310,25 @@ func (c *Condition) TruncateLeft(s string, w int, prefix string) string {
// Wrap return string wrapped with w cells
func (c *Condition) Wrap(s string, w int) string {
width := 0
out := ""
var out strings.Builder
out.Grow(len(s) + len(s)/w + 1)
for _, r := range s {
cw := c.RuneWidth(r)
if r == '\n' {
out += string(r)
out.WriteRune(r)
width = 0
continue
} else if width+cw > w {
out += "\n"
out.WriteByte('\n')
width = 0
out += string(r)
out.WriteRune(r)
width += cw
continue
}
out += string(r)
out.WriteRune(r)
width += cw
}
return out
return out.String()
}
// FillLeft return string filled in left by spaces in w cells
@@ -313,7 +367,12 @@ func RuneWidth(r rune) int {
// IsAmbiguousWidth returns whether is ambiguous width or not.
func IsAmbiguousWidth(r rune) bool {
return inTables(r, private, ambiguous)
return inTable(r, private) || inTable(r, ambiguous)
}
// IsCombiningWidth returns whether is combining width or not.
func IsCombiningWidth(r rune) bool {
return inTable(r, combining)
}
// IsNeutralWidth returns whether is neutral width or not.

650
vendor/github.com/mattn/go-runewidth/runewidth_table.go generated vendored
View File

@@ -5,47 +5,50 @@ package runewidth
var combining = table{
{0x0300, 0x036F}, {0x0483, 0x0489}, {0x07EB, 0x07F3},
{0x0C00, 0x0C00}, {0x0C04, 0x0C04}, {0x0CF3, 0x0CF3},
{0x0D00, 0x0D01}, {0x135D, 0x135F}, {0x1A7F, 0x1A7F},
{0x1AB0, 0x1ACE}, {0x1B6B, 0x1B73}, {0x1DC0, 0x1DFF},
{0x0D00, 0x0D01}, {0x135D, 0x135F}, {0x180B, 0x180D},
{0x180F, 0x180F}, {0x1A7F, 0x1A7F}, {0x1AB0, 0x1ADD},
{0x1AE0, 0x1AEB}, {0x1B6B, 0x1B73}, {0x1DC0, 0x1DFF},
{0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2DE0, 0x2DFF},
{0x3099, 0x309A}, {0xA66F, 0xA672}, {0xA674, 0xA67D},
{0xA69E, 0xA69F}, {0xA6F0, 0xA6F1}, {0xA8E0, 0xA8F1},
{0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, {0x10376, 0x1037A},
{0x10EAB, 0x10EAC}, {0x10F46, 0x10F50}, {0x10F82, 0x10F85},
{0x11300, 0x11301}, {0x1133B, 0x1133C}, {0x11366, 0x1136C},
{0x11370, 0x11374}, {0x16AF0, 0x16AF4}, {0x1CF00, 0x1CF2D},
{0x1CF30, 0x1CF46}, {0x1D165, 0x1D169}, {0x1D16D, 0x1D172},
{0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD},
{0x1D242, 0x1D244}, {0x1E000, 0x1E006}, {0x1E008, 0x1E018},
{0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E026, 0x1E02A},
{0x1E08F, 0x1E08F}, {0x1E8D0, 0x1E8D6},
{0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0x101FD, 0x101FD},
{0x10376, 0x1037A}, {0x10EAB, 0x10EAC}, {0x10F46, 0x10F50},
{0x10F82, 0x10F85}, {0x11300, 0x11301}, {0x1133B, 0x1133C},
{0x11366, 0x1136C}, {0x11370, 0x11374}, {0x16AF0, 0x16AF4},
{0x1CF00, 0x1CF2D}, {0x1CF30, 0x1CF46}, {0x1D165, 0x1D169},
{0x1D16D, 0x1D172}, {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B},
{0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, {0x1E000, 0x1E006},
{0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024},
{0x1E026, 0x1E02A}, {0x1E08F, 0x1E08F}, {0x1E8D0, 0x1E8D6},
{0xE0100, 0xE01EF},
}
var doublewidth = table{
{0x1100, 0x115F}, {0x231A, 0x231B}, {0x2329, 0x232A},
{0x23E9, 0x23EC}, {0x23F0, 0x23F0}, {0x23F3, 0x23F3},
{0x25FD, 0x25FE}, {0x2614, 0x2615}, {0x2648, 0x2653},
{0x267F, 0x267F}, {0x2693, 0x2693}, {0x26A1, 0x26A1},
{0x26AA, 0x26AB}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5},
{0x26CE, 0x26CE}, {0x26D4, 0x26D4}, {0x26EA, 0x26EA},
{0x26F2, 0x26F3}, {0x26F5, 0x26F5}, {0x26FA, 0x26FA},
{0x26FD, 0x26FD}, {0x2705, 0x2705}, {0x270A, 0x270B},
{0x2728, 0x2728}, {0x274C, 0x274C}, {0x274E, 0x274E},
{0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797},
{0x27B0, 0x27B0}, {0x27BF, 0x27BF}, {0x2B1B, 0x2B1C},
{0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x2E80, 0x2E99},
{0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x303E},
{0x3041, 0x3096}, {0x3099, 0x30FF}, {0x3105, 0x312F},
{0x3131, 0x318E}, {0x3190, 0x31E3}, {0x31EF, 0x321E},
{0x3220, 0x3247}, {0x3250, 0x4DBF}, {0x4E00, 0xA48C},
{0xA490, 0xA4C6}, {0xA960, 0xA97C}, {0xAC00, 0xD7A3},
{0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52},
{0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, {0xFF01, 0xFF60},
{0xFFE0, 0xFFE6}, {0x16FE0, 0x16FE4}, {0x16FF0, 0x16FF1},
{0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08},
{0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE},
{0x1B000, 0x1B122}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152},
{0x1B155, 0x1B155}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB},
{0x25FD, 0x25FE}, {0x2614, 0x2615}, {0x2630, 0x2637},
{0x2648, 0x2653}, {0x267F, 0x267F}, {0x268A, 0x268F},
{0x2693, 0x2693}, {0x26A1, 0x26A1}, {0x26AA, 0x26AB},
{0x26BD, 0x26BE}, {0x26C4, 0x26C5}, {0x26CE, 0x26CE},
{0x26D4, 0x26D4}, {0x26EA, 0x26EA}, {0x26F2, 0x26F3},
{0x26F5, 0x26F5}, {0x26FA, 0x26FA}, {0x26FD, 0x26FD},
{0x2705, 0x2705}, {0x270A, 0x270B}, {0x2728, 0x2728},
{0x274C, 0x274C}, {0x274E, 0x274E}, {0x2753, 0x2755},
{0x2757, 0x2757}, {0x2795, 0x2797}, {0x27B0, 0x27B0},
{0x27BF, 0x27BF}, {0x2B1B, 0x2B1C}, {0x2B50, 0x2B50},
{0x2B55, 0x2B55}, {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3},
{0x2F00, 0x2FD5}, {0x2FF0, 0x303E}, {0x3041, 0x3096},
{0x3099, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E},
{0x3190, 0x31E5}, {0x31EF, 0x321E}, {0x3220, 0x3247},
{0x3250, 0xA48C}, {0xA490, 0xA4C6}, {0xA960, 0xA97C},
{0xAC00, 0xD7A3}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19},
{0xFE30, 0xFE52}, {0xFE54, 0xFE66}, {0xFE68, 0xFE6B},
{0xFF01, 0xFF60}, {0xFFE0, 0xFFE6}, {0x16FE0, 0x16FE4},
{0x16FF0, 0x16FF6}, {0x17000, 0x18CD5}, {0x18CFF, 0x18D1E},
{0x18D80, 0x18DF2}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB},
{0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132},
{0x1B150, 0x1B152}, {0x1B155, 0x1B155}, {0x1B164, 0x1B167},
{0x1B170, 0x1B2FB}, {0x1D300, 0x1D356}, {0x1D360, 0x1D376},
{0x1F004, 0x1F004}, {0x1F0CF, 0x1F0CF}, {0x1F18E, 0x1F18E},
{0x1F191, 0x1F19A}, {0x1F200, 0x1F202}, {0x1F210, 0x1F23B},
{0x1F240, 0x1F248}, {0x1F250, 0x1F251}, {0x1F260, 0x1F265},
@@ -56,12 +59,12 @@ var doublewidth = table{
{0x1F54B, 0x1F54E}, {0x1F550, 0x1F567}, {0x1F57A, 0x1F57A},
{0x1F595, 0x1F596}, {0x1F5A4, 0x1F5A4}, {0x1F5FB, 0x1F64F},
{0x1F680, 0x1F6C5}, {0x1F6CC, 0x1F6CC}, {0x1F6D0, 0x1F6D2},
{0x1F6D5, 0x1F6D7}, {0x1F6DC, 0x1F6DF}, {0x1F6EB, 0x1F6EC},
{0x1F6D5, 0x1F6D8}, {0x1F6DC, 0x1F6DF}, {0x1F6EB, 0x1F6EC},
{0x1F6F4, 0x1F6FC}, {0x1F7E0, 0x1F7EB}, {0x1F7F0, 0x1F7F0},
{0x1F90C, 0x1F93A}, {0x1F93C, 0x1F945}, {0x1F947, 0x1F9FF},
{0x1FA70, 0x1FA7C}, {0x1FA80, 0x1FA88}, {0x1FA90, 0x1FABD},
{0x1FABF, 0x1FAC5}, {0x1FACE, 0x1FADB}, {0x1FAE0, 0x1FAE8},
{0x1FAF0, 0x1FAF8}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD},
{0x1FA70, 0x1FA7C}, {0x1FA80, 0x1FA8A}, {0x1FA8E, 0x1FAC6},
{0x1FAC8, 0x1FAC8}, {0x1FACD, 0x1FADC}, {0x1FADF, 0x1FAEA},
{0x1FAEF, 0x1FAF8}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD},
}
var ambiguous = table{
@@ -121,10 +124,9 @@ var ambiguous = table{
{0x26F4, 0x26F4}, {0x26F6, 0x26F9}, {0x26FB, 0x26FC},
{0x26FE, 0x26FF}, {0x273D, 0x273D}, {0x2776, 0x277F},
{0x2B56, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF},
{0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A},
{0x1F110, 0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F18D},
{0x1F18F, 0x1F190}, {0x1F19B, 0x1F1AC}, {0xE0100, 0xE01EF},
{0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD},
{0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, {0x1F110, 0x1F12D},
{0x1F130, 0x1F169}, {0x1F170, 0x1F18D}, {0x1F18F, 0x1F190},
{0x1F19B, 0x1F1AC}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD},
}
var narrow = table{
{0x0020, 0x007E}, {0x00A2, 0x00A3}, {0x00A5, 0x00A6},
@@ -159,115 +161,116 @@ var neutral = table{
{0x0600, 0x070D}, {0x070F, 0x074A}, {0x074D, 0x07B1},
{0x07C0, 0x07FA}, {0x07FD, 0x082D}, {0x0830, 0x083E},
{0x0840, 0x085B}, {0x085E, 0x085E}, {0x0860, 0x086A},
{0x0870, 0x088E}, {0x0890, 0x0891}, {0x0898, 0x0983},
{0x0985, 0x098C}, {0x098F, 0x0990}, {0x0993, 0x09A8},
{0x09AA, 0x09B0}, {0x09B2, 0x09B2}, {0x09B6, 0x09B9},
{0x09BC, 0x09C4}, {0x09C7, 0x09C8}, {0x09CB, 0x09CE},
{0x09D7, 0x09D7}, {0x09DC, 0x09DD}, {0x09DF, 0x09E3},
{0x09E6, 0x09FE}, {0x0A01, 0x0A03}, {0x0A05, 0x0A0A},
{0x0A0F, 0x0A10}, {0x0A13, 0x0A28}, {0x0A2A, 0x0A30},
{0x0A32, 0x0A33}, {0x0A35, 0x0A36}, {0x0A38, 0x0A39},
{0x0A3C, 0x0A3C}, {0x0A3E, 0x0A42}, {0x0A47, 0x0A48},
{0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, {0x0A59, 0x0A5C},
{0x0A5E, 0x0A5E}, {0x0A66, 0x0A76}, {0x0A81, 0x0A83},
{0x0A85, 0x0A8D}, {0x0A8F, 0x0A91}, {0x0A93, 0x0AA8},
{0x0AAA, 0x0AB0}, {0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9},
{0x0ABC, 0x0AC5}, {0x0AC7, 0x0AC9}, {0x0ACB, 0x0ACD},
{0x0AD0, 0x0AD0}, {0x0AE0, 0x0AE3}, {0x0AE6, 0x0AF1},
{0x0AF9, 0x0AFF}, {0x0B01, 0x0B03}, {0x0B05, 0x0B0C},
{0x0B0F, 0x0B10}, {0x0B13, 0x0B28}, {0x0B2A, 0x0B30},
{0x0B32, 0x0B33}, {0x0B35, 0x0B39}, {0x0B3C, 0x0B44},
{0x0B47, 0x0B48}, {0x0B4B, 0x0B4D}, {0x0B55, 0x0B57},
{0x0B5C, 0x0B5D}, {0x0B5F, 0x0B63}, {0x0B66, 0x0B77},
{0x0B82, 0x0B83}, {0x0B85, 0x0B8A}, {0x0B8E, 0x0B90},
{0x0B92, 0x0B95}, {0x0B99, 0x0B9A}, {0x0B9C, 0x0B9C},
{0x0B9E, 0x0B9F}, {0x0BA3, 0x0BA4}, {0x0BA8, 0x0BAA},
{0x0BAE, 0x0BB9}, {0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8},
{0x0BCA, 0x0BCD}, {0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7},
{0x0BE6, 0x0BFA}, {0x0C00, 0x0C0C}, {0x0C0E, 0x0C10},
{0x0C12, 0x0C28}, {0x0C2A, 0x0C39}, {0x0C3C, 0x0C44},
{0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56},
{0x0C58, 0x0C5A}, {0x0C5D, 0x0C5D}, {0x0C60, 0x0C63},
{0x0C66, 0x0C6F}, {0x0C77, 0x0C8C}, {0x0C8E, 0x0C90},
{0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9},
{0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD},
{0x0CD5, 0x0CD6}, {0x0CDD, 0x0CDE}, {0x0CE0, 0x0CE3},
{0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF3}, {0x0D00, 0x0D0C},
{0x0D0E, 0x0D10}, {0x0D12, 0x0D44}, {0x0D46, 0x0D48},
{0x0D4A, 0x0D4F}, {0x0D54, 0x0D63}, {0x0D66, 0x0D7F},
{0x0D81, 0x0D83}, {0x0D85, 0x0D96}, {0x0D9A, 0x0DB1},
{0x0DB3, 0x0DBB}, {0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6},
{0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6},
{0x0DD8, 0x0DDF}, {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4},
{0x0E01, 0x0E3A}, {0x0E3F, 0x0E5B}, {0x0E81, 0x0E82},
{0x0E84, 0x0E84}, {0x0E86, 0x0E8A}, {0x0E8C, 0x0EA3},
{0x0EA5, 0x0EA5}, {0x0EA7, 0x0EBD}, {0x0EC0, 0x0EC4},
{0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECE}, {0x0ED0, 0x0ED9},
{0x0EDC, 0x0EDF}, {0x0F00, 0x0F47}, {0x0F49, 0x0F6C},
{0x0F71, 0x0F97}, {0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC},
{0x0FCE, 0x0FDA}, {0x1000, 0x10C5}, {0x10C7, 0x10C7},
{0x10CD, 0x10CD}, {0x10D0, 0x10FF}, {0x1160, 0x1248},
{0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258},
{0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D},
{0x1290, 0x12B0}, {0x12B2, 0x12B5}, {0x12B8, 0x12BE},
{0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6},
{0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A},
{0x135D, 0x137C}, {0x1380, 0x1399}, {0x13A0, 0x13F5},
{0x13F8, 0x13FD}, {0x1400, 0x169C}, {0x16A0, 0x16F8},
{0x1700, 0x1715}, {0x171F, 0x1736}, {0x1740, 0x1753},
{0x1760, 0x176C}, {0x176E, 0x1770}, {0x1772, 0x1773},
{0x1780, 0x17DD}, {0x17E0, 0x17E9}, {0x17F0, 0x17F9},
{0x1800, 0x1819}, {0x1820, 0x1878}, {0x1880, 0x18AA},
{0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1920, 0x192B},
{0x1930, 0x193B}, {0x1940, 0x1940}, {0x1944, 0x196D},
{0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9},
{0x19D0, 0x19DA}, {0x19DE, 0x1A1B}, {0x1A1E, 0x1A5E},
{0x1A60, 0x1A7C}, {0x1A7F, 0x1A89}, {0x1A90, 0x1A99},
{0x1AA0, 0x1AAD}, {0x1AB0, 0x1ACE}, {0x1B00, 0x1B4C},
{0x1B50, 0x1B7E}, {0x1B80, 0x1BF3}, {0x1BFC, 0x1C37},
{0x1C3B, 0x1C49}, {0x1C4D, 0x1C88}, {0x1C90, 0x1CBA},
{0x1CBD, 0x1CC7}, {0x1CD0, 0x1CFA}, {0x1D00, 0x1F15},
{0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D},
{0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B},
{0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4},
{0x1FB6, 0x1FC4}, {0x1FC6, 0x1FD3}, {0x1FD6, 0x1FDB},
{0x1FDD, 0x1FEF}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFE},
{0x2000, 0x200F}, {0x2011, 0x2012}, {0x2017, 0x2017},
{0x201A, 0x201B}, {0x201E, 0x201F}, {0x2023, 0x2023},
{0x2028, 0x202F}, {0x2031, 0x2031}, {0x2034, 0x2034},
{0x2036, 0x203A}, {0x203C, 0x203D}, {0x203F, 0x2064},
{0x2066, 0x2071}, {0x2075, 0x207E}, {0x2080, 0x2080},
{0x2085, 0x208E}, {0x2090, 0x209C}, {0x20A0, 0x20A8},
{0x20AA, 0x20AB}, {0x20AD, 0x20C0}, {0x20D0, 0x20F0},
{0x2100, 0x2102}, {0x2104, 0x2104}, {0x2106, 0x2108},
{0x210A, 0x2112}, {0x2114, 0x2115}, {0x2117, 0x2120},
{0x2123, 0x2125}, {0x2127, 0x212A}, {0x212C, 0x2152},
{0x2155, 0x215A}, {0x215F, 0x215F}, {0x216C, 0x216F},
{0x217A, 0x2188}, {0x218A, 0x218B}, {0x219A, 0x21B7},
{0x21BA, 0x21D1}, {0x21D3, 0x21D3}, {0x21D5, 0x21E6},
{0x21E8, 0x21FF}, {0x2201, 0x2201}, {0x2204, 0x2206},
{0x2209, 0x220A}, {0x220C, 0x220E}, {0x2210, 0x2210},
{0x2212, 0x2214}, {0x2216, 0x2219}, {0x221B, 0x221C},
{0x2221, 0x2222}, {0x2224, 0x2224}, {0x2226, 0x2226},
{0x222D, 0x222D}, {0x222F, 0x2233}, {0x2238, 0x223B},
{0x223E, 0x2247}, {0x2249, 0x224B}, {0x224D, 0x2251},
{0x2253, 0x225F}, {0x2262, 0x2263}, {0x2268, 0x2269},
{0x226C, 0x226D}, {0x2270, 0x2281}, {0x2284, 0x2285},
{0x2288, 0x2294}, {0x2296, 0x2298}, {0x229A, 0x22A4},
{0x22A6, 0x22BE}, {0x22C0, 0x2311}, {0x2313, 0x2319},
{0x231C, 0x2328}, {0x232B, 0x23E8}, {0x23ED, 0x23EF},
{0x23F1, 0x23F2}, {0x23F4, 0x2426}, {0x2440, 0x244A},
{0x24EA, 0x24EA}, {0x254C, 0x254F}, {0x2574, 0x257F},
{0x2590, 0x2591}, {0x2596, 0x259F}, {0x25A2, 0x25A2},
{0x25AA, 0x25B1}, {0x25B4, 0x25B5}, {0x25B8, 0x25BB},
{0x25BE, 0x25BF}, {0x25C2, 0x25C5}, {0x25C9, 0x25CA},
{0x25CC, 0x25CD}, {0x25D2, 0x25E1}, {0x25E6, 0x25EE},
{0x25F0, 0x25FC}, {0x25FF, 0x2604}, {0x2607, 0x2608},
{0x260A, 0x260D}, {0x2610, 0x2613}, {0x2616, 0x261B},
{0x261D, 0x261D}, {0x261F, 0x263F}, {0x2641, 0x2641},
{0x2643, 0x2647}, {0x2654, 0x265F}, {0x2662, 0x2662},
{0x2666, 0x2666}, {0x266B, 0x266B}, {0x266E, 0x266E},
{0x2670, 0x267E}, {0x2680, 0x2692}, {0x2694, 0x269D},
{0x0870, 0x0891}, {0x0897, 0x0983}, {0x0985, 0x098C},
{0x098F, 0x0990}, {0x0993, 0x09A8}, {0x09AA, 0x09B0},
{0x09B2, 0x09B2}, {0x09B6, 0x09B9}, {0x09BC, 0x09C4},
{0x09C7, 0x09C8}, {0x09CB, 0x09CE}, {0x09D7, 0x09D7},
{0x09DC, 0x09DD}, {0x09DF, 0x09E3}, {0x09E6, 0x09FE},
{0x0A01, 0x0A03}, {0x0A05, 0x0A0A}, {0x0A0F, 0x0A10},
{0x0A13, 0x0A28}, {0x0A2A, 0x0A30}, {0x0A32, 0x0A33},
{0x0A35, 0x0A36}, {0x0A38, 0x0A39}, {0x0A3C, 0x0A3C},
{0x0A3E, 0x0A42}, {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D},
{0x0A51, 0x0A51}, {0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E},
{0x0A66, 0x0A76}, {0x0A81, 0x0A83}, {0x0A85, 0x0A8D},
{0x0A8F, 0x0A91}, {0x0A93, 0x0AA8}, {0x0AAA, 0x0AB0},
{0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9}, {0x0ABC, 0x0AC5},
{0x0AC7, 0x0AC9}, {0x0ACB, 0x0ACD}, {0x0AD0, 0x0AD0},
{0x0AE0, 0x0AE3}, {0x0AE6, 0x0AF1}, {0x0AF9, 0x0AFF},
{0x0B01, 0x0B03}, {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10},
{0x0B13, 0x0B28}, {0x0B2A, 0x0B30}, {0x0B32, 0x0B33},
{0x0B35, 0x0B39}, {0x0B3C, 0x0B44}, {0x0B47, 0x0B48},
{0x0B4B, 0x0B4D}, {0x0B55, 0x0B57}, {0x0B5C, 0x0B5D},
{0x0B5F, 0x0B63}, {0x0B66, 0x0B77}, {0x0B82, 0x0B83},
{0x0B85, 0x0B8A}, {0x0B8E, 0x0B90}, {0x0B92, 0x0B95},
{0x0B99, 0x0B9A}, {0x0B9C, 0x0B9C}, {0x0B9E, 0x0B9F},
{0x0BA3, 0x0BA4}, {0x0BA8, 0x0BAA}, {0x0BAE, 0x0BB9},
{0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCD},
{0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BFA},
{0x0C00, 0x0C0C}, {0x0C0E, 0x0C10}, {0x0C12, 0x0C28},
{0x0C2A, 0x0C39}, {0x0C3C, 0x0C44}, {0x0C46, 0x0C48},
{0x0C4A, 0x0C4D}, {0x0C55, 0x0C56}, {0x0C58, 0x0C5A},
{0x0C5C, 0x0C5D}, {0x0C60, 0x0C63}, {0x0C66, 0x0C6F},
{0x0C77, 0x0C8C}, {0x0C8E, 0x0C90}, {0x0C92, 0x0CA8},
{0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9}, {0x0CBC, 0x0CC4},
{0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD}, {0x0CD5, 0x0CD6},
{0x0CDC, 0x0CDE}, {0x0CE0, 0x0CE3}, {0x0CE6, 0x0CEF},
{0x0CF1, 0x0CF3}, {0x0D00, 0x0D0C}, {0x0D0E, 0x0D10},
{0x0D12, 0x0D44}, {0x0D46, 0x0D48}, {0x0D4A, 0x0D4F},
{0x0D54, 0x0D63}, {0x0D66, 0x0D7F}, {0x0D81, 0x0D83},
{0x0D85, 0x0D96}, {0x0D9A, 0x0DB1}, {0x0DB3, 0x0DBB},
{0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6}, {0x0DCA, 0x0DCA},
{0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF},
{0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4}, {0x0E01, 0x0E3A},
{0x0E3F, 0x0E5B}, {0x0E81, 0x0E82}, {0x0E84, 0x0E84},
{0x0E86, 0x0E8A}, {0x0E8C, 0x0EA3}, {0x0EA5, 0x0EA5},
{0x0EA7, 0x0EBD}, {0x0EC0, 0x0EC4}, {0x0EC6, 0x0EC6},
{0x0EC8, 0x0ECE}, {0x0ED0, 0x0ED9}, {0x0EDC, 0x0EDF},
{0x0F00, 0x0F47}, {0x0F49, 0x0F6C}, {0x0F71, 0x0F97},
{0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC}, {0x0FCE, 0x0FDA},
{0x1000, 0x10C5}, {0x10C7, 0x10C7}, {0x10CD, 0x10CD},
{0x10D0, 0x10FF}, {0x1160, 0x1248}, {0x124A, 0x124D},
{0x1250, 0x1256}, {0x1258, 0x1258}, {0x125A, 0x125D},
{0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0},
{0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0},
{0x12C2, 0x12C5}, {0x12C8, 0x12D6}, {0x12D8, 0x1310},
{0x1312, 0x1315}, {0x1318, 0x135A}, {0x135D, 0x137C},
{0x1380, 0x1399}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD},
{0x1400, 0x169C}, {0x16A0, 0x16F8}, {0x1700, 0x1715},
{0x171F, 0x1736}, {0x1740, 0x1753}, {0x1760, 0x176C},
{0x176E, 0x1770}, {0x1772, 0x1773}, {0x1780, 0x17DD},
{0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1800, 0x180A},
{0x180E, 0x180E}, {0x1810, 0x1819}, {0x1820, 0x1878},
{0x1880, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E},
{0x1920, 0x192B}, {0x1930, 0x193B}, {0x1940, 0x1940},
{0x1944, 0x196D}, {0x1970, 0x1974}, {0x1980, 0x19AB},
{0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x19DE, 0x1A1B},
{0x1A1E, 0x1A5E}, {0x1A60, 0x1A7C}, {0x1A7F, 0x1A89},
{0x1A90, 0x1A99}, {0x1AA0, 0x1AAD}, {0x1AB0, 0x1ADD},
{0x1AE0, 0x1AEB}, {0x1B00, 0x1B4C}, {0x1B4E, 0x1BF3},
{0x1BFC, 0x1C37}, {0x1C3B, 0x1C49}, {0x1C4D, 0x1C8A},
{0x1C90, 0x1CBA}, {0x1CBD, 0x1CC7}, {0x1CD0, 0x1CFA},
{0x1D00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45},
{0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, {0x1F59, 0x1F59},
{0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D},
{0x1F80, 0x1FB4}, {0x1FB6, 0x1FC4}, {0x1FC6, 0x1FD3},
{0x1FD6, 0x1FDB}, {0x1FDD, 0x1FEF}, {0x1FF2, 0x1FF4},
{0x1FF6, 0x1FFE}, {0x2000, 0x200F}, {0x2011, 0x2012},
{0x2017, 0x2017}, {0x201A, 0x201B}, {0x201E, 0x201F},
{0x2023, 0x2023}, {0x2028, 0x202F}, {0x2031, 0x2031},
{0x2034, 0x2034}, {0x2036, 0x203A}, {0x203C, 0x203D},
{0x203F, 0x2064}, {0x2066, 0x2071}, {0x2075, 0x207E},
{0x2080, 0x2080}, {0x2085, 0x208E}, {0x2090, 0x209C},
{0x20A0, 0x20A8}, {0x20AA, 0x20AB}, {0x20AD, 0x20C1},
{0x20D0, 0x20F0}, {0x2100, 0x2102}, {0x2104, 0x2104},
{0x2106, 0x2108}, {0x210A, 0x2112}, {0x2114, 0x2115},
{0x2117, 0x2120}, {0x2123, 0x2125}, {0x2127, 0x212A},
{0x212C, 0x2152}, {0x2155, 0x215A}, {0x215F, 0x215F},
{0x216C, 0x216F}, {0x217A, 0x2188}, {0x218A, 0x218B},
{0x219A, 0x21B7}, {0x21BA, 0x21D1}, {0x21D3, 0x21D3},
{0x21D5, 0x21E6}, {0x21E8, 0x21FF}, {0x2201, 0x2201},
{0x2204, 0x2206}, {0x2209, 0x220A}, {0x220C, 0x220E},
{0x2210, 0x2210}, {0x2212, 0x2214}, {0x2216, 0x2219},
{0x221B, 0x221C}, {0x2221, 0x2222}, {0x2224, 0x2224},
{0x2226, 0x2226}, {0x222D, 0x222D}, {0x222F, 0x2233},
{0x2238, 0x223B}, {0x223E, 0x2247}, {0x2249, 0x224B},
{0x224D, 0x2251}, {0x2253, 0x225F}, {0x2262, 0x2263},
{0x2268, 0x2269}, {0x226C, 0x226D}, {0x2270, 0x2281},
{0x2284, 0x2285}, {0x2288, 0x2294}, {0x2296, 0x2298},
{0x229A, 0x22A4}, {0x22A6, 0x22BE}, {0x22C0, 0x2311},
{0x2313, 0x2319}, {0x231C, 0x2328}, {0x232B, 0x23E8},
{0x23ED, 0x23EF}, {0x23F1, 0x23F2}, {0x23F4, 0x2429},
{0x2440, 0x244A}, {0x24EA, 0x24EA}, {0x254C, 0x254F},
{0x2574, 0x257F}, {0x2590, 0x2591}, {0x2596, 0x259F},
{0x25A2, 0x25A2}, {0x25AA, 0x25B1}, {0x25B4, 0x25B5},
{0x25B8, 0x25BB}, {0x25BE, 0x25BF}, {0x25C2, 0x25C5},
{0x25C9, 0x25CA}, {0x25CC, 0x25CD}, {0x25D2, 0x25E1},
{0x25E6, 0x25EE}, {0x25F0, 0x25FC}, {0x25FF, 0x2604},
{0x2607, 0x2608}, {0x260A, 0x260D}, {0x2610, 0x2613},
{0x2616, 0x261B}, {0x261D, 0x261D}, {0x261F, 0x262F},
{0x2638, 0x263F}, {0x2641, 0x2641}, {0x2643, 0x2647},
{0x2654, 0x265F}, {0x2662, 0x2662}, {0x2666, 0x2666},
{0x266B, 0x266B}, {0x266E, 0x266E}, {0x2670, 0x267E},
{0x2680, 0x2689}, {0x2690, 0x2692}, {0x2694, 0x269D},
{0x26A0, 0x26A0}, {0x26A2, 0x26A9}, {0x26AC, 0x26BC},
{0x26C0, 0x26C3}, {0x26E2, 0x26E2}, {0x26E4, 0x26E7},
{0x2700, 0x2704}, {0x2706, 0x2709}, {0x270C, 0x2727},
@@ -276,175 +279,210 @@ var neutral = table{
{0x2780, 0x2794}, {0x2798, 0x27AF}, {0x27B1, 0x27BE},
{0x27C0, 0x27E5}, {0x27EE, 0x2984}, {0x2987, 0x2B1A},
{0x2B1D, 0x2B4F}, {0x2B51, 0x2B54}, {0x2B5A, 0x2B73},
{0x2B76, 0x2B95}, {0x2B97, 0x2CF3}, {0x2CF9, 0x2D25},
{0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67},
{0x2D6F, 0x2D70}, {0x2D7F, 0x2D96}, {0x2DA0, 0x2DA6},
{0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE},
{0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6},
{0x2DD8, 0x2DDE}, {0x2DE0, 0x2E5D}, {0x303F, 0x303F},
{0x4DC0, 0x4DFF}, {0xA4D0, 0xA62B}, {0xA640, 0xA6F7},
{0xA700, 0xA7CA}, {0xA7D0, 0xA7D1}, {0xA7D3, 0xA7D3},
{0xA7D5, 0xA7D9}, {0xA7F2, 0xA82C}, {0xA830, 0xA839},
{0xA840, 0xA877}, {0xA880, 0xA8C5}, {0xA8CE, 0xA8D9},
{0xA8E0, 0xA953}, {0xA95F, 0xA95F}, {0xA980, 0xA9CD},
{0xA9CF, 0xA9D9}, {0xA9DE, 0xA9FE}, {0xAA00, 0xAA36},
{0xAA40, 0xAA4D}, {0xAA50, 0xAA59}, {0xAA5C, 0xAAC2},
{0xAADB, 0xAAF6}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E},
{0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E},
{0xAB30, 0xAB6B}, {0xAB70, 0xABED}, {0xABF0, 0xABF9},
{0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xD800, 0xDFFF},
{0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB36},
{0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41},
{0xFB43, 0xFB44}, {0xFB46, 0xFBC2}, {0xFBD3, 0xFD8F},
{0xFD92, 0xFDC7}, {0xFDCF, 0xFDCF}, {0xFDF0, 0xFDFF},
{0xFE20, 0xFE2F}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC},
{0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFC}, {0x10000, 0x1000B},
{0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D},
{0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA},
{0x10100, 0x10102}, {0x10107, 0x10133}, {0x10137, 0x1018E},
{0x10190, 0x1019C}, {0x101A0, 0x101A0}, {0x101D0, 0x101FD},
{0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E0, 0x102FB},
{0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x1037A},
{0x10380, 0x1039D}, {0x1039F, 0x103C3}, {0x103C8, 0x103D5},
{0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3},
{0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563},
{0x1056F, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592},
{0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1},
{0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x10600, 0x10736},
{0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785},
{0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10800, 0x10805},
{0x10808, 0x10808}, {0x1080A, 0x10835}, {0x10837, 0x10838},
{0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10857, 0x1089E},
{0x108A7, 0x108AF}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5},
{0x108FB, 0x1091B}, {0x1091F, 0x10939}, {0x1093F, 0x1093F},
{0x10980, 0x109B7}, {0x109BC, 0x109CF}, {0x109D2, 0x10A03},
{0x10A05, 0x10A06}, {0x10A0C, 0x10A13}, {0x10A15, 0x10A17},
{0x10A19, 0x10A35}, {0x10A38, 0x10A3A}, {0x10A3F, 0x10A48},
{0x10A50, 0x10A58}, {0x10A60, 0x10A9F}, {0x10AC0, 0x10AE6},
{0x10AEB, 0x10AF6}, {0x10B00, 0x10B35}, {0x10B39, 0x10B55},
{0x10B58, 0x10B72}, {0x10B78, 0x10B91}, {0x10B99, 0x10B9C},
{0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2},
{0x10CC0, 0x10CF2}, {0x10CFA, 0x10D27}, {0x10D30, 0x10D39},
{0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, {0x10EAB, 0x10EAD},
{0x10EB0, 0x10EB1}, {0x10EFD, 0x10F27}, {0x10F30, 0x10F59},
{0x10F70, 0x10F89}, {0x10FB0, 0x10FCB}, {0x10FE0, 0x10FF6},
{0x11000, 0x1104D}, {0x11052, 0x11075}, {0x1107F, 0x110C2},
{0x110CD, 0x110CD}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9},
{0x11100, 0x11134}, {0x11136, 0x11147}, {0x11150, 0x11176},
{0x11180, 0x111DF}, {0x111E1, 0x111F4}, {0x11200, 0x11211},
{0x11213, 0x11241}, {0x11280, 0x11286}, {0x11288, 0x11288},
{0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A9},
{0x112B0, 0x112EA}, {0x112F0, 0x112F9}, {0x11300, 0x11303},
{0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328},
{0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339},
{0x1133B, 0x11344}, {0x11347, 0x11348}, {0x1134B, 0x1134D},
{0x11350, 0x11350}, {0x11357, 0x11357}, {0x1135D, 0x11363},
{0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11400, 0x1145B},
{0x2B76, 0x2CF3}, {0x2CF9, 0x2D25}, {0x2D27, 0x2D27},
{0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D70},
{0x2D7F, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE},
{0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6},
{0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE},
{0x2DE0, 0x2E5D}, {0x303F, 0x303F}, {0xA4D0, 0xA62B},
{0xA640, 0xA6F7}, {0xA700, 0xA7DC}, {0xA7F1, 0xA82C},
{0xA830, 0xA839}, {0xA840, 0xA877}, {0xA880, 0xA8C5},
{0xA8CE, 0xA8D9}, {0xA8E0, 0xA953}, {0xA95F, 0xA95F},
{0xA980, 0xA9CD}, {0xA9CF, 0xA9D9}, {0xA9DE, 0xA9FE},
{0xAA00, 0xAA36}, {0xAA40, 0xAA4D}, {0xAA50, 0xAA59},
{0xAA5C, 0xAAC2}, {0xAADB, 0xAAF6}, {0xAB01, 0xAB06},
{0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26},
{0xAB28, 0xAB2E}, {0xAB30, 0xAB6B}, {0xAB70, 0xABED},
{0xABF0, 0xABF9}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB},
{0xD800, 0xDFFF}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17},
{0xFB1D, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E},
{0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFDCF},
{0xFDF0, 0xFDFF}, {0xFE20, 0xFE2F}, {0xFE70, 0xFE74},
{0xFE76, 0xFEFC}, {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFC},
{0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A},
{0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D},
{0x10080, 0x100FA}, {0x10100, 0x10102}, {0x10107, 0x10133},
{0x10137, 0x1018E}, {0x10190, 0x1019C}, {0x101A0, 0x101A0},
{0x101D0, 0x101FD}, {0x10280, 0x1029C}, {0x102A0, 0x102D0},
{0x102E0, 0x102FB}, {0x10300, 0x10323}, {0x1032D, 0x1034A},
{0x10350, 0x1037A}, {0x10380, 0x1039D}, {0x1039F, 0x103C3},
{0x103C8, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9},
{0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527},
{0x10530, 0x10563}, {0x1056F, 0x1057A}, {0x1057C, 0x1058A},
{0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1},
{0x105A3, 0x105B1}, {0x105B3, 0x105B9}, {0x105BB, 0x105BC},
{0x105C0, 0x105F3}, {0x10600, 0x10736}, {0x10740, 0x10755},
{0x10760, 0x10767}, {0x10780, 0x10785}, {0x10787, 0x107B0},
{0x107B2, 0x107BA}, {0x10800, 0x10805}, {0x10808, 0x10808},
{0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C},
{0x1083F, 0x10855}, {0x10857, 0x1089E}, {0x108A7, 0x108AF},
{0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B},
{0x1091F, 0x10939}, {0x1093F, 0x10959}, {0x10980, 0x109B7},
{0x109BC, 0x109CF}, {0x109D2, 0x10A03}, {0x10A05, 0x10A06},
{0x10A0C, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35},
{0x10A38, 0x10A3A}, {0x10A3F, 0x10A48}, {0x10A50, 0x10A58},
{0x10A60, 0x10A9F}, {0x10AC0, 0x10AE6}, {0x10AEB, 0x10AF6},
{0x10B00, 0x10B35}, {0x10B39, 0x10B55}, {0x10B58, 0x10B72},
{0x10B78, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF},
{0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2},
{0x10CFA, 0x10D27}, {0x10D30, 0x10D39}, {0x10D40, 0x10D65},
{0x10D69, 0x10D85}, {0x10D8E, 0x10D8F}, {0x10E60, 0x10E7E},
{0x10E80, 0x10EA9}, {0x10EAB, 0x10EAD}, {0x10EB0, 0x10EB1},
{0x10EC2, 0x10EC7}, {0x10ED0, 0x10ED8}, {0x10EFA, 0x10F27},
{0x10F30, 0x10F59}, {0x10F70, 0x10F89}, {0x10FB0, 0x10FCB},
{0x10FE0, 0x10FF6}, {0x11000, 0x1104D}, {0x11052, 0x11075},
{0x1107F, 0x110C2}, {0x110CD, 0x110CD}, {0x110D0, 0x110E8},
{0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11147},
{0x11150, 0x11176}, {0x11180, 0x111DF}, {0x111E1, 0x111F4},
{0x11200, 0x11211}, {0x11213, 0x11241}, {0x11280, 0x11286},
{0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D},
{0x1129F, 0x112A9}, {0x112B0, 0x112EA}, {0x112F0, 0x112F9},
{0x11300, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310},
{0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333},
{0x11335, 0x11339}, {0x1133B, 0x11344}, {0x11347, 0x11348},
{0x1134B, 0x1134D}, {0x11350, 0x11350}, {0x11357, 0x11357},
{0x1135D, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374},
{0x11380, 0x11389}, {0x1138B, 0x1138B}, {0x1138E, 0x1138E},
{0x11390, 0x113B5}, {0x113B7, 0x113C0}, {0x113C2, 0x113C2},
{0x113C5, 0x113C5}, {0x113C7, 0x113CA}, {0x113CC, 0x113D5},
{0x113D7, 0x113D8}, {0x113E1, 0x113E2}, {0x11400, 0x1145B},
{0x1145D, 0x11461}, {0x11480, 0x114C7}, {0x114D0, 0x114D9},
{0x11580, 0x115B5}, {0x115B8, 0x115DD}, {0x11600, 0x11644},
{0x11650, 0x11659}, {0x11660, 0x1166C}, {0x11680, 0x116B9},
{0x116C0, 0x116C9}, {0x11700, 0x1171A}, {0x1171D, 0x1172B},
{0x11730, 0x11746}, {0x11800, 0x1183B}, {0x118A0, 0x118F2},
{0x118FF, 0x11906}, {0x11909, 0x11909}, {0x1190C, 0x11913},
{0x11915, 0x11916}, {0x11918, 0x11935}, {0x11937, 0x11938},
{0x1193B, 0x11946}, {0x11950, 0x11959}, {0x119A0, 0x119A7},
{0x119AA, 0x119D7}, {0x119DA, 0x119E4}, {0x11A00, 0x11A47},
{0x11A50, 0x11AA2}, {0x11AB0, 0x11AF8}, {0x11B00, 0x11B09},
{0x11C00, 0x11C08}, {0x11C0A, 0x11C36}, {0x11C38, 0x11C45},
{0x11C50, 0x11C6C}, {0x11C70, 0x11C8F}, {0x11C92, 0x11CA7},
{0x11CA9, 0x11CB6}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09},
{0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D},
{0x11D3F, 0x11D47}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65},
{0x11D67, 0x11D68}, {0x11D6A, 0x11D8E}, {0x11D90, 0x11D91},
{0x11D93, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF8},
{0x11F00, 0x11F10}, {0x11F12, 0x11F3A}, {0x11F3E, 0x11F59},
{0x116C0, 0x116C9}, {0x116D0, 0x116E3}, {0x11700, 0x1171A},
{0x1171D, 0x1172B}, {0x11730, 0x11746}, {0x11800, 0x1183B},
{0x118A0, 0x118F2}, {0x118FF, 0x11906}, {0x11909, 0x11909},
{0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x11935},
{0x11937, 0x11938}, {0x1193B, 0x11946}, {0x11950, 0x11959},
{0x119A0, 0x119A7}, {0x119AA, 0x119D7}, {0x119DA, 0x119E4},
{0x11A00, 0x11A47}, {0x11A50, 0x11AA2}, {0x11AB0, 0x11AF8},
{0x11B00, 0x11B09}, {0x11B60, 0x11B67}, {0x11BC0, 0x11BE1},
{0x11BF0, 0x11BF9}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C36},
{0x11C38, 0x11C45}, {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F},
{0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x11D00, 0x11D06},
{0x11D08, 0x11D09}, {0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A},
{0x11D3C, 0x11D3D}, {0x11D3F, 0x11D47}, {0x11D50, 0x11D59},
{0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D8E},
{0x11D90, 0x11D91}, {0x11D93, 0x11D98}, {0x11DA0, 0x11DA9},
{0x11DB0, 0x11DDB}, {0x11DE0, 0x11DE9}, {0x11EE0, 0x11EF8},
{0x11F00, 0x11F10}, {0x11F12, 0x11F3A}, {0x11F3E, 0x11F5A},
{0x11FB0, 0x11FB0}, {0x11FC0, 0x11FF1}, {0x11FFF, 0x12399},
{0x12400, 0x1246E}, {0x12470, 0x12474}, {0x12480, 0x12543},
{0x12F90, 0x12FF2}, {0x13000, 0x13455}, {0x14400, 0x14646},
{0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69},
{0x16A6E, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED},
{0x16AF0, 0x16AF5}, {0x16B00, 0x16B45}, {0x16B50, 0x16B59},
{0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F},
{0x16E40, 0x16E9A}, {0x16F00, 0x16F4A}, {0x16F4F, 0x16F87},
{0x16F8F, 0x16F9F}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C},
{0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BCA3},
{0x12F90, 0x12FF2}, {0x13000, 0x13455}, {0x13460, 0x143FA},
{0x14400, 0x14646}, {0x16100, 0x16139}, {0x16800, 0x16A38},
{0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A6E, 0x16ABE},
{0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5},
{0x16B00, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61},
{0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16D40, 0x16D79},
{0x16E40, 0x16E9A}, {0x16EA0, 0x16EB8}, {0x16EBB, 0x16ED3},
{0x16F00, 0x16F4A}, {0x16F4F, 0x16F87}, {0x16F8F, 0x16F9F},
{0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88},
{0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BCA3}, {0x1CC00, 0x1CCFC},
{0x1CD00, 0x1CEB3}, {0x1CEBA, 0x1CED0}, {0x1CEE0, 0x1CEF0},
{0x1CF00, 0x1CF2D}, {0x1CF30, 0x1CF46}, {0x1CF50, 0x1CFC3},
{0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D1EA},
{0x1D200, 0x1D245}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3},
{0x1D300, 0x1D356}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454},
{0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2},
{0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9},
{0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505},
{0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C},
{0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544},
{0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5},
{0x1D6A8, 0x1D7CB}, {0x1D7CE, 0x1DA8B}, {0x1DA9B, 0x1DA9F},
{0x1DAA1, 0x1DAAF}, {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A},
{0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021},
{0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, {0x1E030, 0x1E06D},
{0x1E08F, 0x1E08F}, {0x1E100, 0x1E12C}, {0x1E130, 0x1E13D},
{0x1E140, 0x1E149}, {0x1E14E, 0x1E14F}, {0x1E290, 0x1E2AE},
{0x1E2C0, 0x1E2F9}, {0x1E2FF, 0x1E2FF}, {0x1E4D0, 0x1E4F9},
{0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE},
{0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6},
{0x1E900, 0x1E94B}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F},
{0x1EC71, 0x1ECB4}, {0x1ED01, 0x1ED3D}, {0x1EE00, 0x1EE03},
{0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24},
{0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37},
{0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42},
{0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B},
{0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54},
{0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B},
{0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62},
{0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72},
{0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E},
{0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3},
{0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1EEF0, 0x1EEF1},
{0x1F000, 0x1F003}, {0x1F005, 0x1F02B}, {0x1F030, 0x1F093},
{0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, {0x1F0C1, 0x1F0CE},
{0x1F0D1, 0x1F0F5}, {0x1F10B, 0x1F10F}, {0x1F12E, 0x1F12F},
{0x1F16A, 0x1F16F}, {0x1F1AD, 0x1F1AD}, {0x1F1E6, 0x1F1FF},
{0x1F321, 0x1F32C}, {0x1F336, 0x1F336}, {0x1F37D, 0x1F37D},
{0x1F394, 0x1F39F}, {0x1F3CB, 0x1F3CE}, {0x1F3D4, 0x1F3DF},
{0x1F3F1, 0x1F3F3}, {0x1F3F5, 0x1F3F7}, {0x1F43F, 0x1F43F},
{0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FE}, {0x1F53E, 0x1F54A},
{0x1F54F, 0x1F54F}, {0x1F568, 0x1F579}, {0x1F57B, 0x1F594},
{0x1F597, 0x1F5A3}, {0x1F5A5, 0x1F5FA}, {0x1F650, 0x1F67F},
{0x1F6C6, 0x1F6CB}, {0x1F6CD, 0x1F6CF}, {0x1F6D3, 0x1F6D4},
{0x1F6E0, 0x1F6EA}, {0x1F6F0, 0x1F6F3}, {0x1F700, 0x1F776},
{0x1F77B, 0x1F7D9}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847},
{0x1D377, 0x1D378}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C},
{0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6},
{0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB},
{0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A},
{0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539},
{0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546},
{0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D7CB},
{0x1D7CE, 0x1DA8B}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF},
{0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E000, 0x1E006},
{0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024},
{0x1E026, 0x1E02A}, {0x1E030, 0x1E06D}, {0x1E08F, 0x1E08F},
{0x1E100, 0x1E12C}, {0x1E130, 0x1E13D}, {0x1E140, 0x1E149},
{0x1E14E, 0x1E14F}, {0x1E290, 0x1E2AE}, {0x1E2C0, 0x1E2F9},
{0x1E2FF, 0x1E2FF}, {0x1E4D0, 0x1E4F9}, {0x1E5D0, 0x1E5FA},
{0x1E5FF, 0x1E5FF}, {0x1E6C0, 0x1E6DE}, {0x1E6E0, 0x1E6F5},
{0x1E6FE, 0x1E6FF}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB},
{0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4},
{0x1E8C7, 0x1E8D6}, {0x1E900, 0x1E94B}, {0x1E950, 0x1E959},
{0x1E95E, 0x1E95F}, {0x1EC71, 0x1ECB4}, {0x1ED01, 0x1ED3D},
{0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22},
{0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32},
{0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B},
{0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49},
{0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52},
{0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59},
{0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F},
{0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A},
{0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C},
{0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B},
{0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB},
{0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F003}, {0x1F005, 0x1F02B},
{0x1F030, 0x1F093}, {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF},
{0x1F0C1, 0x1F0CE}, {0x1F0D1, 0x1F0F5}, {0x1F10B, 0x1F10F},
{0x1F12E, 0x1F12F}, {0x1F16A, 0x1F16F}, {0x1F1AD, 0x1F1AD},
{0x1F1E6, 0x1F1FF}, {0x1F321, 0x1F32C}, {0x1F336, 0x1F336},
{0x1F37D, 0x1F37D}, {0x1F394, 0x1F39F}, {0x1F3CB, 0x1F3CE},
{0x1F3D4, 0x1F3DF}, {0x1F3F1, 0x1F3F3}, {0x1F3F5, 0x1F3F7},
{0x1F43F, 0x1F43F}, {0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FE},
{0x1F53E, 0x1F54A}, {0x1F54F, 0x1F54F}, {0x1F568, 0x1F579},
{0x1F57B, 0x1F594}, {0x1F597, 0x1F5A3}, {0x1F5A5, 0x1F5FA},
{0x1F650, 0x1F67F}, {0x1F6C6, 0x1F6CB}, {0x1F6CD, 0x1F6CF},
{0x1F6D3, 0x1F6D4}, {0x1F6E0, 0x1F6EA}, {0x1F6F0, 0x1F6F3},
{0x1F700, 0x1F7D9}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847},
{0x1F850, 0x1F859}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD},
{0x1F8B0, 0x1F8B1}, {0x1F900, 0x1F90B}, {0x1F93B, 0x1F93B},
{0x1F946, 0x1F946}, {0x1FA00, 0x1FA53}, {0x1FA60, 0x1FA6D},
{0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA}, {0x1FBF0, 0x1FBF9},
{0xE0001, 0xE0001}, {0xE0020, 0xE007F},
{0x1F8B0, 0x1F8BB}, {0x1F8C0, 0x1F8C1}, {0x1F8D0, 0x1F8D8},
{0x1F900, 0x1F90B}, {0x1F93B, 0x1F93B}, {0x1F946, 0x1F946},
{0x1FA00, 0x1FA57}, {0x1FA60, 0x1FA6D}, {0x1FB00, 0x1FB92},
{0x1FB94, 0x1FBFA}, {0xE0001, 0xE0001}, {0xE0020, 0xE007F},
}
var emoji = table{
{0x203C, 0x203C}, {0x2049, 0x2049}, {0x2122, 0x2122},
{0x2139, 0x2139}, {0x2194, 0x2199}, {0x21A9, 0x21AA},
{0x231A, 0x231B}, {0x2328, 0x2328}, {0x2388, 0x2388},
{0x23CF, 0x23CF}, {0x23E9, 0x23F3}, {0x23F8, 0x23FA},
{0x24C2, 0x24C2}, {0x25AA, 0x25AB}, {0x25B6, 0x25B6},
{0x25C0, 0x25C0}, {0x25FB, 0x25FE}, {0x2600, 0x2605},
{0x2607, 0x2612}, {0x2614, 0x2685}, {0x2690, 0x2705},
{0x2708, 0x2712}, {0x2714, 0x2714}, {0x2716, 0x2716},
{0x271D, 0x271D}, {0x2721, 0x2721}, {0x2728, 0x2728},
{0x2733, 0x2734}, {0x2744, 0x2744}, {0x2747, 0x2747},
{0x274C, 0x274C}, {0x274E, 0x274E}, {0x2753, 0x2755},
{0x2757, 0x2757}, {0x2763, 0x2767}, {0x2795, 0x2797},
{0x27A1, 0x27A1}, {0x27B0, 0x27B0}, {0x27BF, 0x27BF},
{0x2934, 0x2935}, {0x2B05, 0x2B07}, {0x2B1B, 0x2B1C},
{0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x3030, 0x3030},
{0x303D, 0x303D}, {0x3297, 0x3297}, {0x3299, 0x3299},
{0x1F000, 0x1F0FF}, {0x1F10D, 0x1F10F}, {0x1F12F, 0x1F12F},
{0x1F16C, 0x1F171}, {0x1F17E, 0x1F17F}, {0x1F18E, 0x1F18E},
{0x1F191, 0x1F19A}, {0x1F1AD, 0x1F1E5}, {0x1F201, 0x1F20F},
{0x1F21A, 0x1F21A}, {0x1F22F, 0x1F22F}, {0x1F232, 0x1F23A},
{0x1F23C, 0x1F23F}, {0x1F249, 0x1F3FA}, {0x1F400, 0x1F53D},
{0x1F546, 0x1F64F}, {0x1F680, 0x1F6FF}, {0x1F774, 0x1F77F},
{0x1F7D5, 0x1F7FF}, {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F},
{0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F8FF},
{0x1F90C, 0x1F93A}, {0x1F93C, 0x1F945}, {0x1F947, 0x1FAFF},
{0x231A, 0x231B}, {0x2328, 0x2328}, {0x23CF, 0x23CF},
{0x23E9, 0x23F3}, {0x23F8, 0x23FA}, {0x24C2, 0x24C2},
{0x25AA, 0x25AB}, {0x25B6, 0x25B6}, {0x25C0, 0x25C0},
{0x25FB, 0x25FE}, {0x2600, 0x2604}, {0x260E, 0x260E},
{0x2611, 0x2611}, {0x2614, 0x2615}, {0x2618, 0x2618},
{0x261D, 0x261D}, {0x2620, 0x2620}, {0x2622, 0x2623},
{0x2626, 0x2626}, {0x262A, 0x262A}, {0x262E, 0x262F},
{0x2638, 0x263A}, {0x2640, 0x2640}, {0x2642, 0x2642},
{0x2648, 0x2653}, {0x265F, 0x2660}, {0x2663, 0x2663},
{0x2665, 0x2666}, {0x2668, 0x2668}, {0x267B, 0x267B},
{0x267E, 0x267F}, {0x2692, 0x2697}, {0x2699, 0x2699},
{0x269B, 0x269C}, {0x26A0, 0x26A1}, {0x26A7, 0x26A7},
{0x26AA, 0x26AB}, {0x26B0, 0x26B1}, {0x26BD, 0x26BE},
{0x26C4, 0x26C5}, {0x26C8, 0x26C8}, {0x26CE, 0x26CF},
{0x26D1, 0x26D1}, {0x26D3, 0x26D4}, {0x26E9, 0x26EA},
{0x26F0, 0x26F5}, {0x26F7, 0x26FA}, {0x26FD, 0x26FD},
{0x2702, 0x2702}, {0x2705, 0x2705}, {0x2708, 0x270D},
{0x270F, 0x270F}, {0x2712, 0x2712}, {0x2714, 0x2714},
{0x2716, 0x2716}, {0x271D, 0x271D}, {0x2721, 0x2721},
{0x2728, 0x2728}, {0x2733, 0x2734}, {0x2744, 0x2744},
{0x2747, 0x2747}, {0x274C, 0x274C}, {0x274E, 0x274E},
{0x2753, 0x2755}, {0x2757, 0x2757}, {0x2763, 0x2764},
{0x2795, 0x2797}, {0x27A1, 0x27A1}, {0x27B0, 0x27B0},
{0x27BF, 0x27BF}, {0x2934, 0x2935}, {0x2B05, 0x2B07},
{0x2B1B, 0x2B1C}, {0x2B50, 0x2B50}, {0x2B55, 0x2B55},
{0x3030, 0x3030}, {0x303D, 0x303D}, {0x3297, 0x3297},
{0x3299, 0x3299}, {0x1F004, 0x1F004}, {0x1F02C, 0x1F02F},
{0x1F094, 0x1F09F}, {0x1F0AF, 0x1F0B0}, {0x1F0C0, 0x1F0C0},
{0x1F0CF, 0x1F0D0}, {0x1F0F6, 0x1F0FF}, {0x1F170, 0x1F171},
{0x1F17E, 0x1F17F}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A},
{0x1F1AE, 0x1F1E5}, {0x1F201, 0x1F20F}, {0x1F21A, 0x1F21A},
{0x1F22F, 0x1F22F}, {0x1F232, 0x1F23A}, {0x1F23C, 0x1F23F},
{0x1F249, 0x1F25F}, {0x1F266, 0x1F321}, {0x1F324, 0x1F393},
{0x1F396, 0x1F397}, {0x1F399, 0x1F39B}, {0x1F39E, 0x1F3F0},
{0x1F3F3, 0x1F3F5}, {0x1F3F7, 0x1F3FA}, {0x1F400, 0x1F4FD},
{0x1F4FF, 0x1F53D}, {0x1F549, 0x1F54E}, {0x1F550, 0x1F567},
{0x1F56F, 0x1F570}, {0x1F573, 0x1F57A}, {0x1F587, 0x1F587},
{0x1F58A, 0x1F58D}, {0x1F590, 0x1F590}, {0x1F595, 0x1F596},
{0x1F5A4, 0x1F5A5}, {0x1F5A8, 0x1F5A8}, {0x1F5B1, 0x1F5B2},
{0x1F5BC, 0x1F5BC}, {0x1F5C2, 0x1F5C4}, {0x1F5D1, 0x1F5D3},
{0x1F5DC, 0x1F5DE}, {0x1F5E1, 0x1F5E1}, {0x1F5E3, 0x1F5E3},
{0x1F5E8, 0x1F5E8}, {0x1F5EF, 0x1F5EF}, {0x1F5F3, 0x1F5F3},
{0x1F5FA, 0x1F64F}, {0x1F680, 0x1F6C5}, {0x1F6CB, 0x1F6D2},
{0x1F6D5, 0x1F6E5}, {0x1F6E9, 0x1F6E9}, {0x1F6EB, 0x1F6F0},
{0x1F6F3, 0x1F6FF}, {0x1F7DA, 0x1F7FF}, {0x1F80C, 0x1F80F},
{0x1F848, 0x1F84F}, {0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F},
{0x1F8AE, 0x1F8AF}, {0x1F8BC, 0x1F8BF}, {0x1F8C2, 0x1F8CF},
{0x1F8D9, 0x1F8FF}, {0x1F90C, 0x1F93A}, {0x1F93C, 0x1F945},
{0x1F947, 0x1F9FF}, {0x1FA58, 0x1FA5F}, {0x1FA6E, 0x1FAFF},
{0x1FC00, 0x1FFFD},
}

2
vendor/github.com/minio/minio-go/v7/.golangci.yml generated vendored
View File

@@ -4,7 +4,7 @@ linters:
enable:
- durationcheck
- gocritic
- gomodguard
- gomodguard_v2
- govet
- ineffassign
- misspell

5
vendor/github.com/minio/minio-go/v7/Makefile generated vendored
View File

@@ -8,11 +8,8 @@ all: checks
checks: lint test examples functional-test
lint:
@mkdir -p ${GOPATH}/bin
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
@echo "Running $@ check"
@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
go tool golangci-lint run
vet: lint

8
vendor/github.com/minio/minio-go/v7/README.md generated vendored
View File

@@ -214,6 +214,10 @@ The full API Reference is available here.
- [`RemoveObjects`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObjects)
- [`RemoveIncompleteUpload`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveIncompleteUpload)
- [`SelectObjectContent`](https://min.io/docs/minio/linux/developers/go/API.html#SelectObjectContent)
- [`PutObjectAnnotation`](https://min.io/docs/minio/linux/developers/go/API.html#PutObjectAnnotation)
- [`GetObjectAnnotation`](https://min.io/docs/minio/linux/developers/go/API.html#GetObjectAnnotation)
- [`ListObjectAnnotations`](https://min.io/docs/minio/linux/developers/go/API.html#ListObjectAnnotations)
- [`RemoveObjectAnnotation`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObjectAnnotation)
### API Reference : Presigned Operations
@@ -286,6 +290,10 @@ Full Examples
- [removeobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobject.go)
- [removeincompleteupload.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeincompleteupload.go)
- [removeobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjects.go)
- [putobjectannotation.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobjectannotation.go)
- [getobjectannotation.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobjectannotation.go)
- [listobjectannotations.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjectannotations.go)
- [removeobjectannotation.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjectannotation.go)
### Full Examples : Encrypted Object Operations

18
vendor/github.com/minio/minio-go/v7/api-compose-object.go generated vendored
View File

@@ -85,8 +85,22 @@ type CopyDestOptions struct {
// PartSize specifies the part size for multipart copy operations.
// If not specified, defaults to maxPartSize (5 GiB).
PartSize uint64
// AnnotationDirective controls whether the source object's annotations are
// copied to the destination. Valid values are CopyAnnotationsDirective
// ("COPY", the default when unset) and ExcludeAnnotationsDirective
// ("EXCLUDE"). Sent as the x-amz-annotation-directive header.
AnnotationDirective string
}
// Annotation directives for CopyDestOptions.AnnotationDirective.
const (
// CopyAnnotationsDirective copies the source object's annotations to the destination.
CopyAnnotationsDirective = "COPY"
// ExcludeAnnotationsDirective excludes the source object's annotations from the destination.
ExcludeAnnotationsDirective = "EXCLUDE"
)
// Process custom-metadata to remove a `x-amz-meta-` prefix if
// present and validate that keys are distinct (after this
// prefix removal).
@@ -149,6 +163,10 @@ func (opts CopyDestOptions) Marshal(header http.Header) {
header.Set(amzChecksumAlgo, opts.ChecksumType.String())
}
if opts.AnnotationDirective != "" {
header.Set("x-amz-annotation-directive", opts.AnnotationDirective)
}
if opts.ReplaceMetadata {
header.Set("x-amz-metadata-directive", replaceDirective)
for k, v := range filterCustomMeta(opts.UserMetadata) {

16
vendor/github.com/minio/minio-go/v7/api-get-object.go generated vendored
View File

@@ -46,6 +46,22 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o
}
}
if opts.RDMABuffer != nil && c.rdmaEnabled {
n, err := c.getObjectRDMA(ctx, bucketName, objectName, opts)
if err != nil {
return nil, err
}
return &Object{
mutex: &sync.Mutex{},
isClosed: true,
objectInfo: ObjectInfo{
Key: objectName,
Size: n,
},
objectInfoSet: true,
}, nil
}
gctx, cancel := context.WithCancel(ctx)
// Detect if snowball is server location we are talking to.

7
vendor/github.com/minio/minio-go/v7/api-get-options.go generated vendored
View File

@@ -23,6 +23,7 @@ import (
"net/url"
"strconv"
"time"
"unsafe"
"github.com/minio/minio-go/v7/pkg/encrypt"
)
@@ -48,6 +49,12 @@ type GetObjectOptions struct {
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
Checksum bool
// RDMABuffer, when non-nil and Options.EnableRDMA=true, downloads directly
// into a contiguous buffer via libminiocpp.so. The returned *Object's
// Read() returns EOF immediately; bytes-transferred is in Stat().Size.
RDMABuffer unsafe.Pointer
RDMABufferSize int
// To be not used by external applications
Internal AdvancedGetOptions
}

2
vendor/github.com/minio/minio-go/v7/api-inventory-ext.go generated vendored
View File

@@ -276,6 +276,8 @@ type InventoryJobStatus struct {
User string `json:"user"`
AccessKey string `json:"accessKey"`
Schedule string `json:"schedule"`
ScheduleTime string `json:"scheduleTime,omitempty"`
ScheduleTimezone string `json:"scheduleTimezone,omitempty"`
State string `json:"state"`
NextScheduledTime time.Time `json:"nextScheduledTime,omitempty"`
StartTime time.Time `json:"startTime,omitempty"`

282
vendor/github.com/minio/minio-go/v7/api-object-annotation.go generated vendored Normal file
View File

@@ -0,0 +1,282 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2026 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"encoding/xml"
"io"
"net/http"
"net/url"
"time"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
// Object annotations are named payloads (1 byte to 1 MiB of UTF-8 text)
// attached to a specific object version, independent of the object's data.
// Up to 1,000 annotations may be attached per object version.
const (
amzObjectIfMatchHeader = "x-amz-object-if-match"
// maxAnnotationPayloadBytes is the maximum size of a single annotation payload (1 MiB).
maxAnnotationPayloadBytes = 1 << 20
// maxAnnotationNameBytes is the maximum length of an annotation name.
maxAnnotationNameBytes = 512
)
// validateAnnotationName performs a minimal client-side check on the annotation
// name (non-empty, within the length limit). The server enforces the full
// naming rules (allowed characters, reserved prefixes).
func validateAnnotationName(name string) error {
if name == "" {
return errInvalidArgument("annotation name must not be empty")
}
if len(name) > maxAnnotationNameBytes {
return errInvalidArgument("annotation name exceeds 512 bytes")
}
return nil
}
// PutObjectAnnotationOptions configures a PutObjectAnnotation request.
type PutObjectAnnotationOptions struct {
// VersionID targets a specific object version (versioned buckets).
VersionID string
// IfMatch, when set, only applies the annotation if the parent object's
// ETag matches this value (sent as x-amz-object-if-match).
IfMatch string
}
// GetObjectAnnotationOptions configures a GetObjectAnnotation request.
type GetObjectAnnotationOptions struct {
VersionID string
}
// ListObjectAnnotationsOptions configures a ListObjectAnnotations request.
type ListObjectAnnotationsOptions struct {
VersionID string
}
// RemoveObjectAnnotationOptions configures a DeleteObjectAnnotation request.
type RemoveObjectAnnotationOptions struct {
VersionID string
IfMatch string
}
// ObjectAnnotation describes a single annotation as returned by ListObjectAnnotations.
type ObjectAnnotation struct {
Name string
Size int64
ETag string
LastModified time.Time
}
// listObjectAnnotationsOutput maps the ListObjectAnnotations XML response.
type listObjectAnnotationsOutput struct {
XMLName xml.Name `xml:"ListObjectAnnotationsOutput"`
Annotations []struct {
AnnotationName string `xml:"AnnotationName"`
Size int64 `xml:"Size"`
ETag string `xml:"ETag"`
LastModified string `xml:"LastModified"`
} `xml:"Annotation"`
}
func annotationQueryValues(name, versionID string) url.Values {
urlValues := make(url.Values)
urlValues.Set("annotation", "")
if name != "" {
urlValues.Set("annotationName", name)
}
if versionID != "" {
urlValues.Set("versionId", versionID)
}
return urlValues
}
// PutObjectAnnotation creates or overwrites a named annotation on an object
// version. The payload (1 byte to 1 MiB) is streamed directly from the supplied
// ReadSeeker: its size is taken from a seek to the end, so the body is sent with
// an exact Content-Length and never buffered in memory. It returns the
// annotation's ETag. The parent object's ETag is never modified.
func (c *Client) PutObjectAnnotation(ctx context.Context, bucketName, objectName, annotationName string, payload io.ReadSeeker, opts PutObjectAnnotationOptions) (string, error) {
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return "", err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return "", err
}
if err := validateAnnotationName(annotationName); err != nil {
return "", err
}
if payload == nil {
return "", errInvalidArgument("annotation payload must not be nil")
}
// Derive the payload size from the seeker and enforce the limits before
// uploading; the server reads the body raw, so it is sent with UNSIGNED-PAYLOAD
// (no streaming-chunked signature) and streamed without buffering.
size, err := payload.Seek(0, io.SeekEnd)
if err != nil {
return "", err
}
if _, err := payload.Seek(0, io.SeekStart); err != nil {
return "", err
}
if size == 0 {
return "", errInvalidArgument("annotation payload must be at least 1 byte")
}
if size > maxAnnotationPayloadBytes {
return "", errInvalidArgument("annotation payload exceeds the 1 MiB maximum")
}
headers := make(http.Header)
if opts.IfMatch != "" {
headers.Set(amzObjectIfMatchHeader, opts.IfMatch)
}
resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: annotationQueryValues(annotationName, opts.VersionID),
contentBody: payload,
contentLength: size,
customHeader: headers,
})
defer closeResponse(resp)
if err != nil {
return "", err
}
if resp != nil && resp.StatusCode != http.StatusOK {
return "", httpRespToErrorResponse(resp, bucketName, objectName)
}
return trimEtag(resp.Header.Get("ETag")), nil
}
// GetObjectAnnotation returns the payload of a single named annotation as a
// stream. The returned ReadCloser is the response body; the caller must Close it
// once the payload has been read so the underlying connection can be reused. The
// payload is server-capped at 1 MiB.
func (c *Client) GetObjectAnnotation(ctx context.Context, bucketName, objectName, annotationName string, opts GetObjectAnnotationOptions) (io.ReadCloser, error) {
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return nil, err
}
if err := validateAnnotationName(annotationName); err != nil {
return nil, err
}
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: annotationQueryValues(annotationName, opts.VersionID),
})
if err != nil {
closeResponse(resp)
return nil, err
}
if resp != nil && resp.StatusCode != http.StatusOK {
defer closeResponse(resp)
return nil, httpRespToErrorResponse(resp, bucketName, objectName)
}
return resp.Body, nil
}
// ListObjectAnnotations returns all annotations attached to an object version.
func (c *Client) ListObjectAnnotations(ctx context.Context, bucketName, objectName string, opts ListObjectAnnotationsOptions) ([]ObjectAnnotation, error) {
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return nil, err
}
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: annotationQueryValues("", opts.VersionID),
})
defer closeResponse(resp)
if err != nil {
return nil, err
}
if resp != nil && resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucketName, objectName)
}
var out listObjectAnnotationsOutput
if err := xml.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, err
}
annotations := make([]ObjectAnnotation, 0, len(out.Annotations))
for _, a := range out.Annotations {
// LastModified is an RFC3339 timestamp. A malformed value from the
// server leaves LastModified as the zero time rather than failing the
// entire listing for one bad entry.
var lastModified time.Time
if a.LastModified != "" {
lastModified, _ = time.Parse(time.RFC3339, a.LastModified)
}
annotations = append(annotations, ObjectAnnotation{
Name: a.AnnotationName,
Size: a.Size,
ETag: trimEtag(a.ETag),
LastModified: lastModified,
})
}
return annotations, nil
}
// RemoveObjectAnnotation permanently deletes a single named annotation. Deletion
// is irreversible: annotations have no version history.
func (c *Client) RemoveObjectAnnotation(ctx context.Context, bucketName, objectName, annotationName string, opts RemoveObjectAnnotationOptions) error {
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return err
}
if err := validateAnnotationName(annotationName); err != nil {
return err
}
headers := make(http.Header)
if opts.IfMatch != "" {
headers.Set(amzObjectIfMatchHeader, opts.IfMatch)
}
resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: annotationQueryValues(annotationName, opts.VersionID),
customHeader: headers,
})
defer closeResponse(resp)
if err != nil {
return err
}
if resp != nil && resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, objectName)
}
return nil
}

10
vendor/github.com/minio/minio-go/v7/api-put-object.go generated vendored
View File

@@ -28,6 +28,7 @@ import (
"sort"
"strings"
"time"
"unsafe"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -111,6 +112,12 @@ type PutObjectOptions struct {
ConcurrentStreamParts bool
Internal AdvancedPutOptions
// RDMABuffer, when non-nil and Options.EnableRDMA=true, selects the RDMA
// path via libminiocpp.so. Must reference RDMABufferSize contiguous bytes.
// When set, the reader / size args to PutObject are ignored.
RDMABuffer unsafe.Pointer
RDMABufferSize int
customHeaders http.Header
}
@@ -322,6 +329,9 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part
func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64,
opts PutObjectOptions,
) (info UploadInfo, err error) {
if opts.RDMABuffer != nil && c.rdmaEnabled {
return c.putObjectRDMA(ctx, bucketName, objectName, opts)
}
if size < 0 && opts.DisableMultipart {
return UploadInfo{}, errors.New("object size must be provided with disable multipart upload")
}

15
vendor/github.com/minio/minio-go/v7/api.go generated vendored
View File

@@ -109,6 +109,15 @@ type Client struct {
trailingHeaderSupport bool
maxRetries int
// RDMA dispatch state. rdmaEnabled mirrors Options.EnableRDMA;
// the rest are only touched by rdma.go (built with -tags=rdma) but
// have to live on the struct so the stub and the tagged build share
// one shape.
rdmaEnabled bool
rdmaOnce sync.Once //nolint:unused
rdmaHandle *rdmaClientHandle //nolint:unused
rdmaInitErr error //nolint:unused
}
// Options for New method
@@ -156,6 +165,11 @@ type Options struct {
// Number of times a request is retried. Defaults to 10 retries if this option is not configured.
// Set to 1 to disable retries.
MaxRetries int
// EnableRDMA causes PutObject / GetObject to dispatch to libminiocpp.so
// when the caller supplies PutObjectOptions.RDMABuffer / GetObjectOptions.RDMABuffer.
// No-op unless built with -tags=rdma.
EnableRDMA bool
}
// Global constants.
@@ -311,6 +325,7 @@ func privateNew(endpoint string, opts *Options) (*Client, error) {
}
clnt.trailingHeaderSupport = opts.TrailingHeaders && clnt.overrideSignerType.IsV4()
clnt.rdmaEnabled = opts.EnableRDMA
// Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined
// by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints.

9
vendor/github.com/minio/minio-go/v7/create-session.go generated vendored
View File

@@ -88,14 +88,15 @@ func (c *Client) CreateSession(ctx context.Context, bucketName string, sessionMo
return credentials.Value{}, err
}
defer c.bucketSessionCache.Set(bucketName, cred)
return credentials.Value{
cred = credentials.Value{
AccessKeyID: credSession.Credentials.AccessKey,
SecretAccessKey: credSession.Credentials.SecretKey,
SessionToken: credSession.Credentials.SessionToken,
Expiration: credSession.Credentials.Expiration,
}, nil
}
c.bucketSessionCache.Set(bucketName, cred)
return cred, nil
}
// createSessionRequest - Wrapper creates a new CreateSession request.

4
vendor/github.com/minio/minio-go/v7/endpoints.go generated vendored
View File

@@ -244,6 +244,10 @@ var awsS3EndpointMap = map[string]awsS3Endpoint{
"s3.ap-east-2.amazonaws.com",
"s3.dualstack.ap-east-2.amazonaws.com",
},
"ap-southeast-6": {
"s3.ap-southeast-6.amazonaws.com",
"s3.dualstack.ap-southeast-6.amazonaws.com",
},
}
// getS3ExpressEndpoint get Amazon S3 Express endpoing based on the region

105
vendor/github.com/minio/minio-go/v7/functional_tests.go generated vendored
View File

@@ -1838,6 +1838,110 @@ func testRemoveObjectsWithVersioning() {
logSuccess(testName, function, args, startTime)
}
// Tests {Put,Get,List,Remove}ObjectAnnotation APIs end to end. Servers that do
// not implement annotations are detected and skipped (logIgnored) rather than
// failed: a non-implementing server silently treats the ?annotation request as
// a plain object write, which is caught here via the parent ETag invariant.
func testObjectAnnotations() {
startTime := time.Now()
testName := getFuncName()
function := "{Put,Get,List,Remove}ObjectAnnotation()"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
}
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
args["bucketName"] = bucketName
args["objectName"] = objectName
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil {
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
return
}
defer cleanupBucket(bucketName, c)
const parentContent = "annotation-parent-content"
ui, err := c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(parentContent), int64(len(parentContent)), minio.PutObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject (parent) failed", err)
return
}
annName := "model.labels.json"
annPayload := []byte(`{"label":"cat","score":0.98}`)
args["annotationName"] = annName
_, err = c.PutObjectAnnotation(context.Background(), bucketName, objectName, annName, bytes.NewReader(annPayload), minio.PutObjectAnnotationOptions{})
if err != nil {
if isErrNotImplemented(err) {
logIgnored(testName, function, args, startTime, "PutObjectAnnotation")
return
}
logError(testName, function, args, startTime, "", "PutObjectAnnotation failed", err)
return
}
// On a server without annotation support the request falls through to a
// regular PutObject, changing the parent ETag. Treat that as unsupported.
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "StatObject failed", err)
return
}
if st.ETag != ui.ETag {
logIgnored(testName, function, args, startTime, "PutObjectAnnotation")
return
}
annReader, err := c.GetObjectAnnotation(context.Background(), bucketName, objectName, annName, minio.GetObjectAnnotationOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "GetObjectAnnotation failed", err)
return
}
got, err := io.ReadAll(annReader)
annReader.Close()
if err != nil {
logError(testName, function, args, startTime, "", "GetObjectAnnotation read failed", err)
return
}
if !bytes.Equal(got, annPayload) {
logError(testName, function, args, startTime, "", "GetObjectAnnotation payload mismatch", fmt.Errorf("got %q want %q", got, annPayload))
return
}
anns, err := c.ListObjectAnnotations(context.Background(), bucketName, objectName, minio.ListObjectAnnotationsOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "ListObjectAnnotations failed", err)
return
}
if len(anns) != 1 || anns[0].Name != annName {
logError(testName, function, args, startTime, "", "ListObjectAnnotations unexpected result", fmt.Errorf("got %+v", anns))
return
}
if err = c.RemoveObjectAnnotation(context.Background(), bucketName, objectName, annName, minio.RemoveObjectAnnotationOptions{}); err != nil {
logError(testName, function, args, startTime, "", "RemoveObjectAnnotation failed", err)
return
}
anns, err = c.ListObjectAnnotations(context.Background(), bucketName, objectName, minio.ListObjectAnnotationsOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "ListObjectAnnotations (after remove) failed", err)
return
}
if len(anns) != 0 {
logError(testName, function, args, startTime, "", "annotation not removed", fmt.Errorf("remaining %d", len(anns)))
return
}
logSuccess(testName, function, args, startTime)
}
func testObjectTaggingWithVersioning() {
// initialize logging params
startTime := time.Now()
@@ -15026,6 +15130,7 @@ func main() {
testRemoveObjectWithVersioning()
testRemoveObjectsWithVersioning()
testObjectTaggingWithVersioning()
testObjectAnnotations()
testTrailingChecksums()
testPutObjectWithAutomaticChecksums()
testGetBucketTagging()

2
vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go generated vendored
View File

@@ -26,7 +26,7 @@ import (
"strings"
"time"
"github.com/go-ini/ini"
"gopkg.in/ini.v1"
)
// A externalProcessCredentials stores the output of a credential_process

140
vendor/github.com/minio/minio-go/v7/rdma.go generated vendored Normal file
View File

@@ -0,0 +1,140 @@
//go:build rdma
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2024-2026 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* SPDX-License-Identifier: Apache-2.0
*/
package minio
// #cgo CFLAGS: -DMINIO_CPP_RDMA
// #cgo CXXFLAGS: --std=c++17 -DMINIO_CPP_RDMA
// #cgo LDFLAGS: -lminiocpp
// #include <stdlib.h>
// #include <miniocpp/c_api.h>
import "C"
import (
"context"
"errors"
"fmt"
"unsafe"
)
var ErrRDMANotConnected = errors.New("RDMA infrastructure not connected")
type rdmaClientHandle struct {
cptr *C.miniocpp_client
}
func newRDMAClient(c *Client) (*rdmaClientHandle, error) {
creds, err := c.credsProvider.Get()
if err != nil {
return nil, fmt.Errorf("RDMA: credentials: %w", err)
}
endpoint := C.CString(c.endpointURL.Host)
defer C.free(unsafe.Pointer(endpoint))
region := C.CString(c.region)
defer C.free(unsafe.Pointer(region))
accessKey := C.CString(creds.AccessKeyID)
defer C.free(unsafe.Pointer(accessKey))
secretKey := C.CString(creds.SecretAccessKey)
defer C.free(unsafe.Pointer(secretKey))
sessionToken := C.CString(creds.SessionToken)
defer C.free(unsafe.Pointer(sessionToken))
var useHTTPS C.int
if c.secure {
useHTTPS = 1
}
cptr := C.miniocpp_client_new(endpoint, region, accessKey, secretKey,
sessionToken, useHTTPS)
if cptr == nil {
return nil, fmt.Errorf("RDMA: %s", lastRDMAError())
}
return &rdmaClientHandle{cptr: cptr}, nil
}
func (c *Client) putObjectRDMA(_ context.Context, bucketName, objectName string,
opts PutObjectOptions,
) (UploadInfo, error) {
h, err := c.rdma()
if err != nil {
return UploadInfo{}, err
}
bucketC := C.CString(bucketName)
defer C.free(unsafe.Pointer(bucketC))
objectC := C.CString(objectName)
defer C.free(unsafe.Pointer(objectC))
var etagBuf, checksumBuf [64]C.char
n := C.miniocpp_put_object(h.cptr, bucketC, objectC,
opts.RDMABuffer, C.size_t(opts.RDMABufferSize),
nil, nil, &etagBuf[0], &checksumBuf[0])
if n < 0 {
return UploadInfo{}, fmt.Errorf("RDMA put: %s", lastRDMAError())
}
return UploadInfo{
Bucket: bucketName,
Key: objectName,
Size: int64(n),
ETag: C.GoString(&etagBuf[0]),
ChecksumCRC64NVME: C.GoString(&checksumBuf[0]),
}, nil
}
func (c *Client) getObjectRDMA(_ context.Context, bucketName, objectName string,
opts GetObjectOptions,
) (int64, error) {
h, err := c.rdma()
if err != nil {
return 0, err
}
bucketC := C.CString(bucketName)
defer C.free(unsafe.Pointer(bucketC))
objectC := C.CString(objectName)
defer C.free(unsafe.Pointer(objectC))
n := C.miniocpp_get_object(h.cptr, bucketC, objectC,
opts.RDMABuffer, C.size_t(opts.RDMABufferSize), nil, nil)
if n < 0 {
return 0, fmt.Errorf("RDMA get: %s", lastRDMAError())
}
return int64(n), nil
}
func (c *Client) rdma() (*rdmaClientHandle, error) {
c.rdmaOnce.Do(func() {
c.rdmaHandle, c.rdmaInitErr = newRDMAClient(c)
})
return c.rdmaHandle, c.rdmaInitErr
}
func lastRDMAError() string {
msg := C.miniocpp_last_error()
if msg == nil {
return "unknown error"
}
return C.GoString(msg)
}
// AlignedBuffer allocates a page-aligned buffer of n bytes for use as
// PutObjectOptions.RDMABuffer / GetObjectOptions.RDMABuffer. Release with
// FreeAlignedBuffer. Returns nil on allocation failure.
func AlignedBuffer(n int) unsafe.Pointer {
return C.miniocpp_alloc_aligned(C.size_t(n))
}
// FreeAlignedBuffer releases a buffer from AlignedBuffer.
func FreeAlignedBuffer(p unsafe.Pointer) { C.miniocpp_free_aligned(p) }
// IsRDMAAvailable reports whether cuObj is connected to a cuObjServer.
func IsRDMAAvailable() bool { return C.miniocpp_rdma_available() != 0 }

42
vendor/github.com/minio/minio-go/v7/rdma_stub.go generated vendored Normal file
View File

@@ -0,0 +1,42 @@
//go:build !rdma
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2024-2026 MinIO, Inc.
*
* SPDX-License-Identifier: Apache-2.0
*/
package minio
import (
"context"
"errors"
"unsafe"
)
// ErrRDMANotCompiled is returned when RDMA dispatch is requested but the
// binary was built without -tags=rdma.
var ErrRDMANotCompiled = errors.New("RDMA support not compiled in (build with -tags=rdma)")
// rdmaClientHandle is the no-tag placeholder type so Client (in api.go)
// has a stable shape regardless of build tags. The rdma.go variant
// defines the real struct.
type rdmaClientHandle struct{} //nolint:unused
func (c *Client) putObjectRDMA(_ context.Context, _, _ string, _ PutObjectOptions) (UploadInfo, error) {
return UploadInfo{}, ErrRDMANotCompiled
}
func (c *Client) getObjectRDMA(_ context.Context, _, _ string, _ GetObjectOptions) (int64, error) {
return 0, ErrRDMANotCompiled
}
// AlignedBuffer is unavailable without -tags=rdma; returns nil.
func AlignedBuffer(_ int) unsafe.Pointer { return nil }
// FreeAlignedBuffer is a no-op without -tags=rdma.
func FreeAlignedBuffer(_ unsafe.Pointer) {}
// IsRDMAAvailable always returns false without -tags=rdma.
func IsRDMAAvailable() bool { return false }

1
vendor/github.com/minio/minio-go/v7/utils.go generated vendored
View File

@@ -517,6 +517,7 @@ var supportedHeaders = map[string]bool{
"x-amz-website-redirect-location": true,
"x-amz-object-lock-mode": true,
"x-amz-metadata-directive": true,
"x-amz-annotation-directive": true,
"x-amz-object-lock-retain-until-date": true,
"expires": true,
"x-amz-replication-status": true,

51
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net/permissions.go generated vendored Normal file
View File

@@ -0,0 +1,51 @@
// Copyright 2018-2026 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package net
import (
"context"
"encoding/json"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/conversions"
)
// WebDAVPermissions derives the WebDAV permissions string (the OC-Perm header value) for a
// resource. It is shared by the ocdav gateway's creation-with-upload response and the
// dataprovider's chunked TUS finalize response, so the two report the same permissions.
func WebDAVPermissions(ctx context.Context, ri *provider.ResourceInfo) string {
isPublic := false
if o := ri.GetOpaque(); o != nil && o.Map != nil {
if e := o.Map["link-share"]; e != nil && e.Decoder == "json" {
ls := &link.PublicShare{}
_ = json.Unmarshal(e.Value, ls)
isPublic = ls != nil
}
}
isShared := !IsCurrentUserOwnerOrManager(ctx, ri.GetOwner(), ri)
role := conversions.RoleFromResourcePermissions(ri.GetPermissionSet(), isPublic)
return role.WebDAVPermissions(
ri.GetType() == provider.ResourceType_RESOURCE_TYPE_CONTAINER,
isShared,
false,
isPublic,
)
}

21
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/tus.go generated vendored
View File

@@ -20,7 +20,6 @@ package ocdav
import (
"context"
"encoding/json"
"io"
"net/http"
"path"
@@ -30,14 +29,12 @@ import (
"time"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/errors"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/spacelookup"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/conversions"
"github.com/opencloud-eu/reva/v2/pkg/rhttp"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -374,23 +371,7 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http.
return
}
// get WebDav permissions for file
isPublic := false
if info.Opaque != nil && info.Opaque.Map != nil {
if info.Opaque.Map["link-share"] != nil && info.Opaque.Map["link-share"].Decoder == "json" {
ls := &link.PublicShare{}
_ = json.Unmarshal(info.Opaque.Map["link-share"].Value, ls)
isPublic = ls != nil
}
}
isShared := !net.IsCurrentUserOwnerOrManager(ctx, info.Owner, info)
role := conversions.RoleFromResourcePermissions(info.PermissionSet, isPublic)
permissions := role.WebDAVPermissions(
info.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER,
isShared,
false,
isPublic,
)
permissions := net.WebDAVPermissions(ctx, info)
w.Header().Set(net.HeaderContentType, info.MimeType)
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(info.Id))

39
vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go generated vendored
View File

@@ -108,6 +108,9 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
StoreComposer: composer,
NotifyCompleteUploads: true,
Logger: slog.New(tusdLogger{log: m.log}),
// Add the finalized resource's etag and permissions to the last chunk's response
// (see preFinishResponseCallback). https://github.com/opencloud-eu/opencloud/issues/2409
PreFinishResponseCallback: m.preFinishResponseCallback(fs),
}
if m.conf.CorsEnabled {
@@ -207,6 +210,42 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
return h, nil
}
// preFinishResponseCallback returns the tusd callback that runs when an upload completes. Before
// the final response is written, it looks up the finalized resource and attaches its etag and
// WebDAV permissions to the response headers, so clients do not need a follow-up PROPFIND after
// the last chunk. It mirrors the etag and permissions the ocdav gateway already produces for the
// creation-with-upload path. The storage driver resolves the resource as the upload's executant
// (the data path's transfer token carries no user identity), so this layer does not reconstruct
// the user. It is best effort: it logs any lookup failure and still completes the upload (the
// client falls back to a PROPFIND). https://github.com/opencloud-eu/opencloud/issues/2409
func (m *manager) preFinishResponseCallback(fs storage.FS) func(tusd.HookEvent) (tusd.HTTPResponse, error) {
resolver, ok := fs.(storage.UploadSessionResolver)
return func(hook tusd.HookEvent) (tusd.HTTPResponse, error) {
resp := tusd.HTTPResponse{Header: tusd.HTTPHeader{}}
if !ok {
// the storage driver cannot resolve the upload; the client falls back to a PROPFIND
return resp, nil
}
ri, ctx, err := resolver.ResolveUpload(hook.Context, hook.Upload)
if err != nil || ri == nil {
// The stat can fail for a finished upload (for example an expired upload token or a
// removed node); the client recovers with a PROPFIND, so this is a warning, not an error.
m.log.Warn().Err(err).Str("uploadid", hook.Upload.ID).Msg("could not stat finished upload, not setting etag/permission headers")
return resp, nil
}
if etag := ri.GetEtag(); etag != "" {
resp.Header[net.HeaderOCETag] = etag
resp.Header[net.HeaderETag] = etag
}
resp.Header[net.HeaderOCPermissions] = net.WebDAVPermissions(ctx, ri)
return resp, nil
}
}
func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
id := path.Base(r.URL.Path)

89
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go generated vendored
View File

@@ -26,28 +26,34 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/pkg/errors"
"github.com/pkg/xattr"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
)
const (
TMPDir = ".oc-tmp"
TMPDir = ".oc-tmp"
fsyncEvery = 16 << 20 // 16 MiB
)
// Blobstore provides an interface to an filesystem based blobstore
type Blobstore struct {
root string
canUseRenameForUpload bool
}
// New returns a new Blobstore
func New(root string) (*Blobstore, error) {
return &Blobstore{
root: root,
root: root,
canUseRenameForUpload: true, // let's assume the upload area is on the same device as the blobstore root by default
}, nil
}
@@ -61,29 +67,40 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
return err
}
sourceFile, err := os.Open(source)
if err != nil {
return fmt.Errorf("failed to open source file '%s': %v", source, err)
}
defer func() {
_ = sourceFile.Close()
}()
tempFile, err := os.OpenFile(tempName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("unable to create temp file '%s': %v", tempName, err)
if bs.canUseRenameForUpload {
err := os.Rename(source, tempName)
switch {
case err == nil:
// continue
case errors.Is(err, syscall.EXDEV):
// the upload and target file are on different devices, we need to copy the file instead of renaming it
bs.canUseRenameForUpload = false
default:
return fmt.Errorf("failed to move source file '%s' to temp file '%s' - %v", source, tempName, err)
}
}
if _, err := tempFile.ReadFrom(sourceFile); err != nil {
return fmt.Errorf("failed to write data from source file '%s' to temp file '%s' - %v", source, tempName, err)
}
if !bs.canUseRenameForUpload {
sourceFile, err := os.Open(source)
if err != nil {
return fmt.Errorf("failed to open source file '%s': %v", source, err)
}
defer func() {
_ = sourceFile.Close()
}()
if err := tempFile.Sync(); err != nil {
return fmt.Errorf("failed to sync temp file '%s' - %v", tempName, err)
}
tempFile, err := os.OpenFile(tempName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("unable to create temp file '%s': %v", tempName, err)
}
if err := tempFile.Close(); err != nil {
return fmt.Errorf("failed to close temp file '%s' - %v", tempName, err)
if err := copyWithPeriodicSync(tempFile, sourceFile); err != nil {
return fmt.Errorf("failed to copy source file '%s' to temp file '%s' - %v", source, tempName, err)
}
if err := tempFile.Close(); err != nil {
return fmt.Errorf("failed to close temp file '%s' - %v", tempName, err)
}
}
nodeAttributes, err := n.Xattrs(context.Background())
@@ -137,11 +154,15 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
}
// also "upload" the file to a local path, e.g., for keeping the "current" version of the file
if err := os.MkdirAll(filepath.Dir(copyTarget), 0700); err != nil {
return err
sourceFile, err := os.Open(source)
if err != nil {
return errors.Wrapf(err, "could not open source file '%s' for reading", source)
}
defer func() {
_ = sourceFile.Close()
}()
if _, err := sourceFile.Seek(0, 0); err != nil {
if err := os.MkdirAll(filepath.Dir(copyTarget), 0700); err != nil {
return err
}
@@ -153,7 +174,7 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
_ = copyFile.Close()
}()
if _, err := copyFile.ReadFrom(sourceFile); err != nil {
if err := copyWithPeriodicSync(copyFile, sourceFile); err != nil {
return errors.Wrapf(err, "could not write blob copy of '%s' to '%s'", n.InternalPath(), copyTarget)
}
@@ -173,3 +194,21 @@ func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) {
func (bs *Blobstore) Delete(node *node.Node) error {
return nil
}
func copyWithPeriodicSync(dst, src *os.File) error {
for {
n, err := io.CopyN(dst, src, fsyncEvery)
if n > 0 {
if serr := disk.Fdatasync(dst); serr != nil {
return serr
}
}
if err == io.EOF {
_ = dst.Sync()
return nil
}
if err != nil {
return err
}
}
}

51
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go generated vendored
View File

@@ -41,7 +41,6 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
@@ -398,46 +397,14 @@ func refFromCS3(b []byte) (*provider.Reference, error) {
}, nil
}
// CopyMetadata copies all extended attributes from source to target.
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
// For the source file, a shared lock is acquired.
// NOTE: target resource will be write locked!
func (lu *Lookup) CopyMetadata(ctx context.Context, src, target metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) {
// Acquire a read log on the source node
// write lock existing node before reading treesize or tree time
lock, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(src), os.O_RDONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
if err != nil {
return errors.Wrap(err, "xattrs: Unable to lock source to read")
}
defer func() {
rerr := lock.Close()
// if err is non nil we do not overwrite that
if err == nil {
err = rerr
}
}()
return lu.CopyMetadataWithSourceLock(ctx, src, target, filter, lock, acquireTargetLock)
}
// CopyMetadataWithSourceLock copies all extended attributes from source to target.
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
// For the source file, a matching lockedfile is required.
// NOTE: target resource will be write locked!
func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, src, target metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) {
switch {
case lockedSource == nil:
return errors.New("no lock provided")
case lockedSource.Name() != lu.MetadataBackend().LockfilePath(src):
return errors.New("lockpath does not match filepath")
}
attrs, err := lu.metadataBackend.AllWithLockedSource(ctx, src, lockedSource)
// CopyMetadata copies all extended attributes from source to target. The optional filter
// function can be used to filter by attribute name, e.g. by checking a prefix.
//
// Locking is derived from the nodes: the source's metadata lock is acquired for the duration
// of the read unless the caller already holds it (src.LockHeld()), and the target is write
// locked while its attributes are written unless the caller already holds its lock
func (lu *Lookup) CopyMetadata(ctx context.Context, src, target metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool)) (err error) {
attrs, err := lu.metadataBackend.All(ctx, src)
if err != nil {
return err
}
@@ -453,7 +420,7 @@ func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, src, target me
newAttrs[attrName] = val
}
return lu.MetadataBackend().SetMultiple(ctx, target, newAttrs, acquireTargetLock)
return lu.MetadataBackend().SetMultiple(ctx, target, newAttrs)
}
// GenerateSpaceID generates a space id for the given space type and owner

13
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go generated vendored
View File

@@ -26,9 +26,11 @@ import (
"syscall"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/rs/zerolog"
tusd "github.com/tus/tusd/v2/pkg/handler"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storage"
@@ -244,6 +246,17 @@ func (fs *posixFS) ListUploadSessions(ctx context.Context, filter storage.Upload
return fs.FS.(storage.UploadSessionLister).ListUploadSessions(ctx, filter)
}
// ResolveUpload resolves a finished upload to the resource it produced, read as the upload's
// executant. It is best effort: if the wrapped storage does not resolve uploads it returns
// NotSupported rather than panicking, so the tus finalize falls back to no headers.
func (fs *posixFS) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) {
r, ok := fs.FS.(storage.UploadSessionResolver)
if !ok {
return nil, ctx, errtypes.NotSupported("storage does not resolve upload sessions")
}
return r.ResolveUpload(ctx, info)
}
// UseIn tells the tus upload middleware which extensions it supports.
func (fs *posixFS) UseIn(composer *tusd.StoreComposer) {
fs.FS.(storage.ComposableFS).UseIn(composer)

4
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager/timemanager.go generated vendored
View File

@@ -68,7 +68,7 @@ func (m *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error) {
// If tmtime is nil, the tmtime attribute is removed.
func (m *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
if tmtime == nil {
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true)
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr)
}
return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano))
}
@@ -113,7 +113,7 @@ func (m *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time, er
// If t is nil, the dtime attribute is removed.
func (m *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) {
if t == nil {
return n.RemoveXattr(ctx, prefixes.DTimeAttr, true)
return n.RemoveXattr(ctx, prefixes.DTimeAttr)
}
return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano))
}

16
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/trashbin/trashbin.go generated vendored
View File

@@ -40,6 +40,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -66,6 +67,7 @@ type trashNode struct {
spaceID string
id string
path string
lock goroutinelock.Lock
}
func (tn *trashNode) GetSpaceID() string {
@@ -80,6 +82,18 @@ func (tn *trashNode) InternalPath() string {
return tn.path
}
func (tn *trashNode) LockHeld() bool {
return tn.lock.Held()
}
func (tn *trashNode) SetLockHeld(held bool) {
if held {
tn.lock.Hold()
return
}
tn.lock.Release()
}
const (
trashHeader = `[Trash Info]`
timeFormat = "2006-01-02T15:04:05"
@@ -345,7 +359,7 @@ func (tb *Trashbin) RestoreRecycleItem(ctx context.Context, spaceID string, key,
if err = tb.lu.MetadataBackend().SetMultiple(ctx, trashedNode, map[string][]byte{
prefixes.NameAttr: []byte(filepath.Base(restorePath)),
prefixes.ParentidAttr: []byte(parentID),
}, true); err != nil {
}); err != nil {
return nil, fmt.Errorf("posixfs: failed to update trashed node metadata: %w", err)
}

50
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go generated vendored
View File

@@ -42,6 +42,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
@@ -72,20 +73,33 @@ type assimilationNode struct {
path string
nodeId string
spaceID string
lock goroutinelock.Lock
}
func (d assimilationNode) GetID() string {
func (d *assimilationNode) GetID() string {
return d.nodeId
}
func (d assimilationNode) GetSpaceID() string {
func (d *assimilationNode) GetSpaceID() string {
return d.spaceID
}
func (d assimilationNode) InternalPath() string {
func (d *assimilationNode) InternalPath() string {
return d.path
}
func (d *assimilationNode) LockHeld() bool {
return d.lock.Held()
}
func (d *assimilationNode) SetLockHeld(held bool) {
if held {
d.lock.Hold()
return
}
d.lock.Release()
}
// NewScanDebouncer returns a new SpaceDebouncer instance
func NewScanDebouncer(d time.Duration, f func(item scanItem)) *ScanDebouncer {
return &ScanDebouncer{
@@ -673,7 +687,8 @@ func (t *Tree) assimilate(item scanItem) error {
func (t *Tree) updateFile(path, id, spaceID string, fi fs.FileInfo) (fs.FileInfo, node.Attributes, error) {
retries := 1
parentID := ""
bn := assimilationNode{spaceID: spaceID, nodeId: id, path: path}
bn := &assimilationNode{spaceID: spaceID, nodeId: id, path: path}
bn.SetLockHeld(true)
assimilate:
if id != spaceID {
// read parent
@@ -714,7 +729,7 @@ assimilate:
}
}
attrs, err := t.lookup.MetadataBackend().AllWithLockedSource(context.Background(), bn, nil)
attrs, err := t.lookup.MetadataBackend().All(context.Background(), bn)
if err != nil && !metadata.IsAttrUnset(err) {
return nil, nil, errors.Wrap(err, "failed to get item attribs")
}
@@ -788,6 +803,8 @@ assimilate:
go func() {
// Copy the previous current version to a revision
currentNode := node.NewBaseNode(n.SpaceID, n.ID+node.CurrentIDDelimiter, t.lookup)
currentNode.SetLockHeld(true)
currentPath := currentNode.InternalPath()
stat, err := os.Stat(currentPath)
if err == nil {
@@ -834,7 +851,7 @@ assimilate:
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
})
if err != nil {
t.log.Error().Err(err).Str("currentPath", currentPath).Str("path", path).Msg("failed to copy xattrs to 'current' file")
return
@@ -848,7 +865,7 @@ assimilate:
}
t.log.Debug().Str("path", path).Interface("attributes", attributes).Msg("setting attributes")
err = t.lookup.MetadataBackend().SetMultiple(context.Background(), bn, attributes, false)
err = t.lookup.MetadataBackend().SetMultiple(context.Background(), bn, attributes)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to set attributes")
}
@@ -856,7 +873,7 @@ assimilate:
// clear the status attribute if it was set before, if there was any upload to this file in progress
// it needs notice that this file was changes meanwhile.
if _, ok := previousAttribs[prefixes.StatusPrefix]; ok {
err = t.lookup.MetadataBackend().Remove(context.Background(), bn, prefixes.StatusPrefix, false)
err = t.lookup.MetadataBackend().Remove(context.Background(), bn, prefixes.StatusPrefix)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to clear status attribute")
}
@@ -897,14 +914,19 @@ func (t *Tree) WarmupIDCache(root string, assimilate, onlyDirty bool) error {
}
// skip irrelevant files
if !t.Ignorer.IsSpaceRoot(path) && t.Ignorer.IsIgnored(path) {
if info.IsDir() {
if t.Ignorer.IsIgnored(path) {
switch {
case t.Ignorer.IsSpaceRoot(path):
// ignore the space root itself, but do not skip the whole tree
return nil
case t.Ignorer.IsRootPath(path):
// ignor the root path itself, but do not skip the whole tree
return nil
case info.IsDir():
return filepath.SkipDir
default:
return nil
}
return nil
}
if t.Ignorer.IsRootPath(path) {
return nil // ignore the root paths
}
if !info.IsDir() && !info.Mode().IsRegular() {

14
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/revisions.go generated vendored
View File

@@ -30,7 +30,6 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
@@ -46,7 +45,7 @@ import (
// can be kept in the same location.
// CreateRevision creates a new version of the node
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string, f *lockedfile.File) (string, error) {
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string) (string, error) {
revNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+version, tp.lookup)
versionPath := revNode.InternalPath()
@@ -119,13 +118,13 @@ func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string
}
// copy blob metadata to version node
if err := tp.lookup.CopyMetadataWithSourceLock(ctx, n, revNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
if err := tp.lookup.CopyMetadata(ctx, n, revNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr ||
attributeName == prefixes.MTimeAttr
}, f, true); err != nil {
}); err != nil {
return "", err
}
@@ -300,7 +299,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, srcNode, targetNode metadat
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
})
if err != nil {
return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error())
}
@@ -317,8 +316,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, srcNode, targetNode metadat
err = tp.lookup.MetadataBackend().SetMultiple(ctx, targetNode,
map[string][]byte{
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
},
false)
})
if err != nil {
return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error())
}
@@ -350,7 +348,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, srcNode, targetNode metadat
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
})
if err != nil {
return errtypes.InternalError("failed to copy xattrs to 'current' file: " + err.Error())
}

13
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go generated vendored
View File

@@ -176,6 +176,9 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin *
watchPath = o.Root
}
if o.ScanDebounceDelay == 0 {
log.Warn().Msg("'scan_debounce_delay' is set to 0. This is not recommended for production setups.")
}
go t.watcher.Watch(watchPath)
go t.workScanQueue()
}
@@ -394,7 +397,7 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
attributes[prefixes.MTimeAttr] = []byte(mtime.UTC().Format(time.RFC3339Nano))
}
err = n.SetXattrsWithContext(ctx, attributes, false)
err = n.SetXattrsWithContext(ctx, attributes)
if err != nil {
return err
}
@@ -477,7 +480,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node)
attribs := node.Attributes{}
attribs.SetString(prefixes.ParentidAttr, newNode.ParentID)
attribs.SetString(prefixes.NameAttr, newNode.Name)
if err := newNode.SetXattrsWithContext(ctx, attribs, true); err != nil {
if err := newNode.SetXattrsWithContext(ctx, attribs); err != nil {
return errors.Wrap(err, "posixfs: could not update node attributes")
}
@@ -726,7 +729,7 @@ func (t *Tree) WriteBlob(n *node.Node, source string) error {
attrs.SetString(prefixes.BlobIDAttr, n.BlobID)
attrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize)
err := t.lookup.MetadataBackend().SetMultiple(context.Background(), node.NewBaseNode(n.SpaceID, n.ID+node.CurrentIDDelimiter, t.lookup), attrs, true)
err := t.lookup.MetadataBackend().SetMultiple(context.Background(), node.NewBaseNode(n.SpaceID, n.ID+node.CurrentIDDelimiter, t.lookup), attrs)
if err != nil {
t.log.Error().Err(err).Str("spaceID", n.SpaceID).Str("id", n.ID).Msg("could not copy metadata to current revision")
}
@@ -796,7 +799,7 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met
mtime := fi.ModTime()
err = n.SetXattrsWithContext(ctx, map[string][]byte{
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
}, false)
})
if err != nil {
t.log.Error().Err(err).Str("path", n.InternalPath()).Msg("could not set mtime attribute on new node")
}
@@ -868,5 +871,5 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
if t.options.TreeTimeAccounting || t.options.TreeSizeAccounting {
attributes[prefixes.PropagationAttr] = []byte("1") // mark the node for propagation
}
return n.SetXattrsWithContext(ctx, attributes, false)
return n.SetXattrsWithContext(ctx, attributes)
}

62
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock/lock.go generated vendored Normal file
View File

@@ -0,0 +1,62 @@
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
// SPDX-License-Identifier: Apache-2.0
package goroutinelock
import (
"bytes"
"runtime"
"strconv"
"sync"
"sync/atomic"
"github.com/rs/zerolog/log"
)
// Lock tracks which goroutine currently owns a metadata lock.
//
// Held() only returns true in the owning goroutine, which prevents lock-held
// state from leaking across goroutine boundaries when helper nodes are shared.
type Lock struct {
gid atomic.Uint64
}
var goidLogOnce sync.Once
func (o *Lock) Hold() {
o.gid.Store(goid())
}
func (o *Lock) Release() {
o.gid.Store(0)
}
func (o *Lock) Held() bool {
return o.gid.Load() == goid()
}
func goid() uint64 {
var buf [64]byte
n := runtime.Stack(buf[:], false)
// The header has the form "goroutine 1234 [running]:".
fields := bytes.Fields(buf[:n])
if len(fields) < 2 {
goidLogOnce.Do(func() {
log.Error().
Str("stack_header", string(buf[:n])).
Msg("goroutinelock: could not determine goroutine id from runtime stack header")
})
return 0
}
id, err := strconv.ParseUint(string(fields[1]), 10, 64)
if err != nil {
goidLogOnce.Do(func() {
log.Error().Err(err).
Str("goroutine_id_field", string(fields[1])).
Msg("goroutinelock: could not parse goroutine id")
})
return 0
}
return id
}

1
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go generated vendored
View File

@@ -108,6 +108,7 @@ type IDCachingTree interface {
type SessionStore interface {
New(ctx context.Context) *upload.DecomposedFsSession
SessionFromInfo(info tusd.FileInfo) *upload.DecomposedFsSession
List(ctx context.Context) ([]*upload.DecomposedFsSession, error)
Get(ctx context.Context, id string) (*upload.DecomposedFsSession, error)
Cleanup(ctx context.Context, session upload.Session, revertNodeMetadata, keepUpload, unmarkPostprocessing bool)

12
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk/disk_fallback.go generated vendored Normal file
View File

@@ -0,0 +1,12 @@
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
// SPDX-License-Identifier: Apache-2.0
//go:build !linux && !freebsd && !darwin
package disk
import "os"
func Fdatasync(f *os.File) error {
return nil
}

15
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk/disk_linux.go generated vendored Normal file
View File

@@ -0,0 +1,15 @@
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
// SPDX-License-Identifier: Apache-2.0
//go:build linux
package disk
import (
"os"
"syscall"
)
func Fdatasync(f *os.File) error {
return syscall.Fdatasync(int(f.Fd()))
}

12
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk/disk_unix.go generated vendored Normal file
View File

@@ -0,0 +1,12 @@
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
// SPDX-License-Identifier: Apache-2.0
//go:build freebsd || darwin
package disk
import "os"
func Fdatasync(file *os.File) error {
return file.Sync()
}

4
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/grants.go generated vendored
View File

@@ -223,7 +223,7 @@ func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference
}
}
if err := grantNode.DeleteGrant(ctx, g, false); err != nil {
if err := grantNode.DeleteGrant(ctx, g); err != nil {
return err
}
@@ -349,7 +349,7 @@ func (fs *Decomposedfs) storeGrant(ctx context.Context, n *node.Node, g *provide
attribs := node.Attributes{
prefixes.GrantPrefix + principal: value,
}
if err := n.SetXattrsWithContext(ctx, attribs, false); err != nil {
if err := n.SetXattrsWithContext(ctx, attribs); err != nil {
appctx.GetLogger(ctx).Error().Err(err).
Str("principal", principal).Msg("Could not set grant for principal")
return err

49
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup/lookup.go generated vendored
View File

@@ -36,7 +36,6 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
@@ -314,45 +313,13 @@ func refFromCS3(b []byte) (*provider.Reference, error) {
}, nil
}
// CopyMetadata copies all extended attributes from source to target.
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
// For the source file, a shared lock is acquired.
// NOTE: target resource will be write locked!
func (lu *Lookup) CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) {
// Acquire a read log on the source node
// write lock existing node before reading treesize or tree time
lock, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(sourceNode), os.O_RDONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
if err != nil {
return errors.Wrap(err, "xattrs: Unable to lock source to read")
}
defer func() {
rerr := lock.Close()
// if err is non nil we do not overwrite that
if err == nil {
err = rerr
}
}()
return lu.CopyMetadataWithSourceLock(ctx, sourceNode, targetNode, filter, lock, acquireTargetLock)
}
// CopyMetadataWithSourceLock copies all extended attributes from source to target.
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
// For the source file, a matching lockedfile is required.
// NOTE: target resource will be write locked!
func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) {
switch {
case lockedSource == nil:
return errors.New("no lock provided")
case lockedSource.Name() != lu.MetadataBackend().LockfilePath(sourceNode):
return errors.New("lockpath does not match filepath")
}
// CopyMetadata copies all extended attributes from source to target. The optional filter
// function can be used to filter by attribute name, e.g. by checking a prefix.
//
// Locking is derived from the nodes: the source's metadata lock is acquired for the duration
// of the read unless the caller already holds it (sourceNode.LockHeld()), and the target is
// write locked while its attributes are written unless the caller already holds its lock
func (lu *Lookup) CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool)) (err error) {
attrs, err := lu.metadataBackend.All(ctx, sourceNode)
if err != nil {
return err
@@ -369,7 +336,7 @@ func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, sourceNode, ta
newAttrs[attrName] = val
}
return lu.MetadataBackend().SetMultiple(ctx, targetNode, newAttrs, acquireTargetLock)
return lu.MetadataBackend().SetMultiple(ctx, targetNode, newAttrs)
}
func (lu *Lookup) PurgeNode(n *node.Node) error {

2
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata.go generated vendored
View File

@@ -142,7 +142,7 @@ func (fs *Decomposedfs) UnsetArbitraryMetadata(ctx context.Context, ref *provide
errs := []error{}
for _, k := range keys {
if err = n.RemoveXattr(ctx, prefixes.MetadataPrefix+k, true); err != nil {
if err = n.RemoveXattr(ctx, prefixes.MetadataPrefix+k); err != nil {
if metadata.IsAttrUnset(err) {
continue // already gone, ignore
}

71
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/hybrid_backend.go generated vendored
View File

@@ -3,7 +3,6 @@ package metadata
import (
"context"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
@@ -114,11 +113,11 @@ func (b HybridBackend) list(ctx context.Context, n MetadataNode) (attribs []stri
return nil, &xattr.Error{Op: "HybridBackend.list", Path: n.InternalPath(), Err: syscall.ENOENT} // attribute not found
}
return xattr.List(filePath)
return listXattr(filePath)
}
// All reads all extended attributes for a node, protected by a
// shared file lock
// All reads all extended attributes for a node. It acquires a shared file lock
// unless the caller already holds the node's metadata lock (n.LockHeld()).
func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
attribs := map[string][]byte{}
@@ -127,29 +126,18 @@ func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]by
return attribs, err
}
unlock, err := b.Lock(n)
if err != nil {
return nil, err
}
defer func() { _ = unlock() }()
return b.getAll(ctx, n, false, false)
}
// AllWithLockedSource reads all extended attributes from the given reader.
// The path argument is used for storing the data in the cache
func (b HybridBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, _ io.Reader) (map[string][]byte, error) {
attribs := map[string][]byte{}
err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs)
if err == nil {
return attribs, err
if !n.LockHeld() {
unlock, err := b.Lock(n)
if err != nil {
return nil, err
}
defer func() { _ = unlock() }()
}
return b.getAll(ctx, n, false, false)
return b.getAll(ctx, n, false)
}
func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipCache, skipOffloaded bool) (map[string][]byte, error) {
func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipOffloaded bool) (map[string][]byte, error) {
attribs := map[string][]byte{}
attrNames, err := b.list(ctx, n)
if err != nil {
@@ -215,13 +203,13 @@ func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipCache, sk
// Set sets one attribute for the given path
func (b HybridBackend) Set(ctx context.Context, n MetadataNode, key string, val []byte) (err error) {
return b.SetMultiple(ctx, n, map[string][]byte{key: val}, true)
return b.SetMultiple(ctx, n, map[string][]byte{key: val})
}
// SetMultiple sets a set of attribute for the given path
func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) (err error) {
func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) (err error) {
path := n.InternalPath()
if acquireLock {
if !n.LockHeld() {
unlock, err := b.Lock(n)
if err != nil {
return err
@@ -247,7 +235,7 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
mdSize += len(attribs[key]) + len(key)
}
}
existingAttribs, err := b.getAll(ctx, n, true, true)
existingAttribs, err := b.getAll(ctx, n, true)
if err != nil {
return err
}
@@ -320,7 +308,7 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
}
// Update the cache with the new values
_, err = b.getAll(ctx, n, true, false)
_, err = b.getAll(ctx, n, false)
return err
}
@@ -331,7 +319,7 @@ func (b HybridBackend) offloadMetadata(ctx context.Context, n MetadataNode) erro
var xerr error
// collect attributes to move
existingAttribs, err := b.getAll(ctx, n, true, true)
existingAttribs, err := b.getAll(ctx, n, true)
if err != nil {
return err
}
@@ -377,9 +365,9 @@ func (b HybridBackend) offloadMetadata(ctx context.Context, n MetadataNode) erro
}
// Remove an extended attribute key
func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
path := n.InternalPath()
if acquireLock {
if !n.LockHeld() {
unlock, err := b.Lock(n)
if err != nil {
return err
@@ -444,7 +432,7 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string, a
}
// Update the cache with the new values
_, err := b.getAll(ctx, n, true, false)
_, err := b.getAll(ctx, n, false)
return err
}
@@ -462,7 +450,7 @@ func (b HybridBackend) Purge(ctx context.Context, n MetadataNode) error {
}
defer func() { _ = unlock() }()
attribs, err := b.getAll(ctx, n, true, false)
attribs, err := b.getAll(ctx, n, false)
if err == nil {
for attr := range attribs {
if strings.HasPrefix(attr, prefixes.OcPrefix) {
@@ -504,11 +492,10 @@ func (b HybridBackend) LockfilePath(n MetadataNode) string {
// Lock locks the metadata for the given path
func (b HybridBackend) Lock(n MetadataNode) (UnlockFunc, error) {
f, _, err := b.LockAndRead(n)
return f, err
}
if n.LockHeld() {
return func() error { return nil }, nil
}
func (b HybridBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
metaLockPath := b.LockfilePath(n)
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
if err != nil {
@@ -516,20 +503,22 @@ func (b HybridBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error
// create the parent directory
err = os.MkdirAll(filepath.Dir(metaLockPath), 0700)
if err != nil {
return nil, nil, err
return nil, err
}
mlock, err = lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
if err != nil {
return nil, nil, err
return nil, err
}
} else {
return nil, nil, err
return nil, err
}
}
n.SetLockHeld(true)
return func() error {
n.SetLockHeld(false)
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
return mlock.Close()
}, mlock, nil
}, nil
}
func (b HybridBackend) cacheKey(n MetadataNode) string {

55
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/list.go generated vendored Normal file
View File

@@ -0,0 +1,55 @@
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
// SPDX-License-Identifier: Apache-2.0
package metadata
import (
"errors"
"syscall"
"github.com/pkg/xattr"
)
// maxListXattrRetries bounds the number of attempts made by listXattr when it
// keeps racing with a concurrent writer.
const maxListXattrRetries = 10
// listXattr lists the extended attribute names of the file at path, retrying on
// transient ERANGE/E2BIG errors.
//
// xattr.List is not atomic: internally it issues a listxattr syscall to learn
// the size of the extended-attribute name list, allocates a buffer of that
// size and issues a second listxattr to read the names into it. If another
// goroutine or process adds an extended attribute to the same inode between the
// two syscalls, the name list grows beyond the buffer and the kernel returns
// ERANGE ("numerical result out of range") (or E2BIG on some filesystems).
//
// This is a transient condition: a fresh size query on the next attempt
// reflects the new size, so we simply retry a bounded number of times. See the
// posix fs watcher, whose assimilation can read a node's xattrs while the node
// is still being created, for a concrete source of such concurrent writes.
func listXattr(path string) ([]string, error) {
var (
attrs []string
err error
)
for i := 0; i < maxListXattrRetries; i++ {
attrs, err = xattr.List(path)
if err == nil || !isRangeError(err) {
return attrs, err
}
}
return attrs, err
}
// isRangeError reports whether err is a transient listxattr "buffer too small"
// error (ERANGE or E2BIG) caused by the name list growing concurrently.
func isRangeError(err error) bool {
var xerr *xattr.Error
if errors.As(err, &xerr) {
if serr, ok := xerr.Err.(syscall.Errno); ok {
return serr == syscall.ERANGE || serr == syscall.E2BIG
}
}
return false
}

98
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/messagepack_backend.go generated vendored
View File

@@ -86,12 +86,12 @@ func (b MessagePackBackend) IdentifyPath(_ context.Context, path string) (string
// All reads all extended attributes for a node
func (b MessagePackBackend) All(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
return b.loadAttributes(ctx, n, nil)
return b.loadAttributes(ctx, n)
}
// Get an extended attribute value for the given key
func (b MessagePackBackend) Get(ctx context.Context, n MetadataNode, key string) ([]byte, error) {
attribs, err := b.loadAttributes(ctx, n, nil)
attribs, err := b.loadAttributes(ctx, n)
if err != nil {
return []byte{}, err
}
@@ -104,7 +104,7 @@ func (b MessagePackBackend) Get(ctx context.Context, n MetadataNode, key string)
// GetInt64 reads a string as int64 from the xattrs
func (b MessagePackBackend) GetInt64(ctx context.Context, n MetadataNode, key string) (int64, error) {
attribs, err := b.loadAttributes(ctx, n, nil)
attribs, err := b.loadAttributes(ctx, n)
if err != nil {
return 0, err
}
@@ -121,26 +121,20 @@ func (b MessagePackBackend) GetInt64(ctx context.Context, n MetadataNode, key st
// Set sets one attribute for the given path
func (b MessagePackBackend) Set(ctx context.Context, n MetadataNode, key string, val []byte) error {
return b.SetMultiple(ctx, n, map[string][]byte{key: val}, true)
return b.SetMultiple(ctx, n, map[string][]byte{key: val})
}
// SetMultiple sets a set of attribute for the given path
func (b MessagePackBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) error {
return b.saveAttributes(ctx, n, attribs, nil, acquireLock)
func (b MessagePackBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) error {
return b.saveAttributes(ctx, n, attribs, nil)
}
// Remove an extended attribute key
func (b MessagePackBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
return b.saveAttributes(ctx, n, nil, []string{key}, acquireLock)
func (b MessagePackBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
return b.saveAttributes(ctx, n, nil, []string{key})
}
// AllWithLockedSource reads all extended attributes from the given reader (if possible).
// The path argument is used for storing the data in the cache
func (b MessagePackBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error) {
return b.loadAttributes(ctx, n, source)
}
func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode, setAttribs map[string][]byte, deleteAttribs []string, acquireLock bool) error {
func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode, setAttribs map[string][]byte, deleteAttribs []string) error {
var (
err error
)
@@ -155,7 +149,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode,
}()
metaPath := b.MetadataPath(n)
if acquireLock {
if !n.LockHeld() {
unlock, err := b.Lock(n)
if err != nil {
return err
@@ -211,7 +205,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode,
return err
}
func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error) {
func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
ctx, span := tracer.Start(ctx, "loadAttributes")
defer span.End()
attribs := map[string][]byte{}
@@ -221,39 +215,31 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode,
}
metaPath := b.MetadataPath(n)
var msgBytes []byte
if source == nil {
// // No cached entry found. Read from storage and store in cache
_, subspan := tracer.Start(ctx, "os.OpenFile")
// source, err = lockedfile.Open(metaPath)
source, err = os.Open(metaPath)
subspan.End()
// // No cached entry found. Read from storage and store in cache
if err != nil {
if os.IsNotExist(err) {
// some of the caller rely on ENOTEXISTS to be returned when the
// actual file (not the metafile) does not exist in order to
// determine whether a node exists or not -> stat the actual node
_, subspan := tracer.Start(ctx, "os.Stat")
_, err := os.Stat(n.InternalPath())
subspan.End()
if err != nil {
return nil, err
}
return attribs, nil // no attributes set yet
// No cached entry found. Read from storage and store in cache
_, subspan := tracer.Start(ctx, "os.Open")
source, err := os.Open(metaPath)
subspan.End()
if err != nil {
if os.IsNotExist(err) {
// some of the callers rely on ENOTEXISTS to be returned when the
// actual file (not the metafile) does not exist in order to
// determine whether a node exists or not -> stat the actual node
_, subspan := tracer.Start(ctx, "os.Stat")
_, err := os.Stat(n.InternalPath())
subspan.End()
if err != nil {
return nil, err
}
return attribs, nil // no attributes set yet
}
_, subspan = tracer.Start(ctx, "io.ReadAll")
msgBytes, err = io.ReadAll(source)
source.(*os.File).Close()
subspan.End()
} else {
_, subspan := tracer.Start(ctx, "io.ReadAll")
msgBytes, err = io.ReadAll(source)
subspan.End()
return nil, err
}
_, subspan = tracer.Start(ctx, "io.ReadAll")
msgBytes, err := io.ReadAll(source)
_ = source.Close()
subspan.End()
if err != nil {
return nil, err
}
@@ -264,7 +250,7 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode,
}
}
_, subspan := tracer.Start(ctx, "metaCache.PushToCache")
_, subspan = tracer.Start(ctx, "metaCache.PushToCache")
err = b.metaCache.PushToCache(b.cacheKey(n), attribs)
subspan.End()
if err != nil {
@@ -314,31 +300,23 @@ func (MessagePackBackend) LockfilePath(n MetadataNode) string { return n.Interna
// Lock locks the metadata for the given path
func (b MessagePackBackend) Lock(n MetadataNode) (UnlockFunc, error) {
if n.LockHeld() {
return func() error { return nil }, nil
}
metaLockPath := b.LockfilePath(n)
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
if err != nil {
return nil, err
}
n.SetLockHeld(true)
return func() error {
n.SetLockHeld(false)
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
return mlock.Close()
}, nil
}
func (b MessagePackBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
unlock, err := b.Lock(n)
if err != nil {
return nil, nil, err
}
f, err := os.Open(b.MetadataPath(n))
if err != nil {
_ = unlock()
return nil, nil, err
}
return unlock, f, nil
}
func (b MessagePackBackend) cacheKey(n MetadataNode) string {
return n.GetSpaceID() + "/" + n.GetID()
}

33
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/metadata.go generated vendored
View File

@@ -21,7 +21,6 @@ package metadata
import (
"context"
"errors"
"io"
"time"
"go.opentelemetry.io/otel"
@@ -42,6 +41,11 @@ type MetadataNode interface {
GetSpaceID() string
GetID() string
InternalPath() string
// LockHeld reports whether the caller already holds this node's metadata lock.
// Backends use it to decide whether they must (re-)acquire the lock when reading
// metadata; re-acquiring a held, non-reentrant advisory file lock self-deadlocks.
LockHeld() bool
SetLockHeld(held bool)
}
// Backend defines the interface for file attribute backends
@@ -49,17 +53,21 @@ type Backend interface {
Name() string
IdentifyPath(ctx context.Context, path string) (string, string, string, time.Time, error)
// All reads all extended attributes for a node. It acquires the node's metadata
// lock unless n.LockHeld() reports that the caller already holds it.
All(ctx context.Context, n MetadataNode) (map[string][]byte, error)
AllWithLockedSource(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error)
Get(ctx context.Context, n MetadataNode, key string) ([]byte, error)
GetInt64(ctx context.Context, n MetadataNode, key string) (int64, error)
Set(ctx context.Context, n MetadataNode, key string, val []byte) error
SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) error
Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error
// SetMultiple sets multiple extended attributes for a node. It acquires the node's
// metadata lock unless n.LockHeld() reports that the caller already holds it.
SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) error
// Remove removes an extended attribute key. It acquires the node's metadata lock
// unless n.LockHeld() reports that the caller already holds it.
Remove(ctx context.Context, n MetadataNode, key string) error
Lock(n MetadataNode) (UnlockFunc, error)
LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error)
Purge(ctx context.Context, n MetadataNode) error
Rename(oldNode, newNode MetadataNode) error
MetadataPath(n MetadataNode) string
@@ -100,12 +108,12 @@ func (NullBackend) Set(ctx context.Context, n MetadataNode, key string, val []by
}
// SetMultiple sets a set of attribute for the given path
func (NullBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) error {
func (NullBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) error {
return errUnconfiguredError
}
// Remove removes an extended attribute key
func (NullBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
func (NullBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
return errUnconfiguredError
}
@@ -114,11 +122,6 @@ func (NullBackend) Lock(n MetadataNode) (UnlockFunc, error) {
return nil, nil
}
// LockAndRead locks the metadata for reading
func (NullBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
return nil, nil, nil
}
// IsMetaFile returns whether the given path represents a meta file
func (NullBackend) IsMetaFile(path string) bool { return false }
@@ -133,9 +136,3 @@ func (NullBackend) MetadataPath(n MetadataNode) string { return "" }
// LockfilePath returns the path of the lock file
func (NullBackend) LockfilePath(n MetadataNode) string { return "" }
// AllWithLockedSource reads all extended attributes from the given reader
// The path argument is used for storing the data in the cache
func (NullBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error) {
return nil, errUnconfiguredError
}

66
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/xattrs_backend.go generated vendored
View File

@@ -20,7 +20,6 @@ package metadata
import (
"context"
"io"
"os"
"path/filepath"
"strconv"
@@ -89,7 +88,7 @@ func (b XattrsBackend) GetInt64(ctx context.Context, n MetadataNode, key string)
func (b XattrsBackend) list(ctx context.Context, n MetadataNode, acquireLock bool) (attribs []string, err error) {
filePath := n.InternalPath()
attrs, err := xattr.List(filePath)
attrs, err := listXattr(filePath)
if err == nil {
return attrs, nil
}
@@ -100,17 +99,21 @@ func (b XattrsBackend) list(ctx context.Context, n MetadataNode, acquireLock boo
if err != nil {
return nil, err
}
n.SetLockHeld(true)
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
defer f.Close()
defer func() {
n.SetLockHeld(false)
_ = f.Close()
}()
}
return xattr.List(filePath)
return listXattr(filePath)
}
// All reads all extended attributes for a node, protected by a
// shared file lock
// All reads all extended attributes for a node. It acquires a shared file lock
// unless the caller already holds the node's metadata lock (n.LockHeld()).
func (b XattrsBackend) All(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
return b.getAll(ctx, n, false, true)
return b.getAll(ctx, n, false, !n.LockHeld())
}
func (b XattrsBackend) getAll(ctx context.Context, n MetadataNode, skipCache, acquireLock bool) (map[string][]byte, error) {
@@ -163,13 +166,13 @@ func (b XattrsBackend) getAll(ctx context.Context, n MetadataNode, skipCache, ac
// Set sets one attribute for the given path
func (b XattrsBackend) Set(ctx context.Context, n MetadataNode, key string, val []byte) (err error) {
return b.SetMultiple(ctx, n, map[string][]byte{key: val}, true)
return b.SetMultiple(ctx, n, map[string][]byte{key: val})
}
// SetMultiple sets a set of attribute for the given path
func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) (err error) {
func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) (err error) {
path := n.InternalPath()
if acquireLock {
if !n.LockHeld() {
err := os.MkdirAll(filepath.Dir(path), 0700)
if err != nil {
return err
@@ -178,8 +181,12 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
if err != nil {
return err
}
n.SetLockHeld(true)
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
defer lockedFile.Close()
defer func() {
n.SetLockHeld(false)
_ = lockedFile.Close()
}()
}
// error handling: Count if there are errors while setting the attribs.
@@ -206,15 +213,19 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
}
// Remove an extended attribute key
func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
path := n.InternalPath()
if acquireLock {
if !n.LockHeld() {
lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
n.SetLockHeld(true)
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
defer lockedFile.Close()
defer func() {
n.SetLockHeld(false)
_ = lockedFile.Close()
}()
}
err := xattr.Remove(path, key)
@@ -275,38 +286,23 @@ func (XattrsBackend) LockfilePath(n MetadataNode) string { return n.InternalPath
// Lock locks the metadata for the given path
func (b XattrsBackend) Lock(n MetadataNode) (UnlockFunc, error) {
if n.LockHeld() {
return func() error { return nil }, nil
}
metaLockPath := b.LockfilePath(n)
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
if err != nil {
return nil, err
}
n.SetLockHeld(true)
return func() error {
n.SetLockHeld(false)
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
return mlock.Close()
}, nil
}
// LockAndRead locks the metadata for reading
func (b XattrsBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
unlock, err := b.Lock(n)
if err != nil {
return nil, nil, err
}
f, err := os.Open(b.MetadataPath(n))
if err != nil {
_ = unlock()
return nil, nil, err
}
return unlock, f, nil
}
// AllWithLockedSource reads all extended attributes from the given reader.
// The path argument is used for storing the data in the cache
func (b XattrsBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, _ io.Reader) (map[string][]byte, error) {
return b.All(ctx, n)
}
func (b XattrsBackend) cacheKey(n MetadataNode) string {
// rootPath is guaranteed to have no trailing slash
// the cache key shouldn't begin with a slash as some stores drop it which can cause

87
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go generated vendored
View File

@@ -44,13 +44,13 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/mime"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/grants"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
@@ -139,7 +139,7 @@ type Tree interface {
BuildSpaceIDIndexEntry(spaceID string) string
ResolveSpaceIDIndexEntry(spaceID string) (string, error)
CreateRevision(ctx context.Context, n *Node, version string, f *lockedfile.File) (string, error)
CreateRevision(ctx context.Context, n *Node, version string) (string, error)
ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error)
DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error)
@@ -166,8 +166,7 @@ type PathLookup interface {
MetadataBackend() metadata.Backend
TimeManager() TimeManager
ReadBlobIDAndSizeAttr(ctx context.Context, n metadata.MetadataNode, attrs Attributes) (string, int64, error)
CopyMetadataWithSourceLock(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error)
CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error)
CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool)) (err error)
PurgeNode(n *Node) error
}
@@ -181,6 +180,12 @@ type BaseNode struct {
SpaceID string
ID string
// lock records which goroutine (if any) holds this node's metadata lock, so the
// metadata backend must not (re-)acquire it when reading attributes. Ownership is
// scoped to the acquiring goroutine: a node leaked into another goroutine reports
// LockHeld() == false there and takes its own lock rather than riding a stale flag.
lock goroutinelock.Lock
lu PathLookup
internalPathID string
internalPath string
@@ -197,6 +202,19 @@ func NewBaseNode(spaceID, nodeID string, lu PathLookup) *BaseNode {
func (n *BaseNode) GetSpaceID() string { return n.SpaceID }
func (n *BaseNode) GetID() string { return n.ID }
// LockHeld reports whether the calling goroutine holds this node's metadata lock.
func (n *BaseNode) LockHeld() bool { return n.lock.Held() }
// SetLockHeld records whether the calling goroutine holds this node's metadata
// lock
func (n *BaseNode) SetLockHeld(held bool) {
if held {
n.lock.Hold()
return
}
n.lock.Release()
}
// InternalPath returns the internal path of the Node
func (n *BaseNode) InternalPath() string {
if len(n.internalPath) > 0 && n.ID == n.internalPathID {
@@ -355,15 +373,15 @@ func LockAndReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID, intern
ctx, span := tracer.Start(ctx, "LockAndReadNode")
defer span.End()
_, subspan := tracer.Start(ctx, "lockedfile.OpenFile")
_, subspan := tracer.Start(ctx, "MetadataBackend.Lock")
bn := NewBaseNode(spaceID, nodeID, lu)
unlock, r, err := lu.MetadataBackend().LockAndRead(bn)
unlock, err := lu.MetadataBackend().Lock(bn)
subspan.End()
if err != nil {
return nil, nil, err
}
n, err := readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, r)
n, err := readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, true)
if err != nil {
_ = unlock()
return nil, nil, err
@@ -373,17 +391,22 @@ func LockAndReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID, intern
return n, nil, errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
}
return n, unlock, nil
n.SetLockHeld(true)
return n, func() error {
n.SetLockHeld(false)
return unlock()
}, nil
}
// ReadNode creates a new instance from an id and checks if it exists
func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath string, canListDisabledSpace bool, spaceRoot *Node, skipParentCheck bool) (*Node, error) {
return readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, nil)
return readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, false)
}
// readNode reads a node by its id. If a reader is provided, it will be passed to the metadata backend to read the metadata.
// This is useful when the caller already holds a lock to prevent deadlocks when reading the metadata.
func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath string, canListDisabledSpace bool, spaceRoot *Node, skipParentCheck bool, r io.Reader) (*Node, error) {
// readNode reads a node by its id. When alreadyLocked is true the caller already
// holds the node's metadata lock, so the attributes are read without re-acquiring it
// (which would dead-lock e.g. the hybrid backend).
func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath string, canListDisabledSpace bool, spaceRoot *Node, skipParentCheck bool, alreadyLocked bool) (*Node, error) {
ctx, span := tracer.Start(ctx, "ReadNode")
defer span.End()
var err error
@@ -398,12 +421,15 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath
},
}
spaceRoot.SpaceRoot = spaceRoot
if alreadyLocked && nodeID == spaceID {
spaceRoot.SetLockHeld(true)
}
// If we hold the lock on the space root itself, prime its attribute cache
// through the no-lock path so the owner/name/disabled reads below do not try
// to re-acquire the already-held lock and self-deadlock.
if r != nil && nodeID == spaceID {
_, err = spaceRoot.XattrsWithReader(ctx, r)
if alreadyLocked && nodeID == spaceID {
_, err = spaceRoot.Xattrs(ctx)
switch {
case metadata.IsNotExist(err):
return spaceRoot, nil // swallow not found, the node defaults to exists = false
@@ -464,6 +490,9 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath
},
SpaceRoot: spaceRoot,
}
if alreadyLocked {
n.SetLockHeld(true)
}
if internalPath != "" {
n.internalPath = internalPath
}
@@ -477,7 +506,7 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath
}()
var attrs Attributes
attrs, err = n.XattrsWithReader(ctx, r)
attrs, err = n.Xattrs(ctx)
switch {
case metadata.IsNotExist(err):
return n, nil // swallow not found, the node defaults to exists = false
@@ -570,9 +599,9 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) {
return readNode, nil
}
// ParentWithReader returns the parent node
func (n *Node) ParentWithReader(ctx context.Context, r io.Reader) (*Node, error) {
_, span := tracer.Start(ctx, "ParentWithReader")
// Parent returns the parent node
func (n *Node) Parent(ctx context.Context) (*Node, error) {
_, span := tracer.Start(ctx, "Parent")
defer span.End()
if n.ParentID == "" {
return nil, fmt.Errorf("decomposedfs: root has no parent")
@@ -586,8 +615,7 @@ func (n *Node) ParentWithReader(ctx context.Context, r io.Reader) (*Node, error)
SpaceRoot: n.SpaceRoot,
}
// fill metadata cache using the reader
attrs, err := p.XattrsWithReader(ctx, r)
attrs, err := p.Xattrs(ctx)
switch {
case metadata.IsNotExist(err):
return p, nil // swallow not found, the node defaults to exists = false
@@ -602,11 +630,6 @@ func (n *Node) ParentWithReader(ctx context.Context, r io.Reader) (*Node, error)
return p, err
}
// Parent returns the parent node
func (n *Node) Parent(ctx context.Context) (p *Node, err error) {
return n.ParentWithReader(ctx, nil)
}
// Owner returns the space owner
func (n *Node) Owner() *userpb.UserId {
return n.SpaceRoot.owner
@@ -740,7 +763,7 @@ func (n *Node) SetMtimeString(ctx context.Context, mtime string) error {
// SetMTime writes the UTC mtime to the extended attributes or removes the attribute if nil is passed
func (n *Node) SetMtime(ctx context.Context, t *time.Time) (err error) {
if t == nil {
return n.RemoveXattr(ctx, prefixes.MTimeAttr, true)
return n.RemoveXattr(ctx, prefixes.MTimeAttr)
}
return n.SetXattrString(ctx, prefixes.MTimeAttr, t.UTC().Format(time.RFC3339Nano))
}
@@ -780,7 +803,7 @@ func (n *Node) SetFavorite(ctx context.Context, uid *userpb.UserId) error {
func (n *Node) UnsetFavorite(ctx context.Context, uid *userpb.UserId) error {
// the favorite flag is specific to the user, so we need to incorporate the userid
fa := prefixes.FavoriteKey(uid)
return n.RemoveXattr(ctx, fa, true)
return n.RemoveXattr(ctx, fa)
}
// IsDir returns true if the node is a directory
@@ -1117,7 +1140,7 @@ func (n *Node) SetChecksum(ctx context.Context, csType string, h hash.Hash) (err
// UnsetTempEtag removes the temporary etag attribute
func (n *Node) UnsetTempEtag(ctx context.Context) (err error) {
return n.RemoveXattr(ctx, prefixes.TmpEtagAttr, true)
return n.RemoveXattr(ctx, prefixes.TmpEtagAttr)
}
func isGrantExpired(g *provider.Grant) bool {
@@ -1278,7 +1301,7 @@ func (n *Node) ReadGrant(ctx context.Context, grantee string) (g *provider.Grant
}
// ReadGrant reads a CS3 grant
func (n *Node) DeleteGrant(ctx context.Context, g *provider.Grant, acquireLock bool) (err error) {
func (n *Node) DeleteGrant(ctx context.Context, g *provider.Grant) (err error) {
var attr string
if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP {
@@ -1287,7 +1310,7 @@ func (n *Node) DeleteGrant(ctx context.Context, g *provider.Grant, acquireLock b
attr = prefixes.GrantUserAcePrefix + g.Grantee.GetUserId().OpaqueId
}
if err = n.RemoveXattr(ctx, attr, acquireLock); err != nil {
if err = n.RemoveXattr(ctx, attr); err != nil {
return err
}
@@ -1379,7 +1402,7 @@ func (n *Node) UnmarkProcessing(ctx context.Context, uploadID string) error {
// file started another postprocessing later - do not remove
return nil
}
return n.RemoveXattr(ctx, prefixes.StatusPrefix, true)
return n.RemoveXattr(ctx, prefixes.StatusPrefix)
}
// IsProcessing returns true if the node is currently being processed
@@ -1404,7 +1427,7 @@ func (n *Node) SetScanData(ctx context.Context, info string, date time.Time) err
attribs := Attributes{}
attribs.SetString(prefixes.ScanStatusPrefix, info)
attribs.SetString(prefixes.ScanDatePrefix, date.Format(time.RFC3339Nano))
return n.SetXattrsWithContext(ctx, attribs, true)
return n.SetXattrsWithContext(ctx, attribs)
}
// ScanData returns scanning information of the node

9
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/permissions.go generated vendored
View File

@@ -134,10 +134,6 @@ func (p *Permissions) assemblePermissions(ctx context.Context, n *Node, failOnTr
return NoPermissions(), nil
}
if u.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE {
return ServiceAccountPermissions(), nil
}
// are we reading a revision?
if strings.Contains(n.ID, RevisionIDDelimiter) {
// verify revision key format
@@ -204,6 +200,11 @@ func (p *Permissions) assemblePermissions(ctx context.Context, n *Node, failOnTr
return OwnerPermissions(), nil
}
// merge in the general service account permissions; grants can only add to them
if u.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE {
AddPermissions(ap, ServiceAccountPermissions())
}
appctx.GetLogger(ctx).Debug().Interface("permissions", ap).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Interface("user", u).Msg("returning agregated permissions")
return ap, nil
}

61
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/xattrs.go generated vendored
View File

@@ -20,7 +20,6 @@ package node
import (
"context"
"io"
"io/fs"
"strconv"
"time"
@@ -72,7 +71,7 @@ func (md Attributes) SetTime(key string, t time.Time) {
}
// SetXattrs sets multiple extended attributes on the write-through cache/node
func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]byte, acquireLock bool) (err error) {
func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]byte) (err error) {
_, span := tracer.Start(ctx, "SetXattrsWithContext")
defer span.End()
if n.xattrsCache != nil {
@@ -81,12 +80,12 @@ func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]by
}
}
return n.lu.MetadataBackend().SetMultiple(ctx, n, attribs, acquireLock)
return n.lu.MetadataBackend().SetMultiple(ctx, n, attribs)
}
// SetXattrs sets multiple extended attributes on the write-through cache/node
func (n *Node) SetXattrs(attribs map[string][]byte, acquireLock bool) (err error) {
return n.SetXattrsWithContext(context.Background(), attribs, acquireLock)
func (n *Node) SetXattrs(attribs map[string][]byte) (err error) {
return n.SetXattrsWithContext(context.Background(), attribs)
}
// SetXattr sets an extended attribute on the write-through cache/node
@@ -108,33 +107,28 @@ func (n *Node) SetXattrString(ctx context.Context, key, val string) (err error)
}
// RemoveXattr removes an extended attribute from the write-through cache/node
func (n *Node) RemoveXattr(ctx context.Context, key string, acquireLock bool) error {
func (n *Node) RemoveXattr(ctx context.Context, key string) error {
if n.xattrsCache != nil {
delete(n.xattrsCache, key)
}
return n.lu.MetadataBackend().Remove(ctx, n, key, acquireLock)
return n.lu.MetadataBackend().Remove(ctx, n, key)
}
// XattrsWithReader returns the extended attributes of the node. If the attributes have already
// been cached they are not read from disk again.
func (n *Node) XattrsWithReader(ctx context.Context, r io.Reader) (Attributes, error) {
// loadXattrs reads and caches the node's extended attributes using the given load
// function. It centralizes the empty-node guard and the write-through caching that
// is shared by all the Xattrs accessors below.
func (n *Node) loadXattrs(load func() (Attributes, error)) (Attributes, error) {
if n.ID == "" {
// Do not try to read the attribute of an empty node. The InternalPath points to the
// base nodes directory in this case.
return Attributes{}, &xattr.Error{Op: "node.XattrsWithReader", Path: n.InternalPath(), Err: xattr.ENOATTR}
return Attributes{}, &xattr.Error{Op: "node.Xattrs", Path: n.InternalPath(), Err: xattr.ENOATTR}
}
if n.xattrsCache != nil {
return n.xattrsCache, nil
}
var attrs Attributes
var err error
if r != nil {
attrs, err = n.lu.MetadataBackend().AllWithLockedSource(ctx, n, r)
} else {
attrs, err = n.lu.MetadataBackend().All(ctx, n)
}
attrs, err := load()
if err != nil {
return nil, err
}
@@ -143,37 +137,32 @@ func (n *Node) XattrsWithReader(ctx context.Context, r io.Reader) (Attributes, e
return n.xattrsCache, nil
}
// Xattrs returns the extended attributes of the node. If the attributes have already
// been cached they are not read from disk again.
// Xattrs returns the extended attributes of the node, acquiring the metadata lock as
// needed. If the attributes have already been cached they are not read from disk
// again.
func (n *Node) Xattrs(ctx context.Context) (Attributes, error) {
return n.XattrsWithReader(ctx, nil)
return n.loadXattrs(func() (Attributes, error) {
return n.lu.MetadataBackend().All(ctx, n)
})
}
// Xattr returns an extended attribute of the node. If the attributes have already
// been cached it is not read from disk again.
func (n *Node) Xattr(ctx context.Context, key string) ([]byte, error) {
path := n.InternalPath()
if path == "" {
// Do not try to read the attribute of an non-existing node
// Do not try to read the attribute of a non-existing node
return []byte{}, fs.ErrNotExist
}
if n.ID == "" {
// Do not try to read the attribute of an empty node. The InternalPath points to the
// base nodes directory in this case.
return []byte{}, &xattr.Error{Op: "node.Xattr", Path: path, Name: key, Err: xattr.ENOATTR}
attrs, err := n.loadXattrs(func() (Attributes, error) {
return n.lu.MetadataBackend().All(ctx, n)
})
if err != nil {
return nil, err
}
if n.xattrsCache == nil {
attrs, err := n.lu.MetadataBackend().All(ctx, n)
if err != nil {
return []byte{}, err
}
n.xattrsCache = attrs
}
if val, ok := n.xattrsCache[key]; ok {
if val, ok := attrs[key]; ok {
return val, nil
}
// wrap the error as xattr does

2
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go generated vendored
View File

@@ -127,7 +127,7 @@ func New(m map[string]interface{}) (*Options, error) {
o.GatewayAddr = sharedconf.GetGatewaySVC(o.GatewayAddr)
if o.MetadataBackend == "" {
o.MetadataBackend = "xattrs"
o.MetadataBackend = "messagepack"
}
// ensure user layout has no starting or trailing /

7
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisions.go generated vendored
View File

@@ -27,7 +27,6 @@ import (
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
@@ -99,12 +98,12 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer
}
// write lock node before copying metadata
f, err := lockedfile.OpenFile(fs.lu.MetadataBackend().LockfilePath(n), os.O_RDWR|os.O_CREATE, 0600)
unlock, err := fs.lu.MetadataBackend().Lock(n)
if err != nil {
return err
}
defer func() {
_ = f.Close()
_ = unlock()
_ = os.Remove(fs.lu.MetadataBackend().LockfilePath(n))
}()
@@ -116,7 +115,7 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer
}
// create a revision of the current node
if _, err := fs.tp.CreateRevision(ctx, n, mtime.UTC().Format(time.RFC3339Nano), f); err != nil {
if _, err := fs.tp.CreateRevision(ctx, n, mtime.UTC().Format(time.RFC3339Nano)); err != nil {
return err
}

40
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaces.go generated vendored
View File

@@ -196,7 +196,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr
metadata.SetString(prefixes.SpaceAliasAttr, alias)
}
if err := root.SetXattrsWithContext(ctx, metadata, true); err != nil {
if err := root.SetXattrsWithContext(ctx, metadata); err != nil {
return nil, err
}
@@ -697,7 +697,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up
}
metadata[prefixes.TreeMTimeAttr] = []byte(time.Now().UTC().Format(time.RFC3339Nano))
err = spaceNode.SetXattrsWithContext(ctx, metadata, true)
err = spaceNode.SetXattrsWithContext(ctx, metadata)
if err != nil {
return nil, err
}
@@ -751,7 +751,6 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De
return errtypes.NewErrtypeFromStatus(status.NewInvalid(ctx, "can't purge enabled space"))
}
// TODO invalidate ALL indexes in msgpack, not only by type
spaceType, err := n.XattrString(ctx, prefixes.SpaceTypeAttr)
if err != nil {
return err
@@ -760,6 +759,39 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De
return err
}
// the by-type index is handled above; also remove the space from the user
// and group indexes for the owner and every grantee, or they keep stale
// entries pointing at the purged space. best effort, like the expired
// grant removal: log and continue.
// https://github.com/opencloud-eu/opencloud/issues/2985
sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Logger()
if ownerID, err := n.XattrString(ctx, prefixes.OwnerIDAttr); err != nil {
sublog.Error().Err(err).Msg("could not read space owner")
} else if ownerID != "" {
// remove from user index
if err := fs.userSpaceIndex.Remove(ownerID, spaceID); err != nil {
sublog.Error().Err(err).Msg("could not remove owner from user index")
}
}
grants, err := n.ListGrants(ctx)
if err != nil {
sublog.Error().Err(err).Msg("could not list grants")
}
for _, g := range grants {
switch g.Grantee.Type {
case provider.GranteeType_GRANTEE_TYPE_USER:
// remove from user index
if err := fs.userSpaceIndex.Remove(g.Grantee.GetUserId().GetOpaqueId(), spaceID); err != nil {
sublog.Error().Err(err).Str("grantee", g.Grantee.GetUserId().GetOpaqueId()).Msg("could not remove user from user index")
}
case provider.GranteeType_GRANTEE_TYPE_GROUP:
// remove from group index
if err := fs.groupSpaceIndex.Remove(g.Grantee.GetGroupId().GetOpaqueId(), spaceID); err != nil {
sublog.Error().Err(err).Str("grantee", g.Grantee.GetGroupId().GetOpaqueId()).Msg("could not remove group from group index")
}
}
}
// invalidate cache
if err := fs.lu.MetadataBackend().Purge(ctx, n); err != nil {
return err
@@ -935,7 +967,7 @@ func (fs *Decomposedfs) StorageSpaceFromNode(ctx context.Context, n *node.Node,
// This way we don't have to have a cron job checking the grants in regular intervals.
// The tradeof obviously is that this code is here.
if isGrantExpired(g) {
if err := n.DeleteGrant(ctx, g, true); err != nil {
if err := n.DeleteGrant(ctx, g); err != nil {
sublog.Error().Err(err).Str("grantee", id).
Msg("failed to delete expired space grant")
}

6
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/timemanager/timemanager.go generated vendored
View File

@@ -54,7 +54,7 @@ func (dtm *Manager) MTime(ctx context.Context, n *node.Node) (time.Time, error)
// If the time is nil, the attribute is removed.
func (dtm *Manager) SetMTime(ctx context.Context, n *node.Node, mtime *time.Time) error {
if mtime == nil {
return n.RemoveXattr(ctx, prefixes.MTimeAttr, true)
return n.RemoveXattr(ctx, prefixes.MTimeAttr)
}
return n.SetXattrString(ctx, prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano))
}
@@ -75,7 +75,7 @@ func (dtm *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error)
// If the time is nil, the attribute is removed.
func (dtm *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
if tmtime == nil {
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true)
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr)
}
return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano))
}
@@ -121,7 +121,7 @@ func (dtm *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time,
// If the time is nil, the attribute is removed.
func (dtm *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) {
if t == nil {
return n.RemoveXattr(ctx, prefixes.DTimeAttr, true)
return n.RemoveXattr(ctx, prefixes.DTimeAttr)
}
return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano))
}

35
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/async.go generated vendored
View File

@@ -43,12 +43,6 @@ const (
var _propagationGracePeriod = 3 * time.Minute
type PropagationNode interface {
GetSpaceID() string
GetID() string
InternalPath() string
}
// AsyncPropagator implements asynchronous treetime & treesize propagation
type AsyncPropagator struct {
treeSizeAccounting bool
@@ -130,8 +124,8 @@ func NewAsyncPropagator(treeSizeAccounting, treeTimeAccounting bool, o options.A
now := time.Now()
_ = os.Chtimes(changesDirPath, now, now)
n := node.NewBaseNode(parts[0], strings.TrimSuffix(parts[1], ".processing"), lookup)
p.propagate(context.Background(), n, true, *log)
nodeID := strings.TrimSuffix(parts[1], ".processing")
p.propagate(context.Background(), parts[0], nodeID, true, *log)
}()
}
}
@@ -164,14 +158,17 @@ func (p AsyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff i
SyncTime: time.Now().UTC(),
SizeDiff: sizeDiff,
}
go p.queuePropagation(ctx, n, c, log)
// Hand only the node's logical identity to the goroutine, never the live
// *node.Node: sharing the instance would leak and race its metadata-lock state.
go p.queuePropagation(ctx, n.SpaceID, n.ID, c, log)
return nil
}
func (p AsyncPropagator) queuePropagation(ctx context.Context, n *node.Node, change Change, log zerolog.Logger) {
func (p AsyncPropagator) queuePropagation(ctx context.Context, spaceID, nodeID string, change Change, log zerolog.Logger) {
// add a change to the parent node
changePath := p.changesPath(n.SpaceID, n.ID, uuid.New().String()+".mpk")
changePath := p.changesPath(spaceID, nodeID, uuid.New().String()+".mpk")
data, err := msgpack.Marshal(change)
if err != nil {
@@ -212,7 +209,7 @@ func (p AsyncPropagator) queuePropagation(ctx context.Context, n *node.Node, cha
log.Debug().Msg("propagating")
// add a change to the parent node
changeDirPath := p.changesPath(n.SpaceID, n.ID, "")
changeDirPath := p.changesPath(spaceID, nodeID, "")
// first rename the existing node dir
err = os.Rename(changeDirPath, changeDirPath+".processing")
@@ -224,11 +221,11 @@ func (p AsyncPropagator) queuePropagation(ctx context.Context, n *node.Node, cha
// -> ignore, the previous propagation will pick the new changes up
return
}
p.propagate(ctx, n, false, log)
p.propagate(ctx, spaceID, nodeID, false, log)
}
func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, recalculateTreeSize bool, log zerolog.Logger) {
changeDirPath := p.changesPath(pn.GetSpaceID(), pn.GetID(), "")
func (p AsyncPropagator) propagate(ctx context.Context, spaceID, nodeID string, recalculateTreeSize bool, log zerolog.Logger) {
changeDirPath := p.changesPath(spaceID, nodeID, "")
processingPath := changeDirPath + ".processing"
cleanup := func() {
@@ -286,7 +283,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
attrs := node.Attributes{}
_, subspan = tracer.Start(ctx, "node.LockAndReadNode")
n, unlock, err := node.LockAndReadNode(ctx, p.lookup, pn.GetSpaceID(), pn.GetID(), "", false, nil, false)
n, unlock, err := node.LockAndReadNode(ctx, p.lookup, spaceID, nodeID, "", false, nil, false)
subspan.End()
if err != nil {
if n != nil && !n.Exists {
@@ -385,7 +382,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
log.Debug().Uint64("newSize", newSize).Msg("updated treesize of node")
}
if err = n.SetXattrsWithContext(ctx, attrs, false); err != nil {
if err = n.SetXattrsWithContext(ctx, attrs); err != nil {
log.Error().Err(err).Msg("Failed to update extend attributes of node")
cleanup()
return
@@ -400,7 +397,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
cleanup()
if !n.IsSpaceRoot(ctx) {
p.queuePropagation(ctx, n, pc, log)
p.queuePropagation(ctx, n.SpaceID, n.ID, pc, log)
}
// Check for a changes dir that might have been added meanwhile and pick it up
@@ -416,7 +413,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
// -> ignore, the previous propagation will pick the new changes up
return
}
p.propagate(ctx, n, false, log)
p.propagate(ctx, spaceID, nodeID, false, log)
}
}

2
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/sync.go generated vendored
View File

@@ -190,7 +190,7 @@ func (p SyncPropagator) propagateItem(ctx context.Context, n *node.Node, sTime t
log.Debug().Uint64("newSize", newSize).Msg("updated treesize of parent node")
}
if err = n.SetXattrsWithContext(ctx, attrs, false); err != nil {
if err = n.SetXattrsWithContext(ctx, attrs); err != nil {
log.Error().Err(err).Msg("Failed to update extend attributes of parent node")
return n, true, err
}

16
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/revisions.go generated vendored
View File

@@ -30,7 +30,6 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
@@ -49,8 +48,10 @@ import (
// We can add a background process to move old revisions to a slower storage
// and replace the revision file with a symbolic link in the future, if necessary.
// CreateVersion creates a new version of the node
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string, f *lockedfile.File) (string, error) {
// CreateRevision creates a new version of the node. The caller MUST already hold the
// metadata lock of n, as the node's metadata is read (without re-locking) to copy the
// blob metadata onto the new revision.
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string) (string, error) {
versionNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+version, tp.lookup)
versionPath := versionNode.InternalPath()
@@ -116,13 +117,13 @@ func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string
defer vf.Close()
// copy blob metadata to version node
if err := tp.lookup.CopyMetadataWithSourceLock(ctx, n, versionNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
if err := tp.lookup.CopyMetadata(ctx, n, versionNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr ||
attributeName == prefixes.MTimeAttr
}, f, true); err != nil {
}); err != nil {
return "", err
}
@@ -336,7 +337,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, sourceNode, targetNode meta
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
})
if err != nil {
return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error())
}
@@ -347,8 +348,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, sourceNode, targetNode meta
err = tp.lookup.MetadataBackend().SetMultiple(ctx, targetNode,
map[string][]byte{
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
},
false)
})
if err != nil {
return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error())
}

18
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go generated vendored
View File

@@ -156,7 +156,7 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
return errors.Wrap(err, "Decomposedfs: could not set mtime")
}
}
err = n.SetXattrsWithContext(ctx, attributes, true)
err = n.SetXattrsWithContext(ctx, attributes)
if err != nil {
return err
}
@@ -288,7 +288,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node)
attribs := node.Attributes{}
attribs.SetString(prefixes.ParentidAttr, newNode.ParentID)
attribs.SetString(prefixes.NameAttr, newNode.Name)
if err := oldNode.SetXattrsWithContext(ctx, attribs, true); err != nil {
if err := oldNode.SetXattrsWithContext(ctx, attribs); err != nil {
return errors.Wrap(err, "Decomposedfs: could not update old node attributes")
}
@@ -458,7 +458,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
trashLink := filepath.Join(t.options.Root, "spaces", lookup.Pathify(n.SpaceRoot.ID, 1, 2), "trash", lookup.Pathify(n.ID, 4, 2))
if err := os.MkdirAll(filepath.Dir(trashLink), 0700); err != nil {
// Roll back changes
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
return err
}
@@ -471,7 +471,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
err = os.Symlink("../../../../../nodes/"+lookup.Pathify(n.ID, 4, 2)+node.TrashIDDelimiter+deletionTime, trashLink)
if err != nil {
// Roll back changes
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
return
}
@@ -485,12 +485,12 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
// To roll back changes
// TODO remove symlink
// Roll back changes
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
return
}
err = t.lookup.MetadataBackend().Rename(n, trashNode)
if err != nil {
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
_ = os.Rename(trashPath, nodePath)
return
}
@@ -507,7 +507,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
// TODO revert the rename
// TODO remove symlink
// Roll back changes
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
return
}
@@ -580,7 +580,7 @@ func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPa
// set ParentidAttr to restorePath's node parent id
attrs.SetString(prefixes.ParentidAttr, targetNode.ParentID)
if err = t.lookup.MetadataBackend().SetMultiple(ctx, restoreNode, map[string][]byte(attrs), true); err != nil {
if err = t.lookup.MetadataBackend().SetMultiple(ctx, restoreNode, map[string][]byte(attrs)); err != nil {
return nil, errors.Wrap(err, "Decomposedfs: could not update recycle node")
}
@@ -850,7 +850,7 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
if t.options.TreeTimeAccounting || t.options.TreeSizeAccounting {
attributes[prefixes.PropagationAttr] = []byte("1") // mark the node for propagation
}
return n.SetXattrsWithContext(ctx, attributes, true)
return n.SetXattrsWithContext(ctx, attributes)
}
var nodeIDRegep = regexp.MustCompile(`.*/nodes/([^.]*).*`)

13
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload.go generated vendored
View File

@@ -380,6 +380,19 @@ func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload,
return ul, err
}
// ResolveUpload stats the resource produced by a finished upload, reading it as the upload's
// executant. The session is reconstructed from the given tus FileInfo rather than the session
// store, so it still resolves after a synchronous upload has cleaned up its session. It returns
// the resource info and the executant context, so the caller can derive identity dependent
// values such as the WebDAV permissions. https://github.com/opencloud-eu/opencloud/issues/2409
func (fs *Decomposedfs) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) {
session := fs.sessionStore.SessionFromInfo(info)
ctx = session.Context(ctx)
ref := session.Reference()
ri, err := fs.GetMD(ctx, &ref, nil, nil)
return ri, ctx, err
}
// ListUploadSessions returns the upload sessions for the given filter
func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) {
var sessions []*upload.DecomposedFsSession

25
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go generated vendored
View File

@@ -44,7 +44,6 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/rs/zerolog"
tusd "github.com/tus/tusd/v2/pkg/handler"
)
@@ -100,6 +99,17 @@ func (store DecomposedFsStore) New(ctx context.Context) *DecomposedFsSession {
}
}
// SessionFromInfo wraps an existing tus FileInfo into an upload session without reading the
// session store. It lets callers use an upload's session, e.g. its reference and
// executant context, after the store entry has already been cleaned up, as happens for a
// synchronous upload in its finish response callback.
func (store DecomposedFsStore) SessionFromInfo(info tusd.FileInfo) *DecomposedFsSession {
return &DecomposedFsSession{
store: store,
info: info,
}
}
// List lists all upload sessions
func (store DecomposedFsStore) List(ctx context.Context) ([]*DecomposedFsSession, error) {
uploads := []*DecomposedFsSession{}
@@ -286,7 +296,7 @@ func (store DecomposedFsStore) CreateNodeForUpload(ctx context.Context, session
}
// update node metadata with new blobid etc
err = n.SetXattrsWithContext(ctx, initAttrs, false)
err = n.SetXattrsWithContext(ctx, initAttrs)
if err != nil {
return nil, errors.Wrap(err, "Decomposedfs: could not write metadata")
}
@@ -306,16 +316,11 @@ func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *
defer span.End()
// write lock existing node before reading any metadata
f, err := lockedfile.OpenFile(store.lu.MetadataBackend().LockfilePath(n), os.O_RDWR|os.O_CREATE, 0600)
unlock, err := store.lu.MetadataBackend().Lock(n)
if err != nil {
return nil, err
}
unlock := func() error {
// NOTE: to prevent stale NFS file handles do not remove lock file!
return f.Close()
}
old, _ := node.ReadNode(ctx, store.lu, spaceID, n.ID, "", false, nil, false)
if _, err := node.CheckQuota(ctx, n.SpaceRoot, true, uint64(old.Blobsize), fsize); err != nil {
return unlock, err
@@ -366,7 +371,7 @@ func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *
span.AddEvent("CreateVersion")
timestamp := oldNodeMtime.UTC().Format(time.RFC3339Nano)
versionID := n.ID + node.RevisionIDDelimiter + timestamp
versionPath, err := session.store.tp.CreateRevision(ctx, n, timestamp, f)
versionPath, err := session.store.tp.CreateRevision(ctx, n, timestamp)
if err != nil {
if !errors.Is(err, os.ErrExist) {
return unlock, err
@@ -389,7 +394,7 @@ func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *
}
// clean revision file
if versionPath, err = session.store.tp.CreateRevision(ctx, n, timestamp, f); err != nil {
if versionPath, err = session.store.tp.CreateRevision(ctx, n, timestamp); err != nil {
return unlock, err
}
}

56
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go generated vendored
View File

@@ -35,7 +35,6 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/golang-jwt/jwt/v5"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
tusd "github.com/tus/tusd/v2/pkg/handler"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
@@ -46,6 +45,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -71,6 +71,10 @@ func (session *DecomposedFsSession) WriteChunk(ctx context.Context, _ int64, src
return 0, err
}
defer func() {
// sync the written chunk to disk. This ensures that the upload can be resumed,
// and helps to prevent issues with filesystem/journal freezes at the end of the upload
// when committing a large fsync operation on slow disks.
_ = disk.Fdatasync(file)
_ = file.Close()
}()
@@ -291,7 +295,7 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
ctx, span := tracer.Start(session.Context(ctx), "Finalize")
defer span.End()
revisionNode := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(),
n := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(),
provider.ResourceType_RESOURCE_TYPE_FILE, session.SpaceOwner(), session.store.lu)
var (
@@ -305,19 +309,20 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
case spaceRoot.InternalPath() == "":
return fmt.Errorf("space root for space id %s has no valid internal path", session.SpaceID())
default:
revisionNode.SpaceRoot = spaceRoot
n.SpaceRoot = spaceRoot
}
// lock the node before writing the blob
lockedNode, err := lockedfile.OpenFile(session.store.lu.MetadataBackend().LockfilePath(revisionNode), os.O_RDWR|os.O_CREATE, 0600)
// lock the node before reading its metadata and writing the blob
unlock, err := session.store.lu.MetadataBackend().Lock(n)
if err != nil {
return err
}
defer func() {
_ = lockedNode.Close()
_ = unlock()
}()
attribs, err := revisionNode.XattrsWithReader(ctx, lockedNode)
// Read the node attributes while holding the lock acquired above.
attribs, err := n.Xattrs(ctx)
if err != nil {
return err
}
@@ -327,32 +332,27 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
// another upload on this node is in progress or has finished since we started
if !isProcessing || processingID != session.ID() {
versionID := revisionNode.ID + node.RevisionIDDelimiter + session.MTime().UTC().Format(time.RFC3339Nano)
versionID := n.ID + node.RevisionIDDelimiter + session.MTime().UTC().Format(time.RFC3339Nano)
// There should be a revision node (created by the other upload that finished before us), read it and upload our blob there.
existingRevisionNode, err := node.ReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, spaceRoot, false)
existingRevisionNode, revisionNodeUnlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, spaceRoot, false)
if err != nil || !existingRevisionNode.Exists {
// The revision node has not been created. Likely because the file on disk was modified externally and re-assilimated (watchfs == true)
// Let's create the revision node now and upload the blob to it.
revisionNode, err = session.createRevisionNodeForUpload(ctx, revisionNode, session.MTime().UTC().Format(time.RFC3339Nano), lockedNode)
n, revisionNodeUnlock, err = session.createRevisionNodeForUpload(ctx, n, session.MTime().UTC().Format(time.RFC3339Nano))
if err != nil {
appctx.GetLogger(ctx).Debug().Err(err).Str("versionID", session.MTime().UTC().Format(time.RFC3339Nano)).Msg("failed to create revision node for upload finalization")
return err
}
} else {
revisionNode = existingRevisionNode
n = existingRevisionNode
}
// lock this node as well, before writing the blob
revisionNodeUnlock, err := session.store.lu.MetadataBackend().Lock(revisionNode)
if err != nil {
return err
}
appctx.GetLogger(ctx).Debug().Str("new nodepath", revisionNode.InternalPath()).Msg("uploading to revision node, that was created for us by another upload")
appctx.GetLogger(ctx).Debug().Str("new nodepath", n.InternalPath()).Msg("uploading to revision node, that was created for us by another upload")
defer func() { _ = revisionNodeUnlock() }()
}
// upload the data to the blobstore
_, subspan := tracer.Start(ctx, "WriteBlob")
err = session.store.tp.WriteBlob(revisionNode, session.binPath())
err = session.store.tp.WriteBlob(n, session.binPath())
subspan.End()
if err != nil {
return errors.Wrap(err, "failed to upload file to blobstore")
@@ -361,19 +361,19 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
return nil
}
func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Context, baseNode *node.Node, rev string, lockedNode *lockedfile.File) (*node.Node, error) {
func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Context, baseNode *node.Node, rev string) (*node.Node, func() error, error) {
versionID := baseNode.ID + node.RevisionIDDelimiter + rev
log := appctx.GetLogger(ctx)
_, err := session.store.tp.CreateRevision(ctx, baseNode, rev, lockedNode)
_, err := session.store.tp.CreateRevision(ctx, baseNode, rev)
if err != nil {
log.Error().Err(err).Str("versionID", versionID).Msg("failed to create revision node for upload")
return nil, err
return nil, nil, err
}
// FIXME: We already calculated the checksums in FinishUpload, we should maybe pass them via the session instead of recalculating them here
sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath())
if err != nil {
return nil, err
return nil, nil, err
}
// update checksums
@@ -382,7 +382,7 @@ func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Cont
prefixes.ChecksumPrefix + "md5": md5h.Sum(nil),
prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil),
}
revisionNode, err := node.ReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, baseNode.SpaceRoot, false)
revisionNode, unlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, baseNode.SpaceRoot, false)
if err == nil {
mtime := session.MTime()
attrs.SetString(prefixes.BlobIDAttr, session.ID())
@@ -392,14 +392,16 @@ func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Cont
err = session.store.lu.TimeManager().OverrideMtime(ctx, revisionNode, &attrs, mtime)
if err != nil {
return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime")
_ = unlock()
return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime")
}
err = revisionNode.SetXattrsWithContext(ctx, attrs, false)
err = revisionNode.SetXattrsWithContext(ctx, attrs)
if err != nil {
return nil, errors.Wrap(err, "Decomposedfs: failed to set node attributes")
_ = unlock()
return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set node attributes")
}
}
return revisionNode, err
return revisionNode, unlock, err
}
func checkHash(expected string, h hash.Hash) error {

Some files were not shown because too many files have changed in this diff Show More