From 6bf5eccfa7f731f1a856bdd0d3767aa409773a7c Mon Sep 17 00:00:00 2001 From: Gani Georgiev Date: Thu, 15 Jan 2026 14:27:53 +0200 Subject: [PATCH] added strftime filter function --- CHANGELOG.md | 17 +- core/record_field_resolver.go | 30 +- core/record_field_resolver_multi_match.go | 70 ----- core/record_field_resolver_runner.go | 144 ++++----- core/record_field_resolver_test.go | 21 ++ tools/search/multi_match_subquery.go | 70 +++++ tools/search/multi_match_subquery_test.go | 52 ++++ tools/search/simple_field_resolver.go | 2 +- tools/search/token_functions.go | 136 +++++++++ tools/search/token_functions_test.go | 343 ++++++++++++++++++++++ 10 files changed, 726 insertions(+), 159 deletions(-) delete mode 100644 core/record_field_resolver_multi_match.go create mode 100644 tools/search/multi_match_subquery.go create mode 100644 tools/search/multi_match_subquery_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 84061f71..4460f29a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,25 @@ -## v0.36.0-rc.1 +## v0.36.0 (WIP) - Minor list query and API rules optimizations: - Removed unnecessery correlated subquery expression when using back-relations via single `relation` field. - Replaced `DISTINCT` with `GROUP BY id` when rows deduplication is needed and when deemed safe. _This should help with having a more stable and predictable performance even if the collection records are on the larger side._ + For some queries and data sets the above 2 optimizations have shown significant improvements but if you notice a performance degradation after upgrading, + please open a Q&A discussion with export of your collections structure and the problematic request so that it can be analyzed. + +- Added [`strftime(format, [timevalue, modifiers...])`](@todo link to docs) date formatting filter and API rules function. + It operates similarly to the equivalent [SQLite `strftime` builtin function](https://sqlite.org/lang_datefunc.html) + with the exception that for some operators the result will be coalesced for consistency with the non-nullable behavior of the default PocketBase fields. + Multi-match expressions are also supported and works the same as if the collection field is referenced, for example: + ```js + // requires any/at-least-one-of multiRel records to have created date matching the formatted string "2026-01" + strftime('%Y-%m', multiRel.created) ?= "2026-01" + + // requires ALL multiRel records to have created date matching the formatted string "2026-01" + strftime('%Y-%m', multiRel.created) = "2026-01" + ``` + ## v0.35.1 diff --git a/core/record_field_resolver.go b/core/record_field_resolver.go index da0759dd..a2de214f 100644 --- a/core/record_field_resolver.go +++ b/core/record_field_resolver.go @@ -48,7 +48,7 @@ type RecordFieldResolver struct { requestInfo *RequestInfo staticRequestInfo map[string]any allowedFields []string - joins []*join + joins []*search.Join allowHiddenFields bool // --- listRuleJoins map[string]*Collection // tableAlias->collection @@ -88,7 +88,7 @@ func NewRecordFieldResolver( baseCollection: baseCollection, requestInfo: requestInfo, allowHiddenFields: allowHiddenFields, // note: it is not based only on the requestInfo.auth since it could be used by a non-request internal method - joins: []*join{}, + joins: []*search.Join{}, allowedFields: []string{ `^\w+[\w\.\:]*$`, `^\@request\.context$`, @@ -133,8 +133,8 @@ func (r *RecordFieldResolver) UpdateQuery(query *dbx.SelectQuery) error { for _, join := range r.joins { query.LeftJoin( - (join.tableName + " " + join.tableAlias), - join.on, + (join.TableName + " " + join.TableAlias), + join.On, ) } } @@ -158,11 +158,11 @@ func (r *RecordFieldResolver) updateQueryWithCollectionListRule(c *Collection, t } cloneR := *r - cloneR.joins = []*join{} + cloneR.joins = []*search.Join{} cloneR.baseCollection = c cloneR.baseCollectionAlias = tableAlias cloneR.allowHiddenFields = true - cloneR.joinAliasSuffix = security.PseudorandomString(6) + cloneR.joinAliasSuffix = security.PseudorandomString(8) expr, err := search.FilterData(*c.ListRule).BuildExpr(&cloneR) if err != nil { @@ -176,8 +176,8 @@ func (r *RecordFieldResolver) updateQueryWithCollectionListRule(c *Collection, t for _, j := range cloneR.joins { query.LeftJoin( - (j.tableName + " " + j.tableAlias), - j.on, + (j.TableName + " " + j.TableAlias), + j.On, ) } } @@ -344,7 +344,7 @@ func (r *RecordFieldResolver) resolveStaticRequestField(path ...string) (*search return &search.ResolverResult{Identifier: "NULL"}, nil } - placeholder := "f" + security.PseudorandomString(8) + placeholder := "f" + security.PseudorandomString(10) // @todo consider deprecating with the introduction of filter functions if modifier == lowerModifier { @@ -369,10 +369,10 @@ func (r *RecordFieldResolver) loadCollection(collectionNameOrId string) (*Collec } func (r *RecordFieldResolver) registerJoin(tableName string, tableAlias string, on dbx.Expression) error { - newJoin := &join{ - tableName: tableName, - tableAlias: tableAlias, - on: on, + newJoin := &search.Join{ + TableName: tableName, + TableAlias: tableAlias, + On: on, } // (see updateQueryWithCollectionListRule) @@ -389,13 +389,13 @@ func (r *RecordFieldResolver) registerJoin(tableName string, tableAlias string, if r.listRuleJoins == nil { r.listRuleJoins = map[string]*Collection{} } - r.listRuleJoins[newJoin.tableAlias] = c + r.listRuleJoins[newJoin.TableAlias] = c } } // replace existing join for i, j := range r.joins { - if j.tableAlias == newJoin.tableAlias { + if j.TableAlias == newJoin.TableAlias { r.joins[i] = newJoin return nil } diff --git a/core/record_field_resolver_multi_match.go b/core/record_field_resolver_multi_match.go deleted file mode 100644 index bb51efdf..00000000 --- a/core/record_field_resolver_multi_match.go +++ /dev/null @@ -1,70 +0,0 @@ -package core - -import ( - "fmt" - "strings" - - "github.com/pocketbase/dbx" -) - -var _ dbx.Expression = (*multiMatchSubquery)(nil) - -// join defines the specification for a single SQL JOIN clause. -type join struct { - tableName string - tableAlias string - on dbx.Expression -} - -// multiMatchSubquery defines a record multi-match subquery expression. -type multiMatchSubquery struct { - baseTableAlias string - fromTableName string - fromTableAlias string - valueIdentifier string - joins []*join - params dbx.Params -} - -// Build converts the expression into a SQL fragment. -// -// Implements [dbx.Expression] interface. -func (m *multiMatchSubquery) Build(db *dbx.DB, params dbx.Params) string { - if m.baseTableAlias == "" || m.fromTableName == "" || m.fromTableAlias == "" { - return "0=1" - } - - if params == nil { - params = m.params - } else { - // merge by updating the parent params - for k, v := range m.params { - params[k] = v - } - } - - var mergedJoins strings.Builder - for i, j := range m.joins { - if i > 0 { - mergedJoins.WriteString(" ") - } - mergedJoins.WriteString("LEFT JOIN ") - mergedJoins.WriteString(db.QuoteTableName(j.tableName)) - mergedJoins.WriteString(" ") - mergedJoins.WriteString(db.QuoteTableName(j.tableAlias)) - if j.on != nil { - mergedJoins.WriteString(" ON ") - mergedJoins.WriteString(j.on.Build(db, params)) - } - } - - return fmt.Sprintf( - `SELECT %s as [[multiMatchValue]] FROM %s %s %s WHERE %s = %s`, - db.QuoteColumnName(m.valueIdentifier), - db.QuoteTableName(m.fromTableName), - db.QuoteTableName(m.fromTableAlias), - mergedJoins.String(), - db.QuoteColumnName(m.fromTableAlias+".id"), - db.QuoteColumnName(m.baseTableAlias+".id"), - ) -} diff --git a/core/record_field_resolver_runner.go b/core/record_field_resolver_runner.go index 0d4f406b..0b3a5e20 100644 --- a/core/record_field_resolver_runner.go +++ b/core/record_field_resolver_runner.go @@ -49,13 +49,13 @@ type runner struct { // shared processing state // --------------------------------------------------------------- - activeProps []string // holds the active props that remains to be processed - activeCollectionName string // the last used collection name - activeTableAlias string // the last used table alias - nullifyMisingField bool // indicating whether to return null on missing field or return an error - withMultiMatch bool // indicates whether to attach a multiMatchSubquery condition to the ResolverResult - multiMatchActiveTableAlias string // the last used multi-match table alias - multiMatch *multiMatchSubquery // the multi-match subquery expression generated from the fieldName + activeProps []string // holds the active props that remains to be processed + activeCollectionName string // the last used collection name + activeTableAlias string // the last used table alias + nullifyMisingField bool // indicating whether to return null on missing field or return an error + withMultiMatch bool // indicates whether to attach a MultiMatchSubquery condition to the ResolverResult + multiMatchActiveTableAlias string // the last used multi-match table alias + multiMatch *search.MultiMatchSubquery // the multi-match subquery expression generated from the fieldName } func (r *runner) run() (*search.ResolverResult, error) { @@ -144,13 +144,13 @@ func (r *runner) prepare() { r.nullifyMisingField = r.activeProps[0] == "@request" // prepare a multi-match subquery - r.multiMatch = &multiMatchSubquery{ - baseTableAlias: r.activeTableAlias, - params: dbx.Params{}, + r.multiMatch = &search.MultiMatchSubquery{ + TargetTableAlias: r.activeTableAlias, + Params: dbx.Params{}, } - r.multiMatch.fromTableName = inflector.Columnify(r.activeCollectionName) - r.multiMatch.fromTableAlias = "__mm_" + r.activeTableAlias - r.multiMatchActiveTableAlias = r.multiMatch.fromTableAlias + r.multiMatch.FromTableName = inflector.Columnify(r.activeCollectionName) + r.multiMatch.FromTableAlias = "__mm_" + r.activeTableAlias + r.multiMatchActiveTableAlias = r.multiMatch.FromTableAlias r.withMultiMatch = false } @@ -185,9 +185,9 @@ func (r *runner) processCollectionField() (*search.ResolverResult, error) { // join the collection to the multi-match subquery r.multiMatchActiveTableAlias = "__mm_" + r.activeTableAlias - r.multiMatch.joins = append(r.multiMatch.joins, &join{ - tableName: inflector.Columnify(collection.Name), - tableAlias: r.multiMatchActiveTableAlias, + r.multiMatch.Joins = append(r.multiMatch.Joins, &search.Join{ + TableName: inflector.Columnify(collection.Name), + TableAlias: r.multiMatchActiveTableAlias, }) // leave only the collection fields @@ -230,12 +230,12 @@ func (r *runner) processRequestAuthField() (*search.ResolverResult, error) { // join the auth collection to the multi-match subquery r.multiMatchActiveTableAlias = "__mm_" + r.activeTableAlias - r.multiMatch.joins = append( - r.multiMatch.joins, - &join{ - tableName: inflector.Columnify(r.activeCollectionName), - tableAlias: r.multiMatchActiveTableAlias, - on: dbx.HashExp{ + r.multiMatch.Joins = append( + r.multiMatch.Joins, + &search.Join{ + TableName: inflector.Columnify(r.activeCollectionName), + TableAlias: r.multiMatchActiveTableAlias, + On: dbx.HashExp{ (r.multiMatchActiveTableAlias + ".id"): r.resolver.requestInfo.Auth.Id, }, }, @@ -282,7 +282,7 @@ func (r *runner) processRequestBodyChangedModifier(bodyField Field) (*search.Res return nil, err } - placeholder := "@changed@" + name + security.PseudorandomString(6) + placeholder := "@changed@" + name + security.PseudorandomString(8) result := &search.ResolverResult{ Identifier: placeholder, @@ -302,7 +302,7 @@ func (r *runner) processRequestBodyChangedModifier(bodyField Field) (*search.Res func (r *runner) processRequestBodyLowerModifier(bodyField Field) (*search.ResolverResult, error) { rawValue := cast.ToString(r.resolver.requestInfo.Body[bodyField.GetName()]) - placeholder := "infoLower" + bodyField.GetName() + security.PseudorandomString(6) + placeholder := "infoLower" + bodyField.GetName() + security.PseudorandomString(8) result := &search.ResolverResult{ Identifier: "LOWER({:" + placeholder + "})", @@ -338,7 +338,7 @@ func (r *runner) processRequestBodyEachModifier(bodyField Field) (*search.Resolv return nil, fmt.Errorf("cannot serialize the data for field %q", r.activeProps[2]) } - placeholder := "dataEach" + security.PseudorandomString(6) + placeholder := "dataEach" + security.PseudorandomString(8) cleanFieldName := inflector.Columnify(bodyField.GetName()) jeTable := fmt.Sprintf("json_each({:%s})", placeholder) jeAlias := "__dataEach_je_" + cleanFieldName + r.resolver.joinAliasSuffix @@ -362,12 +362,12 @@ func (r *runner) processRequestBodyEachModifier(bodyField Field) (*search.Resolv jeTable2 := fmt.Sprintf("json_each({:%s})", placeholder2) jeAlias2 := "__mm_" + jeAlias - r.multiMatch.joins = append(r.multiMatch.joins, &join{ - tableName: jeTable2, - tableAlias: jeAlias2, + r.multiMatch.Joins = append(r.multiMatch.Joins, &search.Join{ + TableName: jeTable2, + TableAlias: jeAlias2, }) - r.multiMatch.params[placeholder2] = bodyItemsRaw - r.multiMatch.valueIdentifier = fmt.Sprintf("[[%s.value]]", jeAlias2) + r.multiMatch.Params[placeholder2] = bodyItemsRaw + r.multiMatch.ValueIdentifier = fmt.Sprintf("[[%s.value]]", jeAlias2) result.MultiMatchSubQuery = r.multiMatch } @@ -416,12 +416,12 @@ func (r *runner) processRequestBodyRelationField(bodyField Field) (*search.Resol // join the data rel collection to the multi-match subquery r.multiMatchActiveTableAlias = "__mm_" + r.activeTableAlias - r.multiMatch.joins = append( - r.multiMatch.joins, - &join{ - tableName: r.activeCollectionName, - tableAlias: r.multiMatchActiveTableAlias, - on: dbx.In( + r.multiMatch.Joins = append( + r.multiMatch.Joins, + &search.Join{ + TableName: r.activeCollectionName, + TableAlias: r.multiMatchActiveTableAlias, + On: dbx.In( fmt.Sprintf("[[%s.id]]", r.multiMatchActiveTableAlias), list.ToInterfaceSlice(dataRelIds)..., ), @@ -482,7 +482,7 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) { } if r.withMultiMatch { - r.multiMatch.valueIdentifier = dbutils.JSONExtract(r.multiMatchActiveTableAlias+"."+inflector.Columnify(prop), jsonPathStr) + r.multiMatch.ValueIdentifier = dbutils.JSONExtract(r.multiMatchActiveTableAlias+"."+inflector.Columnify(prop), jsonPathStr) result.MultiMatchSubQuery = r.multiMatch } @@ -598,22 +598,22 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) { newTableAlias2 := r.multiMatchActiveTableAlias + "_" + cleanProp + r.resolver.joinAliasSuffix if !isBackRelMultiple { - r.multiMatch.joins = append( - r.multiMatch.joins, - &join{ - tableName: newCollectionName, - tableAlias: newTableAlias2, - on: dbx.NewExp(fmt.Sprintf("[[%s.%s]] = [[%s.id]]", newTableAlias2, cleanBackFieldName, r.multiMatchActiveTableAlias)), + r.multiMatch.Joins = append( + r.multiMatch.Joins, + &search.Join{ + TableName: newCollectionName, + TableAlias: newTableAlias2, + On: dbx.NewExp(fmt.Sprintf("[[%s.%s]] = [[%s.id]]", newTableAlias2, cleanBackFieldName, r.multiMatchActiveTableAlias)), }, ) } else { jeAlias2 := "__je_" + newTableAlias2 - r.multiMatch.joins = append( - r.multiMatch.joins, - &join{ - tableName: newCollectionName, - tableAlias: newTableAlias2, - on: dbx.NewExp(fmt.Sprintf( + r.multiMatch.Joins = append( + r.multiMatch.Joins, + &search.Join{ + TableName: newCollectionName, + TableAlias: newTableAlias2, + On: dbx.NewExp(fmt.Sprintf( "[[%s.id]] IN (SELECT [[%s.value]] FROM %s {{%s}})", r.multiMatchActiveTableAlias, jeAlias2, @@ -702,26 +702,26 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) { prefixedFieldName2 := r.multiMatchActiveTableAlias + "." + cleanFieldName if !relField.IsMultiple() { - r.multiMatch.joins = append( - r.multiMatch.joins, - &join{ - tableName: inflector.Columnify(newCollectionName), - tableAlias: newTableAlias2, - on: dbx.NewExp(fmt.Sprintf("[[%s.id]] = [[%s]]", newTableAlias2, prefixedFieldName2)), + r.multiMatch.Joins = append( + r.multiMatch.Joins, + &search.Join{ + TableName: inflector.Columnify(newCollectionName), + TableAlias: newTableAlias2, + On: dbx.NewExp(fmt.Sprintf("[[%s.id]] = [[%s]]", newTableAlias2, prefixedFieldName2)), }, ) } else { jeAlias2 := r.multiMatchActiveTableAlias + "_" + cleanFieldName + "_je" - r.multiMatch.joins = append( - r.multiMatch.joins, - &join{ - tableName: dbutils.JSONEach(prefixedFieldName2), - tableAlias: jeAlias2, + r.multiMatch.Joins = append( + r.multiMatch.Joins, + &search.Join{ + TableName: dbutils.JSONEach(prefixedFieldName2), + TableAlias: jeAlias2, }, - &join{ - tableName: inflector.Columnify(newCollectionName), - tableAlias: newTableAlias2, - on: dbx.NewExp(fmt.Sprintf("[[%s.id]] = [[%s.value]]", newTableAlias2, jeAlias2)), + &search.Join{ + TableName: inflector.Columnify(newCollectionName), + TableAlias: newTableAlias2, + On: dbx.NewExp(fmt.Sprintf("[[%s.id]] = [[%s.value]]", newTableAlias2, jeAlias2)), }, ) } @@ -766,7 +766,7 @@ func (r *runner) finalizeActivePropsProcessing(collection *Collection, prop stri if r.withMultiMatch { jePair2 := r.multiMatchActiveTableAlias + "." + cleanFieldName - r.multiMatch.valueIdentifier = dbutils.JSONArrayLength(jePair2) + r.multiMatch.ValueIdentifier = dbutils.JSONArrayLength(jePair2) result.MultiMatchSubQuery = r.multiMatch } @@ -796,11 +796,11 @@ func (r *runner) finalizeActivePropsProcessing(collection *Collection, prop stri jePair2 := r.multiMatchActiveTableAlias + "." + cleanFieldName jeAlias2 := "__je_" + r.multiMatchActiveTableAlias + "_" + cleanFieldName + r.resolver.joinAliasSuffix - r.multiMatch.joins = append(r.multiMatch.joins, &join{ - tableName: dbutils.JSONEach(jePair2), - tableAlias: jeAlias2, + r.multiMatch.Joins = append(r.multiMatch.Joins, &search.Join{ + TableName: dbutils.JSONEach(jePair2), + TableAlias: jeAlias2, }) - r.multiMatch.valueIdentifier = fmt.Sprintf("[[%s.value]]", jeAlias2) + r.multiMatch.ValueIdentifier = fmt.Sprintf("[[%s.value]]", jeAlias2) result.MultiMatchSubQuery = r.multiMatch } @@ -815,7 +815,7 @@ func (r *runner) finalizeActivePropsProcessing(collection *Collection, prop stri } if r.withMultiMatch { - r.multiMatch.valueIdentifier = "[[" + r.multiMatchActiveTableAlias + "." + cleanFieldName + "]]" + r.multiMatch.ValueIdentifier = "[[" + r.multiMatchActiveTableAlias + "." + cleanFieldName + "]]" result.MultiMatchSubQuery = r.multiMatch } @@ -837,7 +837,7 @@ func (r *runner) finalizeActivePropsProcessing(collection *Collection, prop stri result.NoCoalesce = true result.Identifier = dbutils.JSONExtract(r.activeTableAlias+"."+cleanFieldName, "") if r.withMultiMatch { - r.multiMatch.valueIdentifier = dbutils.JSONExtract(r.multiMatchActiveTableAlias+"."+cleanFieldName, "") + r.multiMatch.ValueIdentifier = dbutils.JSONExtract(r.multiMatchActiveTableAlias+"."+cleanFieldName, "") } } @@ -845,7 +845,7 @@ func (r *runner) finalizeActivePropsProcessing(collection *Collection, prop stri if modifier == lowerModifier { result.Identifier = "LOWER(" + result.Identifier + ")" if r.withMultiMatch { - r.multiMatch.valueIdentifier = "LOWER(" + r.multiMatch.valueIdentifier + ")" + r.multiMatch.ValueIdentifier = "LOWER(" + r.multiMatch.ValueIdentifier + ")" } } diff --git a/core/record_field_resolver_test.go b/core/record_field_resolver_test.go index 5416f556..143a0ce9 100644 --- a/core/record_field_resolver_test.go +++ b/core/record_field_resolver_test.go @@ -644,6 +644,27 @@ func TestRecordFieldResolverUpdateQuery(t *testing.T) { false, "SELECT `view1`.* FROM `view1` WHERE (([[view1.point]] = '' OR [[view1.point]] IS NULL) OR (CASE WHEN json_valid([[view1.point]]) THEN JSON_EXTRACT([[view1.point]], '$.lat') ELSE JSON_EXTRACT(json_object('pb', [[view1.point]]), '$.pb.lat') END) > {:TEST} OR (CASE WHEN json_valid([[view1.point]]) THEN JSON_EXTRACT([[view1.point]], '$.lon') ELSE JSON_EXTRACT(json_object('pb', [[view1.point]]), '$.pb.lon') END) < {:TEST} OR (CASE WHEN json_valid([[view1.point]]) THEN JSON_EXTRACT([[view1.point]], '$.something') ELSE JSON_EXTRACT(json_object('pb', [[view1.point]]), '$.pb.something') END) > {:TEST})", }, + { + "strftime with fixed string as time-value", + "demo5", + "strftime('%Y-%m', '2026-01-01') = true", + false, + "SELECT `demo5`.* FROM `demo5` WHERE strftime({:TEST},{:TEST}) = 1", + }, + { + "strftime without multi-match", + "demo5", + "strftime('%Y-%m', rel_one.created) = true", + false, + "SELECT `demo5`.* FROM `demo5` LEFT JOIN `demo4` `demo5_rel_one` ON [[demo5_rel_one.id]] = [[demo5.rel_one]] WHERE strftime({:TEST},[[demo5_rel_one.created]]) = 1 GROUP BY `demo5`.`id`", + }, + { + "strftime with multi-match", + "demo5", + "strftime('%Y-%m', rel_many.created) = true", + false, + "SELECT `demo5`.* FROM `demo5` LEFT JOIN json_each(CASE WHEN iif(json_valid([[demo5.rel_many]]), json_type([[demo5.rel_many]])='array', FALSE) THEN [[demo5.rel_many]] ELSE json_array([[demo5.rel_many]]) END) `__je_demo5_rel_many` LEFT JOIN `demo4` `demo5_rel_many` ON [[demo5_rel_many.id]] = [[__je_demo5_rel_many.value]] WHERE (((strftime({:TEST},[[demo5_rel_many.created]]) = 1) AND (NOT EXISTS (SELECT 1 FROM (SELECT strftime({:TEST},[[__mm_demo5_rel_many.created]]) as [[multiMatchValue]] FROM `demo5` `__mm_demo5` LEFT JOIN json_each(CASE WHEN iif(json_valid([[__mm_demo5.rel_many]]), json_type([[__mm_demo5.rel_many]])='array', FALSE) THEN [[__mm_demo5.rel_many]] ELSE json_array([[__mm_demo5.rel_many]]) END) `__mm_demo5_rel_many_je` LEFT JOIN `demo4` `__mm_demo5_rel_many` ON [[__mm_demo5_rel_many.id]] = [[__mm_demo5_rel_many_je.value]] WHERE `__mm_demo5`.`id` = `demo5`.`id`) {{__smTEST}} WHERE NOT ([[__smTEST.multiMatchValue]] = 1))))) GROUP BY `demo5`.`id`", + }, } for _, s := range scenarios { diff --git a/tools/search/multi_match_subquery.go b/tools/search/multi_match_subquery.go new file mode 100644 index 00000000..d1ed8bac --- /dev/null +++ b/tools/search/multi_match_subquery.go @@ -0,0 +1,70 @@ +package search + +import ( + "fmt" + "strings" + + "github.com/pocketbase/dbx" +) + +var _ dbx.Expression = (*MultiMatchSubquery)(nil) + +// Join defines common fields required for a single SQL JOIN clause. +type Join struct { + TableName string + TableAlias string + On dbx.Expression +} + +// MultiMatchSubquery defines a multi-match record subquery expression. +type MultiMatchSubquery struct { + TargetTableAlias string + FromTableName string + FromTableAlias string + ValueIdentifier string + Joins []*Join + Params dbx.Params +} + +// Build converts the expression into a SQL fragment. +// +// Implements [dbx.Expression] interface. +func (m *MultiMatchSubquery) Build(db *dbx.DB, params dbx.Params) string { + if m.TargetTableAlias == "" || m.FromTableName == "" || m.FromTableAlias == "" { + return "0=1" + } + + if params == nil { + params = m.Params + } else { + // merge by updating the parent params + for k, v := range m.Params { + params[k] = v + } + } + + var mergedJoins strings.Builder + for i, j := range m.Joins { + if i > 0 { + mergedJoins.WriteString(" ") + } + mergedJoins.WriteString("LEFT JOIN ") + mergedJoins.WriteString(db.QuoteTableName(j.TableName)) + mergedJoins.WriteString(" ") + mergedJoins.WriteString(db.QuoteTableName(j.TableAlias)) + if j.On != nil { + mergedJoins.WriteString(" ON ") + mergedJoins.WriteString(j.On.Build(db, params)) + } + } + + return fmt.Sprintf( + `SELECT %s as [[multiMatchValue]] FROM %s %s %s WHERE %s = %s`, + db.QuoteColumnName(m.ValueIdentifier), + db.QuoteTableName(m.FromTableName), + db.QuoteTableName(m.FromTableAlias), + mergedJoins.String(), + db.QuoteColumnName(m.FromTableAlias+".id"), + db.QuoteColumnName(m.TargetTableAlias+".id"), + ) +} diff --git a/tools/search/multi_match_subquery_test.go b/tools/search/multi_match_subquery_test.go new file mode 100644 index 00000000..6caae968 --- /dev/null +++ b/tools/search/multi_match_subquery_test.go @@ -0,0 +1,52 @@ +package search_test + +import ( + "bytes" + "database/sql" + "encoding/json" + "testing" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/tools/search" +) + +func TestMultiMatchSubqueryBuild(t *testing.T) { + // create a dummy db + sqlDB, err := sql.Open("sqlite", "file::memory:?cache=shared") + if err != nil { + t.Fatal(err) + } + db := dbx.NewFromDB(sqlDB, "sqlite") + + mm := search.MultiMatchSubquery{ + TargetTableAlias: "test_TargetTableAlias", + FromTableName: "test_FromTableName", + FromTableAlias: "test_FromTableAlias", + ValueIdentifier: "({:mm},{:external})", + Joins: []*search.Join{ + {TableName: "join_table1", TableAlias: "join_alias1"}, + {TableName: "join_table2", TableAlias: "join_alias2", On: dbx.NewExp("123={:join}", dbx.Params{"join": "test_join"})}, + }, + Params: dbx.Params{"mm": "test_mm"}, + } + + params := dbx.Params{"external": "test_external"} + + result := mm.Build(db, params) + + expectedResult := "SELECT ({:mm},{:external}) as [[multiMatchValue]] FROM `test_FromTableName` `test_FromTableAlias` LEFT JOIN `join_table1` `join_alias1` LEFT JOIN `join_table2` `join_alias2` ON 123={:join} WHERE `test_FromTableAlias`.`id` = `test_TargetTableAlias`.`id`" + if expectedResult != result { + t.Fatalf("Expected build result\n%v\ngot\n%v", expectedResult, result) + } + + // the params from all expressions should be merged in the root + rawParams, err := json.Marshal(params) + if err != nil { + t.Fatal(err) + } + + expectedParams := []byte(`{"external":"test_external","join":"test_join","mm":"test_mm"}`) + if !bytes.Equal(rawParams, expectedParams) { + t.Fatalf("Expected final params\n%s\ngot\n%s", expectedParams, rawParams) + } +} diff --git a/tools/search/simple_field_resolver.go b/tools/search/simple_field_resolver.go index bfd96ada..e7970e55 100644 --- a/tools/search/simple_field_resolver.go +++ b/tools/search/simple_field_resolver.go @@ -26,7 +26,7 @@ type ResolverResult struct { // MultiMatchSubQuery is an optional sub query expression that will be added // in addition to the combined ResolverResult expression during build. - MultiMatchSubQuery dbx.Expression + MultiMatchSubQuery *MultiMatchSubquery // AfterBuild is an optional function that will be called after building // and combining the result of both resolved operands/sides in a single expression. diff --git a/tools/search/token_functions.go b/tools/search/token_functions.go index 15f24717..7d257f4d 100644 --- a/tools/search/token_functions.go +++ b/tools/search/token_functions.go @@ -1,9 +1,13 @@ package search import ( + "errors" "fmt" + "slices" + "strings" "github.com/ganigeorgiev/fexpr" + "github.com/pocketbase/dbx" ) var TokenFunctions = map[string]func( @@ -53,4 +57,136 @@ var TokenFunctions = map[string]func( Params: mergeParams(resolvedArgs[0].Params, resolvedArgs[1].Params, resolvedArgs[2].Params, resolvedArgs[3].Params), }, nil }, + + // strftime(format, [timeValue, modifier1, modifier2, ...]) returns + // a date string formatted according to the specified format argument. + // + // It is similar to the builtin SQLite strftime function (https://sqlite.org/lang_datefunc.html). + // + // It accepts 1, 2 or 3+ arguments. + // + // (1) The first (format) argument must be always a formatting string + // with valid substitutions listed in https://sqlite.org/lang_datefunc.html. + // + // (2) The second (time-value) argument is optional and must be either a date string, number or collection field identifier + // that matches one of the formats listed in https://sqlite.org/lang_datefunc.html#time_values. + // + // (3+) The remaining (modifiers) optional arguments are expected to be + // string literals matching the listed modifiers in https://sqlite.org/lang_datefunc.html#modifiers. + // + // Note that an invalid format, time-value, or modifier could result in COALESCE(strftime(...), null) + // for consistency with the non-null nature of the default PocketBase fields. + // + // A multi-match constraint will be also applied in case the time-value + // is an identifier as a result of a multi-value relation field. + "strftime": func(argTokenResolverFunc func(fexpr.Token) (*ResolverResult, error), args ...fexpr.Token) (*ResolverResult, error) { + totalArgs := len(args) + + if totalArgs < 1 { + return nil, fmt.Errorf("[strftime] expected at least 1 arguments, got %d", len(args)) + } + + // limit the number of arguments to prevent abuse + if totalArgs > 10 { + return nil, fmt.Errorf("[strftime] too many arguments (max allowed 10, got %d)", totalArgs) + } + + // format arg + // ----------------------------------------------------------- + if args[0].Type != fexpr.TokenText { + return nil, errors.New("[strftime] expects the first argument to be a format string") + } + + formatArgResult, err := argTokenResolverFunc(args[0]) + if err != nil { + return nil, fmt.Errorf("[strftime] failed to resolve format argument: %w", err) + } + + // no further arguments + if totalArgs == 1 { + formatArgResult.Identifier = "strftime(" + formatArgResult.Identifier + ")" + return formatArgResult, nil + } + + // time-value arg + // ----------------------------------------------------------- + allowedTimeValueTokens := []fexpr.TokenType{fexpr.TokenText, fexpr.TokenIdentifier, fexpr.TokenNumber} + if !slices.Contains(allowedTimeValueTokens, args[1].Type) { + return nil, errors.New("[strftime] expects the second argument to be of a valid time-value type") + } + + timeValueArgResult, err := argTokenResolverFunc(args[1]) + if err != nil { + return nil, fmt.Errorf("[strftime] failed to resolve time-value argument: %w", err) + } + + // modifiers args + // ----------------------------------------------------------- + resolvedModifierArgs := make([]*ResolverResult, totalArgs-2) + for i, arg := range args[2:] { + if arg.Type != fexpr.TokenText { + return nil, fmt.Errorf("[strftime] invalid modifier argument %d - can be only string", i) + } + + resolved, err := argTokenResolverFunc(arg) + if err != nil { + return nil, fmt.Errorf("[strftime] failed to resolve modifier argument %d: %w", i, err) + } + + resolvedModifierArgs[i] = resolved + } + + // generating new ResolverResult + // ----------------------------------------------------------- + result := &ResolverResult{Params: dbx.Params{}} + + identifiers := make([]string, 0, totalArgs) + + identifiers = append(identifiers, formatArgResult.Identifier) + if err = concatUniqueParams(result.Params, formatArgResult.Params); err != nil { + return nil, err + } + + identifiers = append(identifiers, timeValueArgResult.Identifier) + if err = concatUniqueParams(result.Params, timeValueArgResult.Params); err != nil { + return nil, err + } + + for _, m := range resolvedModifierArgs { + identifiers = append(identifiers, m.Identifier) + err = concatUniqueParams(result.Params, m.Params) + if err != nil { + return nil, err + } + } + + result.Identifier = "strftime(" + strings.Join(identifiers, ",") + ")" + + if timeValueArgResult.MultiMatchSubQuery != nil { + // replace the regular time-value identifier with the multi-match one + identifiers[1] = timeValueArgResult.MultiMatchSubQuery.ValueIdentifier + result.MultiMatchSubQuery = timeValueArgResult.MultiMatchSubQuery + result.MultiMatchSubQuery.ValueIdentifier = "strftime(" + strings.Join(identifiers, ",") + ")" + + err = concatUniqueParams(result.MultiMatchSubQuery.Params, result.Params) + if err != nil { + return nil, err + } + } + + return result, nil + }, +} + +func concatUniqueParams(destParams, newParams dbx.Params) error { + for k, v := range newParams { + found, ok := destParams[k] + if ok && v != found { + return fmt.Errorf("conflicting param key %s", k) + } + + destParams[k] = v + } + + return nil } diff --git a/tools/search/token_functions_test.go b/tools/search/token_functions_test.go index bc92f121..a9e50dd1 100644 --- a/tools/search/token_functions_test.go +++ b/tools/search/token_functions_test.go @@ -209,6 +209,345 @@ func TestTokenFunctionsGeoDistanceExec(t *testing.T) { } } +func TestTokenFunctionsStrftime(t *testing.T) { + t.Parallel() + + testDB, err := createTestDB() + if err != nil { + t.Fatal(err) + } + defer testDB.Close() + + fn, ok := TokenFunctions["strftime"] + if !ok { + t.Error("Expected strftime token function to be registered.") + } + + baseTokenResolver := func(t fexpr.Token) (*ResolverResult, error) { + placeholder := "t" + security.PseudorandomString(5) + return &ResolverResult{Identifier: "{:" + placeholder + "}", Params: map[string]any{placeholder: t.Literal}}, nil + } + + scenarios := []struct { + name string + args []fexpr.Token + resolver func(t fexpr.Token) (*ResolverResult, error) + result *ResolverResult + expectErr bool + }{ + { + "no args", + nil, + baseTokenResolver, + nil, + true, + }, + + // format arg + // ----------------------------------------------------------- + { + "(format arg) invalid token type function", + []fexpr.Token{ + {Literal: "abc", Type: fexpr.TokenFunction}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(format arg) invalid token type ws", + []fexpr.Token{ + {Literal: "abc", Type: fexpr.TokenWS}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(format arg) invalid token type number", + []fexpr.Token{ + {Literal: "abc", Type: fexpr.TokenNumber}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(format arg) invalid token type identifier", + []fexpr.Token{ + {Literal: "abc", Type: fexpr.TokenIdentifier}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(format arg) valid token type text", + []fexpr.Token{ + {Literal: "abc", Type: fexpr.TokenText}, + }, + baseTokenResolver, + &ResolverResult{ + Identifier: `strftime({:format})`, + Params: map[string]any{"format": "abc"}, + }, + false, + }, + + // format + time-value args + // ----------------------------------------------------------- + { + "(format arg) invalid token type function", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenFunction}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(format arg) invalid token type ws", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenWS}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(format arg) valid token type number", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenNumber}, + }, + baseTokenResolver, + &ResolverResult{ + Identifier: `strftime({:format},{:time})`, + Params: map[string]any{"format": "1", "time": "2"}, + }, + false, + }, + { + "(format arg) valid token type identifier", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenIdentifier}, + }, + baseTokenResolver, + &ResolverResult{ + Identifier: `strftime({:format},{:time})`, + Params: map[string]any{"format": "1", "time": "2"}, + }, + false, + }, + { + "(format arg) valid token type text", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenText}, + }, + baseTokenResolver, + &ResolverResult{ + Identifier: `strftime({:format},{:time})`, + Params: map[string]any{"format": "1", "time": "2"}, + }, + false, + }, + + // format + time-value + modifier args + // ----------------------------------------------------------- + { + "(modifiers arg) invalid token type function", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, // valid format + {Literal: "2", Type: fexpr.TokenText}, // valid time-value + {Literal: "3", Type: fexpr.TokenText}, // valid modifier + {Literal: "4", Type: fexpr.TokenFunction}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(modifiers arg) invalid token type ws", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, // valid format + {Literal: "2", Type: fexpr.TokenText}, // valid time-value + {Literal: "3", Type: fexpr.TokenText}, // valid modifier + {Literal: "4", Type: fexpr.TokenWS}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(modifiers arg) valid token type number", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, // valid format + {Literal: "2", Type: fexpr.TokenText}, // valid time-value + {Literal: "3", Type: fexpr.TokenText}, // valid modifier + {Literal: "4", Type: fexpr.TokenNumber}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(modifiers arg) valid token type identifier", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, // valid format + {Literal: "2", Type: fexpr.TokenText}, // valid time-value + {Literal: "3", Type: fexpr.TokenText}, // valid modifier + {Literal: "4", Type: fexpr.TokenIdentifier}, + }, + baseTokenResolver, + nil, + true, + }, + { + "(modifiers arg) valid token type text", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, // valid format + {Literal: "2", Type: fexpr.TokenText}, // valid time-value + {Literal: "3", Type: fexpr.TokenText}, // valid modifier + {Literal: "4", Type: fexpr.TokenText}, // valid modifier + }, + baseTokenResolver, + &ResolverResult{ + Identifier: `strftime({:format},{:time},{:m1},{:m2})`, + Params: map[string]any{"format": "1", "time": "2", "m1": "3", "m2": "4"}, + }, + false, + }, + + // ----------------------------------------------------------- + + { + "= 10 args limit", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenText}, + {Literal: "3", Type: fexpr.TokenText}, + {Literal: "4", Type: fexpr.TokenText}, + {Literal: "5", Type: fexpr.TokenText}, + {Literal: "6", Type: fexpr.TokenText}, + {Literal: "7", Type: fexpr.TokenText}, + {Literal: "8", Type: fexpr.TokenText}, + {Literal: "9", Type: fexpr.TokenText}, + {Literal: "10", Type: fexpr.TokenText}, + }, + baseTokenResolver, + &ResolverResult{ + Identifier: `strftime({:format},{:time},{:m1},{:m2},{:m3},{:m4},{:m5},{:m6},{:m7},{:m8})`, + Params: map[string]any{ + "format": "1", + "time": "2", + "m1": "3", + "m2": "4", + "m3": "5", + "m4": "6", + "m5": "7", + "m6": "8", + "m7": "9", + "m8": "10", + }, + }, + false, + }, + { + "> 10 args limit", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenText}, + {Literal: "3", Type: fexpr.TokenText}, + {Literal: "4", Type: fexpr.TokenText}, + {Literal: "5", Type: fexpr.TokenText}, + {Literal: "6", Type: fexpr.TokenText}, + {Literal: "7", Type: fexpr.TokenText}, + {Literal: "8", Type: fexpr.TokenText}, + {Literal: "9", Type: fexpr.TokenText}, + {Literal: "10", Type: fexpr.TokenText}, + {Literal: "11", Type: fexpr.TokenText}, + }, + baseTokenResolver, + nil, + true, + }, + { + "valid arguments but with resolver error", + []fexpr.Token{ + {Literal: "1", Type: fexpr.TokenText}, + {Literal: "2", Type: fexpr.TokenText}, + {Literal: "3", Type: fexpr.TokenText}, + }, + func(t fexpr.Token) (*ResolverResult, error) { + return nil, errors.New("test") + }, + nil, + true, + }, + } + + for _, s := range scenarios { + t.Run(s.name, func(t *testing.T) { + result, err := fn(s.resolver, s.args...) + + hasErr := err != nil + if hasErr != s.expectErr { + t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectErr, hasErr, err) + } + + testCompareResults(t, s.result, result) + }) + } +} + +func TestTokenFunctionsStrftimeExec(t *testing.T) { + t.Parallel() + + testDB, err := createTestDB() + if err != nil { + t.Fatal(err) + } + defer testDB.Close() + + fn, ok := TokenFunctions["strftime"] + if !ok { + t.Error("Expected strftime token function to be registered.") + } + + result, err := fn( + func(t fexpr.Token) (*ResolverResult, error) { + placeholder := "t" + security.PseudorandomString(5) + return &ResolverResult{Identifier: "{:" + placeholder + "}", Params: map[string]any{placeholder: t.Literal}}, nil + }, + fexpr.Token{Literal: "%Y-%m", Type: fexpr.TokenText}, + fexpr.Token{Literal: "2026-01-02 01:02:03.456Z", Type: fexpr.TokenText}, + fexpr.Token{Literal: "+1 years", Type: fexpr.TokenText}, + fexpr.Token{Literal: "+5 months", Type: fexpr.TokenText}, + ) + if err != nil { + t.Fatal(err) + } + + column := []string{} + err = testDB.NewQuery("select " + result.Identifier).Bind(result.Params).Column(&column) + if err != nil { + t.Fatal(err) + } + + if len(column) != 1 { + t.Fatalf("Expected exactly 1 column value as result, got %v", column) + } + + expected := "2027-06" + if column[0] != expected { + t.Fatalf("Expected date value %s, got %s", expected, column[0]) + } +} + // ------------------------------------------------------------------- func testCompareResults(t *testing.T, a, b *ResolverResult) { @@ -262,6 +601,10 @@ func testCompareResults(t *testing.T, a, b *ResolverResult) { t.Fatalf("Expected NoCoalesce to match, got %v vs %v", a.NoCoalesce, b.NoCoalesce) } + if len(a.Params) != len(b.Params) { + t.Fatalf("Expected equal number of params, got %v vs %v", len(a.Params), len(b.Params)) + } + // loose placeholders replacement var aResolved = a.Identifier for k, v := range a.Params {