From f0e4a5624f4371e08fb0c41510891c9f8cce6ba0 Mon Sep 17 00:00:00 2001 From: boojack Date: Mon, 15 Jun 2026 23:22:28 +0800 Subject: [PATCH] feat(filter): expand CEL filter surface with startsWith/endsWith, matches(), and all() Let users write three more CEL constructs in the filter field, each compiled to SQL across SQLite/MySQL/Postgres: - Scalar startsWith()/endsWith() on content/filename/mime_type (case-insensitive) - matches() regex: PG ~, MySQL/SQLite REGEXP (Go-backed SQLite fn), validated at compile time via cel.ValidateRegexLiterals() - all() comprehension over tags via per-element subqueries, non-empty required Also: contains() now escapes LIKE metacharacters (%, _, \); cross-dialect render tests plus behavioral tests; cel-go bumped to v0.28.1; new operators surfaced in the frontend shortcut guide. --- ...2026-06-15-cel-filter-surface-expansion.md | 887 ++++++++++++++++++ ...-15-cel-filter-surface-expansion-design.md | 257 +++++ go.mod | 2 +- go.sum | 4 +- internal/filter/README.md | 14 + internal/filter/engine_test.go | 158 ++++ internal/filter/ir.go | 26 +- internal/filter/parser.go | 60 +- internal/filter/render.go | 118 ++- internal/filter/schema.go | 13 +- store/db/sqlite/functions.go | 60 ++ store/db/sqlite/functions_test.go | 24 + store/db/sqlite/sqlite.go | 4 + store/test/attachment_filter_test.go | 36 + store/test/memo_filter_test.go | 184 ++++ web/src/pages/Shortcuts.tsx | 22 + 16 files changed, 1835 insertions(+), 34 deletions(-) create mode 100644 docs/superpowers/plans/2026-06-15-cel-filter-surface-expansion.md create mode 100644 docs/superpowers/specs/2026-06-15-cel-filter-surface-expansion-design.md create mode 100644 store/db/sqlite/functions_test.go diff --git a/docs/superpowers/plans/2026-06-15-cel-filter-surface-expansion.md b/docs/superpowers/plans/2026-06-15-cel-filter-surface-expansion.md new file mode 100644 index 000000000..177b95262 --- /dev/null +++ b/docs/superpowers/plans/2026-06-15-cel-filter-surface-expansion.md @@ -0,0 +1,887 @@ +# CEL Filter Surface Expansion — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let users write three more CEL constructs in the `filter` field — scalar `startsWith()`/`endsWith()` (case-insensitive), `matches(regex)`, and `all()` over tags (non-empty) — each compiled to SQL across SQLite/MySQL/Postgres. + +**Architecture:** The `internal/filter` engine parses CEL with `cel-go`, walks the AST into a dialect-agnostic IR (`ir.go`), and renders dialect SQL (`render.go`). cel-go never evaluates — every feature must become a SQL `WHERE` fragment. We add IR nodes + parser recognition + per-dialect rendering, and register a Go-backed `REGEXP` function for SQLite (which has no built-in one). + +**Tech Stack:** Go, `github.com/google/cel-go v0.28.0`, `modernc.org/sqlite` (pure-Go), `github.com/stretchr/testify/require`. + +**Spec:** `docs/superpowers/specs/2026-06-15-cel-filter-surface-expansion-design.md` + +--- + +## File structure + +| File | Change | Responsibility | +|------|--------|----------------| +| `internal/filter/ir.go` | Modify | Replace `ContainsCondition` with `TextMatchCondition`; add `RegexCondition`; add `ComprehensionAll` kind | +| `internal/filter/parser.go` | Modify | Recognize top-level `contains`/`startsWith`/`endsWith`/`matches`; accept `all()` comprehension | +| `internal/filter/render.go` | Modify | Render text-match (LIKE), regex, and `all()` per-element subqueries; shared `foldedLike`/`likePattern`/`escapeLikeLiteral` helpers | +| `internal/filter/schema.go` | Modify | Add `cel.ValidateRegexLiterals()` validator; enable text matching on attachment `mime_type` | +| `internal/filter/engine_test.go` | Modify | Compile-level accept/reject unit tests | +| `store/db/sqlite/functions.go` | Modify | Register a Go-backed `regexp(pattern, value)` scalar function with a compiled-pattern cache | +| `store/db/sqlite/sqlite.go` | Modify | Call `ensureRegexpRegistered()` in `NewDB` | +| `store/test/memo_filter_test.go` | Modify | Behavioral tests for the new memo filters | +| `store/test/attachment_filter_test.go` | Modify | Behavioral tests for `filename`/`mime_type` | +| `internal/filter/README.md` | Modify | Document new syntax + regex cross-dialect caveat | + +**Key design choices locked in:** +- The existing `Field.SupportsContains` flag is **reused** as the gate for *all* text-matching ops (`contains`/`startsWith`/`endsWith`/`matches`) — no rename, lower risk. We just enable it on `mime_type`. +- New scalar `startsWith`/`endsWith`/`contains` are **case-insensitive** (reuse the existing `memos_unicode_lower` / `ILIKE` machinery). `matches()` and `==` are case-sensitive. +- `all()` over a memo with zero tags does **not** match (non-empty guard). +- LIKE patterns escape `%` `_` `\`; only SQLite needs an explicit `ESCAPE '\'` clause (Postgres/MySQL default the escape char to backslash, and patterns are passed as bound parameters so no SQL-literal backslash hazard). + +**Suggested task order** lands the two cheap features (text-match refactor, scalar prefix/suffix, regex) before the heavy `all()` work, giving a natural stop point. `all()` (Task 5) is the largest piece and could be deferred to a follow-up if needed. + +--- + +### Task 1: Refactor `ContainsCondition` → `TextMatchCondition` (+ LIKE escaping) + +Foundation refactor. No new user-facing behavior except that LIKE metacharacters in `contains()` values are now treated literally. Existing tests must stay green. + +**Files:** +- Modify: `internal/filter/ir.go` (replace `ContainsCondition`) +- Modify: `internal/filter/parser.go` (`buildContainsCondition` → shared builder) +- Modify: `internal/filter/render.go` (`renderContainsCondition` → `renderTextMatch` + helpers) + +- [ ] **Step 1: Add a failing escaping test** in `internal/filter/engine_test.go` + +```go +func TestCompileContainsEscapesLikeWildcards(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + stmt, err := engine.CompileToStatement(context.Background(), `content.contains("50%_off")`, RenderOptions{Dialect: DialectSQLite}) + require.NoError(t, err) + // The % and _ in the value must be escaped so they are matched literally, + // and SQLite needs an explicit ESCAPE clause. + require.Contains(t, stmt.SQL, `ESCAPE '\'`) + require.Equal(t, []any{`%50\%\_off%`}, stmt.Args) +} +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `go test ./internal/filter/ -run TestCompileContainsEscapesLikeWildcards -v` +Expected: FAIL (current renderer emits `%50%_off%` with no `ESCAPE`). + +- [ ] **Step 3: Replace `ContainsCondition` in `internal/filter/ir.go`** + +Delete the `ContainsCondition` struct + its `isCondition()` (lines ~76-82) and add: + +```go +// TextMatchMode enumerates LIKE-based string match modes. +type TextMatchMode string + +const ( + TextMatchContains TextMatchMode = "contains" + TextMatchPrefix TextMatchMode = "prefix" + TextMatchSuffix TextMatchMode = "suffix" +) + +// TextMatchCondition models a case-insensitive LIKE match on a scalar string field +// (content.contains/startsWith/endsWith). +type TextMatchCondition struct { + Field string + Mode TextMatchMode + Value string +} + +func (*TextMatchCondition) isCondition() {} +``` + +- [ ] **Step 4: Update the parser in `internal/filter/parser.go`** + +In `buildCallCondition`, replace the `case "contains":` line with: + +```go + case "contains": + return buildTextMatchCondition(call, schema, TextMatchContains) +``` + +Delete `buildContainsCondition` (lines ~196-227) and add: + +```go +func buildTextMatchCondition(call *exprv1.Expr_Call, schema Schema, mode TextMatchMode) (Condition, error) { + if call.Target == nil { + return nil, errors.New("text match requires a target") + } + targetName, err := getIdentName(call.Target) + if err != nil { + return nil, err + } + field, ok := schema.Field(targetName) + if !ok { + return nil, errors.Errorf("unknown identifier %q", targetName) + } + if !field.SupportsContains { + return nil, errors.Errorf("identifier %q does not support text matching", targetName) + } + if len(call.Args) != 1 { + return nil, errors.New("text match expects exactly one argument") + } + value, err := getConstValue(call.Args[0]) + if err != nil { + return nil, errors.Wrap(err, "text match only supports literal arguments") + } + str, ok := value.(string) + if !ok { + return nil, errors.New("text match argument must be a string") + } + return &TextMatchCondition{Field: targetName, Mode: mode, Value: str}, nil +} +``` + +- [ ] **Step 5: Update the renderer in `internal/filter/render.go`** + +In `renderCondition`, replace `case *ContainsCondition:` / `return r.renderContainsCondition(c)` with: + +```go + case *TextMatchCondition: + return r.renderTextMatch(c) +``` + +Delete `renderContainsCondition` (lines ~449-469) and add: + +```go +func (r *renderer) renderTextMatch(cond *TextMatchCondition) (renderResult, error) { + field, ok := r.schema.Field(cond.Field) + if !ok { + return renderResult{}, errors.Errorf("unknown field %q", cond.Field) + } + column := field.columnExpr(r.dialect) + pattern := likePattern(cond.Mode, cond.Value) + return renderResult{sql: r.foldedLike(column, pattern)}, nil +} + +// foldedLike renders a case-insensitive LIKE comparison of colExpr against a +// (already metacharacter-escaped) pattern, using each dialect's case-folding. +func (r *renderer) foldedLike(colExpr, pattern string) string { + switch r.dialect { + case DialectSQLite: + // memos_unicode_lower gives Unicode-aware folding; ESCAPE '\' is required + // because SQLite has no default LIKE escape character. + return fmt.Sprintf(`memos_unicode_lower(%s) LIKE memos_unicode_lower(%s) ESCAPE '\'`, colExpr, r.addArg(pattern)) + case DialectPostgres: + // ILIKE is case-insensitive; backslash is the default escape character. + return fmt.Sprintf("%s ILIKE %s", colExpr, r.addArg(pattern)) + default: // MySQL: default collation is case-insensitive; backslash is the default escape. + return fmt.Sprintf("%s LIKE %s", colExpr, r.addArg(pattern)) + } +} + +// likePattern escapes LIKE metacharacters in value and wraps it for the mode. +func likePattern(mode TextMatchMode, value string) string { + escaped := escapeLikeLiteral(value) + switch mode { + case TextMatchPrefix: + return escaped + "%" + case TextMatchSuffix: + return "%" + escaped + default: + return "%" + escaped + "%" + } +} + +// escapeLikeLiteral escapes the LIKE metacharacters \, %, and _ so user input +// is matched literally. Backslash is the escape character on all three dialects. +func escapeLikeLiteral(s string) string { + return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s) +} +``` + +- [ ] **Step 6: Run the new test + existing suites to verify green** + +Run: `go test ./internal/filter/ -v` +Expected: PASS (including `TestCompileContainsEscapesLikeWildcards`). + +Run: `go test ./store/test/ -run TestMemoFilterContent -v` +Expected: PASS (existing `contains` behavioral tests, including special-characters/unicode, still pass). + +- [ ] **Step 7: Commit** + +```bash +git add internal/filter/ir.go internal/filter/parser.go internal/filter/render.go internal/filter/engine_test.go +git commit -m "refactor(filter): unify string matching into TextMatchCondition with LIKE escaping + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 2: Scalar `startsWith()` / `endsWith()` + +Wire the new prefix/suffix modes through the parser and enable text matching on attachment `mime_type`. Rendering already exists from Task 1. + +**Files:** +- Modify: `internal/filter/parser.go` (add `startsWith`/`endsWith` cases) +- Modify: `internal/filter/schema.go` (enable `SupportsContains` on `mime_type`) +- Modify: `store/test/memo_filter_test.go`, `store/test/attachment_filter_test.go` (behavioral tests) +- Modify: `internal/filter/engine_test.go` (reject on unsupported field) + +- [ ] **Step 1: Add failing behavioral tests** in `store/test/memo_filter_test.go` + +```go +func TestMemoFilterContentStartsWith(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-todo", tc.User.ID).Content("TODO: buy milk")) + tc.CreateMemo(NewMemoBuilder("memo-done", tc.User.ID).Content("Done with milk")) + + // Prefix match, case-insensitive (consistent with contains()). + memos := tc.ListWithFilter(`content.startsWith("todo")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-todo", memos[0].UID) + + memos = tc.ListWithFilter(`content.startsWith("nope")`) + require.Len(t, memos, 0) +} + +func TestMemoFilterContentEndsWith(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-md", tc.User.ID).Content("notes.md")) + tc.CreateMemo(NewMemoBuilder("memo-txt", tc.User.ID).Content("notes.txt")) + + memos := tc.ListWithFilter(`content.endsWith(".md")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-md", memos[0].UID) +} +``` + +And in `store/test/attachment_filter_test.go`: + +```go +func TestAttachmentFilterFilenameStartsWith(t *testing.T) { + t.Parallel() + tc := NewAttachmentFilterTestContextWithUser(t) + defer tc.Close() + + tc.CreateAttachment(NewAttachmentBuilder(tc.CreatorID).Filename("invoice-2026.pdf").MimeType("application/pdf")) + tc.CreateAttachment(NewAttachmentBuilder(tc.CreatorID).Filename("photo.png").MimeType("image/png")) + + got := tc.ListWithFilter(`filename.startsWith("invoice")`) + require.Len(t, got, 1) + require.Equal(t, "invoice-2026.pdf", got[0].Filename) + + // mime_type prefix matching (newly enabled). + got = tc.ListWithFilter(`mime_type.startsWith("image/")`) + require.Len(t, got, 1) + require.Equal(t, "photo.png", got[0].Filename) +} +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `go test ./store/test/ -run 'TestMemoFilterContentStartsWith|TestMemoFilterContentEndsWith|TestAttachmentFilterFilenameStartsWith' -v` +Expected: FAIL — `startsWith` hits `buildCallCondition`'s default branch ("unsupported call expression"), and `mime_type` is not yet text-matchable. + +- [ ] **Step 3: Add parser cases** in `internal/filter/parser.go` `buildCallCondition` + +Immediately after the `case "contains":` line, add: + +```go + case "startsWith": + return buildTextMatchCondition(call, schema, TextMatchPrefix) + case "endsWith": + return buildTextMatchCondition(call, schema, TextMatchSuffix) +``` + +- [ ] **Step 4: Enable text matching on `mime_type`** in `internal/filter/schema.go` + +In `NewAttachmentSchema`, add `SupportsContains: true` to the `mime_type` field entry: + +```go + "mime_type": { + Name: "mime_type", + Kind: FieldKindScalar, + Type: FieldTypeString, + Column: Column{Table: "attachment", Name: "type"}, + SupportsContains: true, + Expressions: map[DialectName]string{}, + }, +``` + +- [ ] **Step 5: Add a compile-reject unit test** in `internal/filter/engine_test.go` + +```go +func TestCompileRejectsStartsWithOnUnsupportedField(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + _, err = engine.Compile(context.Background(), `visibility.startsWith("P")`) + require.Error(t, err) + require.Contains(t, err.Error(), "does not support text matching") +} +``` + +- [ ] **Step 6: Run tests to verify green** + +Run: `go test ./internal/filter/ ./store/test/ -run 'StartsWith|EndsWith|TextMatch' -v` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add internal/filter/parser.go internal/filter/schema.go internal/filter/engine_test.go store/test/memo_filter_test.go store/test/attachment_filter_test.go +git commit -m "feat(filter): support startsWith()/endsWith() on scalar string fields + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 3: Register a SQLite `REGEXP` function + +`modernc.org/sqlite` has no built-in `REGEXP`. SQLite desugars `X REGEXP Y` to `regexp(Y, X)`, so register a 2-arg `regexp(pattern, value)` scalar function backed by Go's `regexp`, mirroring `ensureUnicodeLowerRegistered`. + +**Files:** +- Modify: `store/db/sqlite/functions.go` +- Modify: `store/db/sqlite/sqlite.go` +- Test: `store/db/sqlite/functions_test.go` (create) + +- [ ] **Step 1: Write a failing test** — create `store/db/sqlite/functions_test.go` + +```go +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRegexpFunctionMatches(t *testing.T) { + require.NoError(t, ensureRegexpRegistered()) + + re, err := compileRegexp(`^v\d+$`) + require.NoError(t, err) + require.True(t, re.MatchString("v12")) + require.False(t, re.MatchString("version")) + + // Caching returns the same compiled instance. + re2, err := compileRegexp(`^v\d+$`) + require.NoError(t, err) + require.Same(t, re, re2) + + _, err = compileRegexp(`(`) + require.Error(t, err) +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `go test ./store/db/sqlite/ -run TestRegexpFunctionMatches -v` +Expected: FAIL — `ensureRegexpRegistered`/`compileRegexp` undefined. + +- [ ] **Step 3: Implement in `store/db/sqlite/functions.go`** + +Add `"errors"` and `"regexp"` to the imports, then append: + +```go +var ( + registerRegexpOnce sync.Once + registerRegexpErr error + // regexpCache memoizes compiled patterns; keys are pattern strings. + regexpCache sync.Map +) + +// ensureRegexpRegistered registers a Go-backed `regexp(pattern, value)` scalar +// function so SQLite's `value REGEXP pattern` operator works (modernc.org/sqlite +// has no built-in implementation). Patterns use Go's RE2 syntax. Registered once +// globally; safe to call multiple times. +func ensureRegexpRegistered() error { + registerRegexpOnce.Do(func() { + registerRegexpErr = msqlite.RegisterScalarFunction("regexp", 2, func(_ *msqlite.FunctionContext, args []driver.Value) (driver.Value, error) { + if len(args) != 2 || args[0] == nil || args[1] == nil { + return int64(0), nil + } + pattern, ok := args[0].(string) + if !ok { + return nil, errors.New("regexp pattern must be a string") + } + var value string + switch v := args[1].(type) { + case string: + value = v + case []byte: + value = string(v) + default: + return int64(0), nil + } + re, err := compileRegexp(pattern) + if err != nil { + return nil, err + } + if re.MatchString(value) { + return int64(1), nil + } + return int64(0), nil + }) + }) + return registerRegexpErr +} + +// compileRegexp compiles and caches a RE2 pattern. +func compileRegexp(pattern string) (*regexp.Regexp, error) { + if cached, ok := regexpCache.Load(pattern); ok { + return cached.(*regexp.Regexp), nil + } + re, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + regexpCache.Store(pattern, re) + return re, nil +} +``` + +- [ ] **Step 4: Wire into `NewDB`** in `store/db/sqlite/sqlite.go` + +Right after the `ensureUnicodeLowerRegistered()` block, add: + +```go + if err := ensureRegexpRegistered(); err != nil { + return nil, errors.Wrap(err, "failed to register sqlite regexp function") + } +``` + +- [ ] **Step 5: Run to verify green** + +Run: `go test ./store/db/sqlite/ -run TestRegexpFunctionMatches -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add store/db/sqlite/functions.go store/db/sqlite/sqlite.go store/db/sqlite/functions_test.go +git commit -m "feat(sqlite): register Go-backed REGEXP function + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 4: `matches(regex)` on string fields + +Add the IR node, parser recognition, per-dialect rendering, and the compile-time regex validator. + +**Files:** +- Modify: `internal/filter/ir.go` (add `RegexCondition`) +- Modify: `internal/filter/parser.go` (`matches` case + builder) +- Modify: `internal/filter/render.go` (`renderRegex`) +- Modify: `internal/filter/schema.go` (add `cel.ValidateRegexLiterals()` to both schemas) +- Modify: `internal/filter/engine_test.go`, `store/test/memo_filter_test.go` + +- [ ] **Step 1: Add failing tests** — compile-level in `internal/filter/engine_test.go`: + +```go +func TestCompileRejectsMalformedRegex(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + _, err = engine.Compile(context.Background(), `content.matches("(")`) + require.Error(t, err) +} + +func TestCompileMatchesRendersRegexOperator(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + stmt, err := engine.CompileToStatement(context.Background(), `content.matches("v[0-9]+")`, RenderOptions{Dialect: DialectPostgres}) + require.NoError(t, err) + require.Contains(t, stmt.SQL, "~") + require.Equal(t, []any{"v[0-9]+"}, stmt.Args) +} +``` + +And behavioral in `store/test/memo_filter_test.go` (runs against SQLite by default, exercising the registered `REGEXP` function): + +```go +func TestMemoFilterContentMatches(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-v1", tc.User.ID).Content("release v12 shipped")) + tc.CreateMemo(NewMemoBuilder("memo-plain", tc.User.ID).Content("no version here")) + + memos := tc.ListWithFilter(`content.matches("v[0-9]+")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-v1", memos[0].UID) + + memos = tc.ListWithFilter(`content.matches("^xyz")`) + require.Len(t, memos, 0) +} +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `go test ./internal/filter/ -run 'Malformed|MatchesRenders' -v && go test ./store/test/ -run TestMemoFilterContentMatches -v` +Expected: FAIL — `matches` is unhandled and no regex validator is configured. + +- [ ] **Step 3: Add the IR node** in `internal/filter/ir.go` + +```go +// RegexCondition models field.matches("pattern") on a string field. +type RegexCondition struct { + Field string + Pattern string +} + +func (*RegexCondition) isCondition() {} +``` + +- [ ] **Step 4: Add parser support** in `internal/filter/parser.go` + +In `buildCallCondition`, after the `case "endsWith":` block, add: + +```go + case "matches": + return buildMatchesCondition(call, schema) +``` + +Then add the builder: + +```go +func buildMatchesCondition(call *exprv1.Expr_Call, schema Schema) (Condition, error) { + if call.Target == nil { + return nil, errors.New("matches requires a target") + } + targetName, err := getIdentName(call.Target) + if err != nil { + return nil, err + } + field, ok := schema.Field(targetName) + if !ok { + return nil, errors.Errorf("unknown identifier %q", targetName) + } + if !field.SupportsContains { + return nil, errors.Errorf("identifier %q does not support matches()", targetName) + } + if len(call.Args) != 1 { + return nil, errors.New("matches expects exactly one argument") + } + value, err := getConstValue(call.Args[0]) + if err != nil { + return nil, errors.Wrap(err, "matches only supports literal arguments") + } + pattern, ok := value.(string) + if !ok { + return nil, errors.New("matches argument must be a string") + } + return &RegexCondition{Field: targetName, Pattern: pattern}, nil +} +``` + +- [ ] **Step 5: Add the renderer** in `internal/filter/render.go` + +In `renderCondition`, after the `case *TextMatchCondition:` arm, add: + +```go + case *RegexCondition: + return r.renderRegex(c) +``` + +Then add: + +```go +func (r *renderer) renderRegex(cond *RegexCondition) (renderResult, error) { + field, ok := r.schema.Field(cond.Field) + if !ok { + return renderResult{}, errors.Errorf("unknown field %q", cond.Field) + } + column := field.columnExpr(r.dialect) + switch r.dialect { + case DialectPostgres: + // POSIX regex match operator. + return renderResult{sql: fmt.Sprintf("%s ~ %s", column, r.addArg(cond.Pattern))}, nil + case DialectMySQL, DialectSQLite: + // MySQL has a native REGEXP operator; SQLite uses the registered regexp() function. + return renderResult{sql: fmt.Sprintf("%s REGEXP %s", column, r.addArg(cond.Pattern))}, nil + default: + return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect) + } +} +``` + +- [ ] **Step 6: Add the regex validator** in `internal/filter/schema.go` + +Add the `cel` import line already present. In **both** `NewSchema` and `NewAttachmentSchema`, append the validator to the `envOptions` slice (e.g. after `nowFunction`): + +```go + cel.ASTValidators(cel.ValidateRegexLiterals()), +``` + +- [ ] **Step 7: Run tests to verify green** + +Run: `go test ./internal/filter/ -run 'Malformed|MatchesRenders' -v && go test ./store/test/ -run TestMemoFilterContentMatches -v` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add internal/filter/ir.go internal/filter/parser.go internal/filter/render.go internal/filter/schema.go internal/filter/engine_test.go store/test/memo_filter_test.go +git commit -m "feat(filter): support matches() regex on string fields + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 5: `all()` comprehension on tags (non-empty) + +The heaviest task. `exists()` matches against the *serialized* JSON array and cannot express "every element matches", so `all()` needs real per-element iteration via `json_each` / `jsonb_array_elements_text` / `JSON_TABLE`, plus a non-empty guard. + +**Files:** +- Modify: `internal/filter/ir.go` (add `ComprehensionAll`) +- Modify: `internal/filter/parser.go` (accept `all()` in `detectComprehensionKind`) +- Modify: `internal/filter/render.go` (`renderTagAll` + element predicate SQL; branch in `renderListComprehension`) +- Modify: `store/test/memo_filter_test.go` + +- [ ] **Step 1: Add failing behavioral tests** in `store/test/memo_filter_test.go` + +```go +func TestMemoFilterTagsAll(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-all-work", tc.User.ID).Content("all work").Tags("work/a", "work/b")) + tc.CreateMemo(NewMemoBuilder("memo-mixed", tc.User.ID).Content("mixed").Tags("work/a", "home")) + tc.CreateMemo(NewMemoBuilder("memo-untagged", tc.User.ID).Content("untagged")) + + // Every tag starts with "work/": only the all-work memo qualifies. + memos := tc.ListWithFilter(`tags.all(t, t.startsWith("work/"))`) + require.Len(t, memos, 1) + require.Equal(t, "memo-all-work", memos[0].UID) + + // Untagged memos must NOT match (non-empty guard, decision B). + require.NotContains(t, uids(memos), "memo-untagged") +} + +func TestMemoFilterTagsAllEquals(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-only-x", tc.User.ID).Content("only x").Tags("x", "x")) + tc.CreateMemo(NewMemoBuilder("memo-x-and-y", tc.User.ID).Content("x and y").Tags("x", "y")) + + memos := tc.ListWithFilter(`tags.all(t, t == "x")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-only-x", memos[0].UID) +} +``` + +Add this helper near the top of `store/test/memo_filter_test.go` (after the imports) if not already present: + +```go +func uids(memos []*store.Memo) []string { + out := make([]string, 0, len(memos)) + for _, m := range memos { + out = append(out, m.UID) + } + return out +} +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `go test ./store/test/ -run 'TestMemoFilterTagsAll' -v` +Expected: FAIL — `detectComprehensionKind` returns "all() comprehension is not supported". + +- [ ] **Step 3: Add the IR kind** in `internal/filter/ir.go` + +In the `ComprehensionKind` const block, add `ComprehensionAll`: + +```go +const ( + ComprehensionExists ComprehensionKind = "exists" + ComprehensionAll ComprehensionKind = "all" +) +``` + +- [ ] **Step 4: Accept `all()` in the parser** in `internal/filter/parser.go` + +In `detectComprehensionKind`, replace the `all()` rejection block: + +```go + // all() starts with true and uses AND (&&) - not supported + if accuInit.GetBoolValue() { + if step := comp.LoopStep.GetCallExpr(); step != nil && step.Function == "_&&_" { + return "", errors.New("all() comprehension is not supported; use exists() instead") + } + } +``` + +with: + +```go + // all() starts with true and uses AND (&&) in the loop step. + if accuInit.GetBoolValue() { + if step := comp.LoopStep.GetCallExpr(); step != nil && step.Function == "_&&_" { + return ComprehensionAll, nil + } + } +``` + +- [ ] **Step 5: Branch and render in `internal/filter/render.go`** + +At the top of `renderListComprehension`, right after the `field.Kind != FieldKindJSONList` guard, add: + +```go + if cond.Kind == ComprehensionAll { + return r.renderTagAll(field, cond.Predicate) + } +``` + +Then add the new render path + element-predicate helper: + +```go +// renderTagAll renders tags.all(t, ): the array is non-empty AND no element +// fails the predicate. Element predicates use plain CEL semantics (case-insensitive +// for startsWith/endsWith/contains, case-sensitive for ==), evaluated per element. +func (r *renderer) renderTagAll(field Field, pred PredicateExpr) (renderResult, error) { + arrayExpr := jsonArrayExpr(r.dialect, field) + elemCond, err := r.elementPredicateSQL(pred) + if err != nil { + return renderResult{}, err + } + switch r.dialect { + case DialectSQLite: + nonEmpty := fmt.Sprintf("%s IS NOT NULL AND %s != '[]'", arrayExpr, arrayExpr) + sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM json_each(%s) WHERE NOT (%s))", arrayExpr, elemCond) + return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil + case DialectMySQL: + nonEmpty := fmt.Sprintf("%s IS NOT NULL AND JSON_LENGTH(%s) > 0", arrayExpr, arrayExpr) + sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM JSON_TABLE(%s, '$[*]' COLUMNS (value VARCHAR(512) PATH '$')) AS elem WHERE NOT (%s))", arrayExpr, elemCond) + return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil + case DialectPostgres: + nonEmpty := fmt.Sprintf("%s IS NOT NULL AND jsonb_array_length(%s) > 0", arrayExpr, arrayExpr) + sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(%s) AS elem(value) WHERE NOT (%s))", arrayExpr, elemCond) + return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil + default: + return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect) + } +} + +// elementPredicateSQL builds the per-element SQL condition for an all() predicate. +// The iterated element is exposed as the unqualified column `value` on all dialects +// (json_each.value / JSON_TABLE column / elem(value)). +func (r *renderer) elementPredicateSQL(pred PredicateExpr) (string, error) { + switch p := pred.(type) { + case *EqualsPredicate: + return fmt.Sprintf("value = %s", r.addArg(p.Value)), nil + case *StartsWithPredicate: + return r.foldedLike("value", likePattern(TextMatchPrefix, p.Prefix)), nil + case *EndsWithPredicate: + return r.foldedLike("value", likePattern(TextMatchSuffix, p.Suffix)), nil + case *ContainsPredicate: + return r.foldedLike("value", likePattern(TextMatchContains, p.Substring)), nil + default: + return "", errors.Errorf("unsupported predicate %T in all()", pred) + } +} +``` + +> Note: `foldedLike`, `likePattern`, and `escapeLikeLiteral` were added in Task 1; reuse them as-is. + +- [ ] **Step 6: Run tests to verify green** + +Run: `go test ./store/test/ -run 'TestMemoFilterTagsAll' -v` +Expected: PASS. + +Run: `go test ./store/test/ -run 'TestMemoFilterTagsExists' -v` +Expected: PASS (exists() rendering untouched). + +- [ ] **Step 7: Commit** + +```bash +git add internal/filter/ir.go internal/filter/parser.go internal/filter/render.go store/test/memo_filter_test.go +git commit -m "feat(filter): support all() comprehension over tags + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 6: Docs + full verification + +**Files:** +- Modify: `internal/filter/README.md` + +- [ ] **Step 1: Document the new syntax** — append to the "SQL Generation Notes" section of `internal/filter/README.md`: + +```markdown +- **String Matching** — `content.contains(x)`, `content.startsWith(x)`, and + `content.endsWith(x)` render as case-insensitive `LIKE`/`ILIKE` with LIKE + metacharacters (`%`, `_`, `\`) escaped. Available on scalar string fields whose + schema sets `SupportsContains` (memo `content`; attachment `filename`, + `mime_type`). +- **Regex** — `field.matches("pattern")` renders to `~` (Postgres) or `REGEXP` + (MySQL/SQLite). SQLite uses a Go-backed `regexp` function registered in + `store/db/sqlite/functions.go`. Patterns are validated at compile time against + Go's RE2 via `cel.ValidateRegexLiterals()`. **Caveat:** regex *syntax* differs + per engine (Go RE2 on SQLite, POSIX ERE on Postgres, ICU on MySQL 8.0+), so + engine-specific patterns may not be portable. +- **Tag `all()`** — `tags.all(t, )` matches only non-empty tag sets where + every element satisfies the predicate, via per-element iteration + (`json_each` / `jsonb_array_elements_text` / `JSON_TABLE`). +``` + +- [ ] **Step 2: Run the full engine + store suite (SQLite)** + +Run: `go test ./internal/filter/... ./store/...` +Expected: PASS. + +- [ ] **Step 3: Vet and lint** + +Run: `go vet ./internal/filter/... ./store/db/sqlite/...` +Expected: no output. + +Run: `golangci-lint run internal/filter/... store/db/sqlite/...` (if available; skip if the binary is absent). +Expected: no findings. + +- [ ] **Step 4: Cross-dialect verification (if Docker/CI DSNs available)** + +Run MySQL and Postgres suites to confirm the `all()` subqueries and regex operators render correctly: + +```bash +DRIVER=mysql go test ./store/test/ -run 'TagsAll|Matches|StartsWith|EndsWith' +DRIVER=postgres go test ./store/test/ -run 'TagsAll|Matches|StartsWith|EndsWith' +``` + +Expected: PASS. (These require the project's standard test DB setup; if unavailable locally, rely on CI which runs all three drivers.) + +- [ ] **Step 5: Commit** + +```bash +git add internal/filter/README.md +git commit -m "docs(filter): document string matching, regex, and tag all() support + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Self-review notes + +- **Spec coverage:** ① scalar `startsWith`/`endsWith` → Task 2; ② `all()` non-empty → Task 5; ④ `matches()` + SQLite REGEXP fn + `ValidateRegexLiterals` → Tasks 3-4; the LIKE-escaping fix → Task 1; docs/caveat → Task 6. `lowerAscii`/`upperAscii` correctly omitted (dropped in spec). Hardening/native-AST migration correctly deferred to the follow-up spec. +- **Type consistency:** `TextMatchCondition`/`TextMatchMode`/`likePattern`/`foldedLike`/`escapeLikeLiteral` (Task 1) are reused by Tasks 2 and 5; `RegexCondition`/`renderRegex` (Task 4) and `ensureRegexpRegistered`/`compileRegexp` (Task 3) names match across their call sites; `ComprehensionAll` (Task 5) matches its parser and render references. +- **Element reference:** the unqualified `value` column is produced by `json_each` (SQLite), the `JSON_TABLE(... COLUMNS (value ...))` (MySQL), and `elem(value)` (Postgres), so `elementPredicateSQL` is dialect-agnostic. +``` diff --git a/docs/superpowers/specs/2026-06-15-cel-filter-surface-expansion-design.md b/docs/superpowers/specs/2026-06-15-cel-filter-surface-expansion-design.md new file mode 100644 index 000000000..001ae0855 --- /dev/null +++ b/docs/superpowers/specs/2026-06-15-cel-filter-surface-expansion-design.md @@ -0,0 +1,257 @@ +# Design: Expand the CEL filter surface + +- **Date:** 2026-06-15 +- **Status:** Approved (design); ready for implementation planning +- **Area:** `internal/filter` (memo & attachment filter engine) +- **Follow-up spec:** CEL engine hardening + native-AST migration (separate, sequenced after this) + +## Summary + +memos lets API clients pass a CEL expression in the `filter` field of list +requests. The `internal/filter` engine uses `cel-go` purely as a **parse + +type-check frontend**, then walks the AST and translates it into a SQL `WHERE` +fragment for the active dialect (SQLite / MySQL / Postgres). cel-go never +evaluates anything. + +This spec adds three new CEL constructs that users can write, each with a SQL +translation across all three dialects: + +1. `startsWith()` / `endsWith()` on scalar string fields (case-insensitive). +2. `all()` comprehension on tag lists (matches only non-empty tag sets). +3. `matches(regex)` on string fields. + +## Goals + +- Expose the three constructs above through the existing + parse → IR → render pipeline. +- Keep parity across SQLite, MySQL, and Postgres, with golden tests for each. +- Preserve the engine's invariant: only schema-declared fields and explicitly + supported operations are accepted; everything else is rejected with a clear + error. + +## Non-goals + +- **Value-producing CEL features with no SQL form** are explicitly out of scope: + optional types (`?.`, `optional.of`), `map()` / `filter()` transforms, the + math extension, string-manipulation extensions (`replace`, `split`, + `substring`, `format`), and two-variable comprehensions. There is nothing to + push into a `WHERE` clause for these. +- **`lowerAscii()` / `upperAscii()`** — dropped. `contains()` is already + case-insensitive on all dialects, and the new `startsWith`/`endsWith` are + case-insensitive too (see decisions), so explicit case-folding adds little. + Revisit only if users ask. +- **Parser hardening and the native-AST proto migration** are a separate + follow-up spec. The one exception that rides along here is + `cel.ValidateRegexLiterals()`, which feature ③ requires for safety. + +## Background: how the engine works today + +Pipeline (see `internal/filter/README.md`): + +1. **Parse** — `env.Compile(filter)` parses and type-checks against the + memo/attachment environment declared in `schema.go`; the AST is converted via + `cel.AstToParsedExpr()`. +2. **Normalize** — `parser.go` walks the CEL `Expr` and builds a + dialect-agnostic IR (`ir.go`): logical ops, comparisons, `IN`, `contains()`, + and `exists()` comprehensions over tag lists. +3. **Render** — `render.go` walks the IR and emits dialect-specific SQL plus + placeholder args. + +Two existing facts that shaped this design: + +- **`contains()` is already case-insensitive** on all three dialects + (`render.go` `renderContainsCondition`): SQLite uses the custom + `memos_unicode_lower` function, Postgres uses `ILIKE`, MySQL relies on its + default case-insensitive collation. +- **Custom SQLite scalar functions are already registered** + (`store/db/sqlite/functions.go`, `ensureUnicodeLowerRegistered` via + `modernc.org/sqlite`'s `RegisterScalarFunction`, invoked from + `store/db/sqlite/sqlite.go`). The new `REGEXP` function follows this exact + pattern. + +cel-go version: `v0.28.0` (latest is `v0.28.1`, a patch with nothing relevant to +memos). No version bump is required for this work. + +## Resolved decisions + +| # | Decision | Choice | +|---|----------|--------| +| A | Case-sensitivity of new scalar `startsWith`/`endsWith` | **Case-insensitive**, consistent with existing `contains()`. `==` stays case-sensitive (exact match). | +| B | `all()` over a memo with zero tags | **Require non-empty**: an untagged memo does NOT match an `all()` filter. (Diverges from strict CEL vacuous-truth, but matches search-box intuition.) | +| C | Keep `lowerAscii()` / `upperAscii()`? | **Drop** from this spec. | + +## Detailed design + +### ① `startsWith()` / `endsWith()` on scalar string fields + +**Surface.** Allow `field.startsWith("x")` and `field.endsWith("x")` as +top-level boolean calls for scalar string fields. Today these functions are only +recognized *inside* tag comprehensions (`parser.go` `extractPredicate`). + +Applicable fields: memo `content`; attachment `filename`, `mime_type`. +`creator` is intentionally **excluded**: it is an identity field with `==`/`!=` +semantics whose column is wrapped as `'users/' || username`, so prefix/suffix +matching there would match against the `users/` prefix and surprise users. + +**Schema.** Generalize the per-field text-matching capability. Today `Field` has +`SupportsContains bool`. Replace/extend with a capability that also covers +prefix/suffix matching (e.g. a `SupportsTextMatch bool`, or reuse +`SupportsContains` to gate all three LIKE-based ops). Fields that already set +`SupportsContains: true` gain prefix/suffix support. + +**Parser.** In `buildCallCondition`, recognize `startsWith` / `endsWith` calls +whose target is a scalar string field and whose single argument is a string +literal. Reject non-literal arguments and fields without the capability. + +**IR.** Generalize `ContainsCondition` into a single node: + +```go +type TextMatchMode string +const ( + TextMatchContains TextMatchMode = "contains" + TextMatchPrefix TextMatchMode = "prefix" + TextMatchSuffix TextMatchMode = "suffix" +) + +type TextMatchCondition struct { + Field string + Mode TextMatchMode + Value string +} +``` + +`contains()` migrates to `TextMatchCondition{Mode: TextMatchContains}`. + +**Render.** Build a `LIKE` pattern from the (escaped) literal: + +- prefix → `value%` +- suffix → `%value` +- contains → `%value%` + +Reuse the existing case-insensitive rendering already used by `contains()`: +SQLite `memos_unicode_lower(col) LIKE memos_unicode_lower(?)`, Postgres +`col ILIKE $n`, MySQL `col LIKE ?`. + +**LIKE-escaping fix.** The current `contains()` renderer interpolates the raw +value into the pattern without escaping `%`, `_`, or `\`. This means a search +for `50%` behaves as a wildcard. The new shared path will escape these +metacharacters (and emit `ESCAPE '\'` where required by the dialect). This +closes a small latent wildcard-injection inconsistency and applies uniformly to +contains/prefix/suffix. + +### ② `all()` comprehension on tag lists + +**Surface.** Allow `tags.all(t, )` where `` is one of the predicates +already supported for `exists()`: `t == "x"`, `t.startsWith("x")`, +`t.endsWith("x")`, `t.contains("x")`. + +**Parser.** `detectComprehensionKind` currently accepts only `exists()` and +explicitly rejects `all()`. Add a `ComprehensionAll` kind (accumulator inits to +`true`, loop step uses `_&&_`). Reuse the existing predicate extraction. + +**IR.** Add `ComprehensionAll` to the `ComprehensionKind` enum; the existing +`ListComprehensionCondition` already carries `Kind`. + +**Render — proper per-element semantics.** The existing `exists()` +implementation matches the *serialized* JSON array text with `LIKE`, which works +for "at least one element matches a substring" but **cannot** express "every +element matches." `all()` therefore needs real per-element iteration. Decision B +(require non-empty) means: array is non-empty **AND** no element fails the +predicate. + +- **SQLite:** + ```sql + ( IS NOT NULL AND != '[]' + AND NOT EXISTS (SELECT 1 FROM json_each() + WHERE NOT ())) + ``` +- **Postgres:** + ```sql + ( IS NOT NULL AND jsonb_array_length() > 0 + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text() AS e(value) + WHERE NOT ())) + ``` +- **MySQL:** + ```sql + ( IS NOT NULL AND JSON_LENGTH() > 0 + AND NOT EXISTS (SELECT 1 FROM JSON_TABLE(, '$[*]' + COLUMNS (value VARCHAR(512) PATH '$')) AS j + WHERE NOT ())) + ``` + +The per-element predicate reuses LIKE/`=` against the element `value` +(case-insensitive for `startsWith`/`endsWith`/`contains`, consistent with ①). +Hierarchical-tag prefix behavior should match the existing `exists()` rendering +(a prefix matches the exact tag or a `tag/...` child). + +> Note: this introduces correlated subqueries against the same `memo.payload` +> column the outer query already reads; confirm the generated SQL composes with +> the surrounding `WHERE` and placeholder offsets in `helpers.AppendConditions`. + +### ④ `matches(regex)` on string fields + +**Surface.** Allow `field.matches("pattern")` for the same free-text fields as ① +(`content`, `filename`, `mime_type`; `creator` excluded), literal pattern only. + +**Env / validation.** Add `cel.ValidateRegexLiterals()` to the env options in +`schema.go` so malformed patterns fail at compile time with a clear message +(validated against Go's RE2). + +**Parser / IR.** Recognize `matches` calls; add: + +```go +type RegexCondition struct { + Field string + Pattern string +} +``` + +Reject non-literal patterns and fields without text-match capability. + +**Render.** + +- **Postgres:** `col ~ $n` +- **MySQL:** `col REGEXP ?` +- **SQLite:** `col REGEXP ?`. SQLite desugars `X REGEXP Y` to the function call + `regexp(Y, X)`, so register a 2-arg scalar function named `regexp(pattern, + value)` returning 1/0, backed by Go's `regexp` package, following the + `ensureUnicodeLowerRegistered` pattern in `store/db/sqlite/functions.go`. + Compile patterns lazily with a small cache (or rely on RE2 compile per call; + decide during implementation based on measured cost). + +**Documented caveats** (engine differences are inherent, not bugs): + +- Regex *syntax* differs per engine: SQLite uses Go RE2; Postgres uses POSIX + ERE; MySQL 8.0+ uses ICU. Portable patterns work everywhere; engine-specific + constructs may not. Document this in `internal/filter/README.md`. +- ReDoS risk is low: RE2 (SQLite path) is linear-time; Postgres/MySQL POSIX + engines do not catastrophically backtrack. `ValidateRegexLiterals()` rejects + patterns that don't compile under RE2 as a first-line guard. + +## Testing strategy + +For each feature, add golden tests in +`store/db/{sqlite,mysql,postgres}/memo_filter_test.go` (and the attachment +filter tests where applicable): + +- **Happy path:** assert the exact SQL fragment and args per dialect. +- **Error paths:** non-literal argument, unsupported field, malformed regex, + unsupported predicate inside `all()`. +- **`all()` empty-set:** confirm an untagged memo does not match (decision B). +- **LIKE escaping:** confirm `%`, `_`, `\` in `contains`/`startsWith`/`endsWith` + values are treated literally. + +Run `go test ./...` (engine unit tests plus all three dialect suites). The +`contains()` → `TextMatchCondition` refactor must keep existing golden outputs +unchanged except for the intentional escaping fix. + +## Rollout / sequencing + +This is the first of two specs. The second (already agreed) covers engine +hardening: tightened parser limits (`ParserExpressionSizeLimit`, +`ParserRecursionLimit`, `ParserErrorRecoveryLimit`), the +`ValidateComprehensionNestingLimit` / `ValidateHomogeneousAggregateLiterals` +validators, and migrating `parser.go` off the deprecated +`genproto/.../expr/v1alpha1` proto to the native `common/ast` API. Building this +surface-expansion spec first is acceptable; the hardening migration is a pure +refactor that the golden tests written here will help protect. diff --git a/go.mod b/go.mod index ef884125d..b4b434f2b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.15 github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0 github.com/go-sql-driver/mysql v1.9.3 - github.com/google/cel-go v0.28.0 + github.com/google/cel-go v0.28.1 github.com/google/jsonschema-go v0.4.3 github.com/google/uuid v1.6.0 github.com/gorilla/feeds v1.2.0 diff --git a/go.sum b/go.sum index 46cb0a9ff..00cbc63c2 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.28.0 h1:KjSWstCpz/MN5t4a8gnGJNIYUsJRpdi/r97xWDphIQc= -github.com/google/cel-go v0.28.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= +github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= diff --git a/internal/filter/README.md b/internal/filter/README.md index ac1aec4b6..32b00d920 100644 --- a/internal/filter/README.md +++ b/internal/filter/README.md @@ -53,6 +53,20 @@ stmt, _ := engine.CompileToStatement(ctx, `has_task_list && visibility == "PUBLI Postgres uses `@>`. - **Boolean Flags** — Fields such as `has_task_list` render as `IS TRUE` equality checks, or comparisons against `CAST('true' AS JSON)` depending on the dialect. +- **String Matching** — `content.contains(x)`, `content.startsWith(x)`, and + `content.endsWith(x)` render as case-insensitive `LIKE`/`ILIKE` with LIKE + metacharacters (`%`, `_`, `\`) escaped. Available on scalar string fields whose + schema sets `SupportsContains` (memo `content`; attachment `filename`, + `mime_type`). +- **Regex** — `field.matches("pattern")` renders to `~` (Postgres) or `REGEXP` + (MySQL/SQLite). SQLite uses a Go-backed `regexp` function registered in + `store/db/sqlite/functions.go`. Patterns are validated at compile time against + Go's RE2 via `cel.ValidateRegexLiterals()`. **Caveat:** regex *syntax* differs + per engine (Go RE2 on SQLite, POSIX ERE on Postgres, ICU on MySQL 8.0+), so + engine-specific patterns may not be portable. +- **Tag `all()`** — `tags.all(t, )` matches only non-empty tag sets where + every element satisfies the predicate, via per-element iteration + (`json_each` / `jsonb_array_elements_text` / `JSON_TABLE`). ## Typical Integration diff --git a/internal/filter/engine_test.go b/internal/filter/engine_test.go index f9e72c224..213b30ceb 100644 --- a/internal/filter/engine_test.go +++ b/internal/filter/engine_test.go @@ -37,3 +37,161 @@ func TestCompileRejectsNonBooleanTopLevelConstant(t *testing.T) { _, err = engine.Compile(context.Background(), `1`) require.EqualError(t, err, "filter must evaluate to a boolean value") } + +func TestCompileRejectsMalformedRegex(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + _, err = engine.Compile(context.Background(), `content.matches("(")`) + require.Error(t, err) +} + +func TestCompileMatchesRendersRegexOperator(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + stmt, err := engine.CompileToStatement(context.Background(), `content.matches("v[0-9]+")`, RenderOptions{Dialect: DialectPostgres}) + require.NoError(t, err) + require.Contains(t, stmt.SQL, "~") + require.Equal(t, []any{"v[0-9]+"}, stmt.Args) +} + +func TestCompileRejectsStartsWithOnUnsupportedField(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + _, err = engine.Compile(context.Background(), `visibility.startsWith("P")`) + require.Error(t, err) + require.Contains(t, err.Error(), "does not support text matching") +} + +func TestCompileContainsEscapesLikeWildcards(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + stmt, err := engine.CompileToStatement(context.Background(), `content.contains("50%_off")`, RenderOptions{Dialect: DialectSQLite}) + require.NoError(t, err) + // The % and _ in the value must be escaped so they are matched literally, + // and SQLite needs an explicit ESCAPE clause. + require.Contains(t, stmt.SQL, `ESCAPE '\'`) + require.Equal(t, []any{`%50\%\_off%`}, stmt.Args) +} + +// ============================================================================= +// Cross-dialect rendering tests (no DB required; complements the SQLite-only +// behavioral tests in store/test by asserting MySQL/Postgres SQL generation). +// ============================================================================= + +func TestRenderStartsWithPerDialect(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + cases := []struct { + dialect DialectName + fragments []string + }{ + {DialectSQLite, []string{"memos_unicode_lower(", "`memo`.`content`", `ESCAPE '\'`}}, + {DialectPostgres, []string{"memo.content ILIKE $1"}}, + {DialectMySQL, []string{"`memo`.`content` LIKE ?"}}, + } + for _, tc := range cases { + stmt, err := engine.CompileToStatement(context.Background(), `content.startsWith("TODO")`, RenderOptions{Dialect: tc.dialect}) + require.NoError(t, err, tc.dialect) + for _, frag := range tc.fragments { + require.Contains(t, stmt.SQL, frag, "dialect %s", tc.dialect) + } + require.Equal(t, []any{"TODO%"}, stmt.Args, "dialect %s", tc.dialect) + } +} + +func TestRenderEndsWithPerDialect(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + for _, dialect := range []DialectName{DialectSQLite, DialectPostgres, DialectMySQL} { + stmt, err := engine.CompileToStatement(context.Background(), `content.endsWith(".md")`, RenderOptions{Dialect: dialect}) + require.NoError(t, err, dialect) + require.Equal(t, []any{"%.md"}, stmt.Args, "dialect %s", dialect) + } +} + +func TestRenderMatchesPerDialect(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + cases := []struct { + dialect DialectName + fragment string + }{ + {DialectSQLite, "`memo`.`content` REGEXP ?"}, + {DialectMySQL, "`memo`.`content` REGEXP ?"}, + {DialectPostgres, "memo.content ~ $1"}, + } + for _, tc := range cases { + stmt, err := engine.CompileToStatement(context.Background(), `content.matches("v[0-9]+")`, RenderOptions{Dialect: tc.dialect}) + require.NoError(t, err, tc.dialect) + require.Contains(t, stmt.SQL, tc.fragment, "dialect %s", tc.dialect) + require.Equal(t, []any{"v[0-9]+"}, stmt.Args, "dialect %s", tc.dialect) + } +} + +func TestRenderTagsAllPerDialect(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + cases := []struct { + dialect DialectName + fragments []string + }{ + {DialectSQLite, []string{"NOT EXISTS", "json_each(", "!= '[]'", "memos_unicode_lower(value)"}}, + {DialectPostgres, []string{"NOT EXISTS", "jsonb_array_elements_text(", "jsonb_array_length(", "value ILIKE"}}, + {DialectMySQL, []string{"NOT EXISTS", "JSON_TABLE(", "JSON_LENGTH(", "value LIKE"}}, + } + for _, tc := range cases { + stmt, err := engine.CompileToStatement(context.Background(), `tags.all(t, t.startsWith("work/"))`, RenderOptions{Dialect: tc.dialect}) + require.NoError(t, err, tc.dialect) + for _, frag := range tc.fragments { + require.Contains(t, stmt.SQL, frag, "dialect %s", tc.dialect) + } + require.Equal(t, []any{"work/%"}, stmt.Args, "dialect %s", tc.dialect) + } +} + +func TestRenderTextMatchEscaping(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + // Both % and _ in the value must be escaped so they match literally. + stmt, err := engine.CompileToStatement(context.Background(), `content.contains("a%b_c")`, RenderOptions{Dialect: DialectSQLite}) + require.NoError(t, err) + require.Equal(t, []any{`%a\%b\_c%`}, stmt.Args) +} + +func TestRenderAllRejectsUnsupportedPredicate(t *testing.T) { + t.Parallel() + + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + // size() is not a valid per-element predicate inside all(). + _, err = engine.CompileToStatement(context.Background(), `tags.all(t, size(t) > 2)`, RenderOptions{Dialect: DialectSQLite}) + require.Error(t, err) +} diff --git a/internal/filter/ir.go b/internal/filter/ir.go index b5a995dda..fe6bbb259 100644 --- a/internal/filter/ir.go +++ b/internal/filter/ir.go @@ -73,13 +73,32 @@ type ElementInCondition struct { func (*ElementInCondition) isCondition() {} -// ContainsCondition models the .contains() call. -type ContainsCondition struct { +// TextMatchMode enumerates LIKE-based string match modes. +type TextMatchMode string + +const ( + TextMatchContains TextMatchMode = "contains" + TextMatchPrefix TextMatchMode = "prefix" + TextMatchSuffix TextMatchMode = "suffix" +) + +// TextMatchCondition models a case-insensitive LIKE match on a scalar string field +// (content.contains/startsWith/endsWith). +type TextMatchCondition struct { Field string + Mode TextMatchMode Value string } -func (*ContainsCondition) isCondition() {} +func (*TextMatchCondition) isCondition() {} + +// RegexCondition models field.matches("pattern") on a string field. +type RegexCondition struct { + Field string + Pattern string +} + +func (*RegexCondition) isCondition() {} // ConstantCondition captures a literal boolean outcome. type ConstantCondition struct { @@ -130,6 +149,7 @@ type ComprehensionKind string const ( ComprehensionExists ComprehensionKind = "exists" + ComprehensionAll ComprehensionKind = "all" ) // PredicateExpr represents predicates used in comprehensions. diff --git a/internal/filter/parser.go b/internal/filter/parser.go index 2aff1074e..11c651764 100644 --- a/internal/filter/parser.go +++ b/internal/filter/parser.go @@ -87,7 +87,13 @@ func buildCallCondition(call *exprv1.Expr_Call, schema Schema) (Condition, error case "@in": return buildInCondition(call, schema) case "contains": - return buildContainsCondition(call, schema) + return buildTextMatchCondition(call, schema, TextMatchContains) + case "startsWith": + return buildTextMatchCondition(call, schema, TextMatchPrefix) + case "endsWith": + return buildTextMatchCondition(call, schema, TextMatchSuffix) + case "matches": + return buildMatchesCondition(call, schema) default: val, ok, err := evaluateBool(call) if err != nil { @@ -193,9 +199,9 @@ func buildInCondition(call *exprv1.Expr_Call, schema Schema) (Condition, error) return nil, errors.New("invalid use of in operator") } -func buildContainsCondition(call *exprv1.Expr_Call, schema Schema) (Condition, error) { +func buildTextMatchCondition(call *exprv1.Expr_Call, schema Schema, mode TextMatchMode) (Condition, error) { if call.Target == nil { - return nil, errors.New("contains requires a target") + return nil, errors.New("text match requires a target") } targetName, err := getIdentName(call.Target) if err != nil { @@ -207,25 +213,59 @@ func buildContainsCondition(call *exprv1.Expr_Call, schema Schema) (Condition, e return nil, errors.Errorf("unknown identifier %q", targetName) } if !field.SupportsContains { - return nil, errors.Errorf("identifier %q does not support contains()", targetName) + return nil, errors.Errorf("identifier %q does not support text matching", targetName) } if len(call.Args) != 1 { - return nil, errors.New("contains expects exactly one argument") + return nil, errors.New("text match expects exactly one argument") } value, err := getConstValue(call.Args[0]) if err != nil { - return nil, errors.Wrap(err, "contains only supports literal arguments") + return nil, errors.Wrap(err, "text match only supports literal arguments") } str, ok := value.(string) if !ok { - return nil, errors.New("contains argument must be a string") + return nil, errors.New("text match argument must be a string") } - return &ContainsCondition{ + return &TextMatchCondition{ Field: targetName, + Mode: mode, Value: str, }, nil } +func buildMatchesCondition(call *exprv1.Expr_Call, schema Schema) (Condition, error) { + if call.Target == nil { + return nil, errors.New("matches requires a target") + } + targetName, err := getIdentName(call.Target) + if err != nil { + return nil, err + } + + field, ok := schema.Field(targetName) + if !ok { + return nil, errors.Errorf("unknown identifier %q", targetName) + } + if !field.SupportsContains { + return nil, errors.Errorf("identifier %q does not support matches()", targetName) + } + if len(call.Args) != 1 { + return nil, errors.New("matches expects exactly one argument") + } + value, err := getConstValue(call.Args[0]) + if err != nil { + return nil, errors.Wrap(err, "matches only supports literal arguments") + } + pattern, ok := value.(string) + if !ok { + return nil, errors.New("matches argument must be a string") + } + return &RegexCondition{ + Field: targetName, + Pattern: pattern, + }, nil +} + func buildValueExpr(expr *exprv1.Expr, schema Schema) (ValueExpr, error) { if identName, err := getIdentName(expr); err == nil { if _, ok := schema.Field(identName); !ok { @@ -466,10 +506,10 @@ func detectComprehensionKind(comp *exprv1.Expr_Comprehension) (ComprehensionKind } } - // all() starts with true and uses AND (&&) - not supported + // all() starts with true and uses AND (&&) in the loop step. if accuInit.GetBoolValue() { if step := comp.LoopStep.GetCallExpr(); step != nil && step.Function == "_&&_" { - return "", errors.New("all() comprehension is not supported; use exists() instead") + return ComprehensionAll, nil } } diff --git a/internal/filter/render.go b/internal/filter/render.go index 39eaaec01..3f5d4d721 100644 --- a/internal/filter/render.go +++ b/internal/filter/render.go @@ -72,8 +72,10 @@ func (r *renderer) renderCondition(cond Condition) (renderResult, error) { return r.renderInCondition(c) case *ElementInCondition: return r.renderElementInCondition(c) - case *ContainsCondition: - return r.renderContainsCondition(c) + case *TextMatchCondition: + return r.renderTextMatch(c) + case *RegexCondition: + return r.renderRegex(c) case *ListComprehensionCondition: return r.renderListComprehension(c) case *ConstantCondition: @@ -446,28 +448,69 @@ func (r *renderer) renderScalarInCondition(field Field, values []ValueExpr) (ren }, nil } -func (r *renderer) renderContainsCondition(cond *ContainsCondition) (renderResult, error) { +func (r *renderer) renderTextMatch(cond *TextMatchCondition) (renderResult, error) { field, ok := r.schema.Field(cond.Field) if !ok { return renderResult{}, errors.Errorf("unknown field %q", cond.Field) } column := field.columnExpr(r.dialect) - arg := fmt.Sprintf("%%%s%%", cond.Value) + pattern := likePattern(cond.Mode, cond.Value) + return renderResult{sql: r.foldedLike(column, pattern)}, nil +} + +func (r *renderer) renderRegex(cond *RegexCondition) (renderResult, error) { + field, ok := r.schema.Field(cond.Field) + if !ok { + return renderResult{}, errors.Errorf("unknown field %q", cond.Field) + } + column := field.columnExpr(r.dialect) + switch r.dialect { + case DialectPostgres: + // POSIX regex match operator. + return renderResult{sql: fmt.Sprintf("%s ~ %s", column, r.addArg(cond.Pattern))}, nil + case DialectMySQL, DialectSQLite: + // MySQL has a native REGEXP operator; SQLite uses the registered regexp() function. + return renderResult{sql: fmt.Sprintf("%s REGEXP %s", column, r.addArg(cond.Pattern))}, nil + default: + return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect) + } +} + +// foldedLike renders a case-insensitive LIKE comparison of colExpr against a +// (already metacharacter-escaped) pattern, using each dialect's case-folding. +func (r *renderer) foldedLike(colExpr, pattern string) string { switch r.dialect { case DialectSQLite: - // Use custom Unicode-aware case folding function for case-insensitive comparison. - // This overcomes SQLite's ASCII-only LOWER() limitation. - sql := fmt.Sprintf("memos_unicode_lower(%s) LIKE memos_unicode_lower(%s)", column, r.addArg(arg)) - return renderResult{sql: sql}, nil + // memos_unicode_lower gives Unicode-aware folding; ESCAPE '\' is required + // because SQLite has no default LIKE escape character. + return fmt.Sprintf(`memos_unicode_lower(%s) LIKE memos_unicode_lower(%s) ESCAPE '\'`, colExpr, r.addArg(pattern)) case DialectPostgres: - sql := fmt.Sprintf("%s ILIKE %s", column, r.addArg(arg)) - return renderResult{sql: sql}, nil - default: - sql := fmt.Sprintf("%s LIKE %s", column, r.addArg(arg)) - return renderResult{sql: sql}, nil + // ILIKE is case-insensitive; backslash is the default escape character. + return fmt.Sprintf("%s ILIKE %s", colExpr, r.addArg(pattern)) + default: // MySQL: default collation is case-insensitive; backslash is the default escape. + return fmt.Sprintf("%s LIKE %s", colExpr, r.addArg(pattern)) } } +// likePattern escapes LIKE metacharacters in value and wraps it for the mode. +func likePattern(mode TextMatchMode, value string) string { + escaped := escapeLikeLiteral(value) + switch mode { + case TextMatchPrefix: + return escaped + "%" + case TextMatchSuffix: + return "%" + escaped + default: + return "%" + escaped + "%" + } +} + +// escapeLikeLiteral escapes the LIKE metacharacters \, %, and _ so user input +// is matched literally. Backslash is the escape character on all three dialects. +func escapeLikeLiteral(s string) string { + return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s) +} + func (r *renderer) renderListComprehension(cond *ListComprehensionCondition) (renderResult, error) { field, ok := r.schema.Field(cond.Field) if !ok { @@ -478,6 +521,10 @@ func (r *renderer) renderListComprehension(cond *ListComprehensionCondition) (re return renderResult{}, errors.Errorf("field %q is not a JSON list", cond.Field) } + if cond.Kind == ComprehensionAll { + return r.renderTagAll(field, cond.Predicate) + } + // Render based on predicate type switch pred := cond.Predicate.(type) { case *EqualsPredicate: @@ -493,6 +540,51 @@ func (r *renderer) renderListComprehension(cond *ListComprehensionCondition) (re } } +// renderTagAll renders tags.all(t, ): the array is non-empty AND no element +// fails the predicate. Element predicates use plain CEL semantics (case-insensitive +// for startsWith/endsWith/contains, case-sensitive for ==), evaluated per element. +func (r *renderer) renderTagAll(field Field, pred PredicateExpr) (renderResult, error) { + arrayExpr := jsonArrayExpr(r.dialect, field) + elemCond, err := r.elementPredicateSQL(pred) + if err != nil { + return renderResult{}, err + } + switch r.dialect { + case DialectSQLite: + nonEmpty := fmt.Sprintf("%s IS NOT NULL AND %s != '[]'", arrayExpr, arrayExpr) + sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM json_each(%s) WHERE NOT (%s))", arrayExpr, elemCond) + return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil + case DialectMySQL: + nonEmpty := fmt.Sprintf("%s IS NOT NULL AND JSON_LENGTH(%s) > 0", arrayExpr, arrayExpr) + sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM JSON_TABLE(%s, '$[*]' COLUMNS (value VARCHAR(512) PATH '$')) AS elem WHERE NOT (%s))", arrayExpr, elemCond) + return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil + case DialectPostgres: + nonEmpty := fmt.Sprintf("%s IS NOT NULL AND jsonb_array_length(%s) > 0", arrayExpr, arrayExpr) + sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(%s) AS elem(value) WHERE NOT (%s))", arrayExpr, elemCond) + return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil + default: + return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect) + } +} + +// elementPredicateSQL builds the per-element SQL condition for an all() predicate. +// The iterated element is exposed as the unqualified column `value` on all dialects +// (json_each.value / JSON_TABLE column / elem(value)). +func (r *renderer) elementPredicateSQL(pred PredicateExpr) (string, error) { + switch p := pred.(type) { + case *EqualsPredicate: + return fmt.Sprintf("value = %s", r.addArg(p.Value)), nil + case *StartsWithPredicate: + return r.foldedLike("value", likePattern(TextMatchPrefix, p.Prefix)), nil + case *EndsWithPredicate: + return r.foldedLike("value", likePattern(TextMatchSuffix, p.Suffix)), nil + case *ContainsPredicate: + return r.foldedLike("value", likePattern(TextMatchContains, p.Substring)), nil + default: + return "", errors.Errorf("unsupported predicate %T in all()", pred) + } +} + // renderTagEquals generates SQL for tags.exists(t, t == "value"). func (r *renderer) renderTagEquals(field Field, value string, _ ComprehensionKind) (renderResult, error) { arrayExpr := jsonArrayExpr(r.dialect, field) diff --git a/internal/filter/schema.go b/internal/filter/schema.go index ad70e1a35..08867d8b7 100644 --- a/internal/filter/schema.go +++ b/internal/filter/schema.go @@ -256,6 +256,7 @@ func NewSchema() Schema { cel.Variable("has_code", cel.BoolType), cel.Variable("has_incomplete_tasks", cel.BoolType), nowFunction, + cel.ASTValidators(cel.ValidateRegexLiterals()), } return Schema{ @@ -277,11 +278,12 @@ func NewAttachmentSchema() Schema { Expressions: map[DialectName]string{}, }, "mime_type": { - Name: "mime_type", - Kind: FieldKindScalar, - Type: FieldTypeString, - Column: Column{Table: "attachment", Name: "type"}, - Expressions: map[DialectName]string{}, + Name: "mime_type", + Kind: FieldKindScalar, + Type: FieldTypeString, + Column: Column{Table: "attachment", Name: "type"}, + SupportsContains: true, + Expressions: map[DialectName]string{}, }, "create_time": { Name: "create_time", @@ -315,6 +317,7 @@ func NewAttachmentSchema() Schema { cel.Variable("create_time", cel.IntType), cel.Variable("memo_id", cel.AnyType), nowFunction, + cel.ASTValidators(cel.ValidateRegexLiterals()), } return Schema{ diff --git a/store/db/sqlite/functions.go b/store/db/sqlite/functions.go index 6b3021ca7..4e87f927a 100644 --- a/store/db/sqlite/functions.go +++ b/store/db/sqlite/functions.go @@ -5,6 +5,8 @@ package sqlite import ( "database/sql/driver" + "errors" + "regexp" "sync" "golang.org/x/text/cases" @@ -42,3 +44,61 @@ func ensureUnicodeLowerRegistered() error { }) return registerUnicodeLowerErr } + +var ( + registerRegexpOnce sync.Once + registerRegexpErr error + // regexpCache memoizes compiled patterns; keys are pattern strings. + regexpCache sync.Map +) + +// ensureRegexpRegistered registers a Go-backed `regexp(pattern, value)` scalar +// function so SQLite's `value REGEXP pattern` operator works (modernc.org/sqlite +// has no built-in implementation). Patterns use Go's RE2 syntax. Registered once +// globally; safe to call multiple times. +func ensureRegexpRegistered() error { + registerRegexpOnce.Do(func() { + registerRegexpErr = msqlite.RegisterScalarFunction("regexp", 2, func(_ *msqlite.FunctionContext, args []driver.Value) (driver.Value, error) { + if len(args) != 2 || args[0] == nil || args[1] == nil { + return int64(0), nil + } + pattern, ok := args[0].(string) + if !ok { + return nil, errors.New("regexp pattern must be a string") + } + var value string + switch v := args[1].(type) { + case string: + value = v + case []byte: + value = string(v) + default: + return int64(0), nil + } + re, err := compileRegexp(pattern) + if err != nil { + return nil, err + } + if re.MatchString(value) { + return int64(1), nil + } + return int64(0), nil + }) + }) + return registerRegexpErr +} + +// compileRegexp compiles and caches a RE2 pattern. +func compileRegexp(pattern string) (*regexp.Regexp, error) { + if cached, ok := regexpCache.Load(pattern); ok { + if re, ok := cached.(*regexp.Regexp); ok { + return re, nil + } + } + re, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + regexpCache.Store(pattern, re) + return re, nil +} diff --git a/store/db/sqlite/functions_test.go b/store/db/sqlite/functions_test.go new file mode 100644 index 000000000..413708bea --- /dev/null +++ b/store/db/sqlite/functions_test.go @@ -0,0 +1,24 @@ +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRegexpFunctionMatches(t *testing.T) { + require.NoError(t, ensureRegexpRegistered()) + + re, err := compileRegexp(`^v\d+$`) + require.NoError(t, err) + require.True(t, re.MatchString("v12")) + require.False(t, re.MatchString("version")) + + // Caching returns the same compiled instance. + re2, err := compileRegexp(`^v\d+$`) + require.NoError(t, err) + require.Same(t, re, re2) + + _, err = compileRegexp(`(`) + require.Error(t, err) +} diff --git a/store/db/sqlite/sqlite.go b/store/db/sqlite/sqlite.go index 892918e6a..549514740 100644 --- a/store/db/sqlite/sqlite.go +++ b/store/db/sqlite/sqlite.go @@ -31,6 +31,10 @@ func NewDB(profile *profile.Profile) (store.Driver, error) { return nil, errors.Wrap(err, "failed to register sqlite unicode lower function") } + if err := ensureRegexpRegistered(); err != nil { + return nil, errors.Wrap(err, "failed to register sqlite regexp function") + } + // Connect to the database with some sane settings: // - No shared-cache: it's obsolete; WAL journal mode is a better solution. // - No foreign key constraints: it's currently disabled by default, but it's a diff --git a/store/test/attachment_filter_test.go b/store/test/attachment_filter_test.go index a2f6c6af3..c315a4c9f 100644 --- a/store/test/attachment_filter_test.go +++ b/store/test/attachment_filter_test.go @@ -35,6 +35,42 @@ func TestAttachmentFilterFilenameContains(t *testing.T) { require.Len(t, attachments, 0) } +func TestAttachmentFilterFilenameEndsWith(t *testing.T) { + t.Parallel() + tc := NewAttachmentFilterTestContext(t) + defer tc.Close() + + tc.CreateAttachment(NewAttachmentBuilder(tc.CreatorID).Filename("report.pdf").MimeType("application/pdf")) + tc.CreateAttachment(NewAttachmentBuilder(tc.CreatorID).Filename("photo.png").MimeType("image/png")) + + got := tc.ListWithFilter(`filename.endsWith(".pdf")`) + require.Len(t, got, 1) + require.Equal(t, "report.pdf", got[0].Filename) + + // matches() on mime_type, anchored. + got = tc.ListWithFilter(`mime_type.matches("^image/")`) + require.Len(t, got, 1) + require.Equal(t, "photo.png", got[0].Filename) +} + +func TestAttachmentFilterFilenameStartsWith(t *testing.T) { + t.Parallel() + tc := NewAttachmentFilterTestContext(t) + defer tc.Close() + + tc.CreateAttachment(NewAttachmentBuilder(tc.CreatorID).Filename("invoice-2026.pdf").MimeType("application/pdf")) + tc.CreateAttachment(NewAttachmentBuilder(tc.CreatorID).Filename("photo.png").MimeType("image/png")) + + got := tc.ListWithFilter(`filename.startsWith("invoice")`) + require.Len(t, got, 1) + require.Equal(t, "invoice-2026.pdf", got[0].Filename) + + // mime_type prefix matching (newly enabled). + got = tc.ListWithFilter(`mime_type.startsWith("image/")`) + require.Len(t, got, 1) + require.Equal(t, "photo.png", got[0].Filename) +} + func TestAttachmentFilterFilenameSpecialCharacters(t *testing.T) { t.Parallel() tc := NewAttachmentFilterTestContext(t) diff --git a/store/test/memo_filter_test.go b/store/test/memo_filter_test.go index 09f49854c..90c07d766 100644 --- a/store/test/memo_filter_test.go +++ b/store/test/memo_filter_test.go @@ -10,6 +10,14 @@ import ( "github.com/usememos/memos/store" ) +func uids(memos []*store.Memo) []string { + out := make([]string, 0, len(memos)) + for _, m := range memos { + out = append(out, m.UID) + } + return out +} + // ============================================================================= // Content Field Tests // Schema: content (string, supports contains) @@ -94,6 +102,182 @@ func TestMemoFilterContentCaseSensitivity(t *testing.T) { } } +func TestMemoFilterTagsAll(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-all-work", tc.User.ID).Content("all work").Tags("work/a", "work/b")) + tc.CreateMemo(NewMemoBuilder("memo-mixed", tc.User.ID).Content("mixed").Tags("work/a", "home")) + tc.CreateMemo(NewMemoBuilder("memo-untagged", tc.User.ID).Content("untagged")) + + // Every tag starts with "work/": only the all-work memo qualifies. + memos := tc.ListWithFilter(`tags.all(t, t.startsWith("work/"))`) + require.Len(t, memos, 1) + require.Equal(t, "memo-all-work", memos[0].UID) + + // Untagged memos must NOT match (non-empty guard, decision B). + require.NotContains(t, uids(memos), "memo-untagged") +} + +func TestMemoFilterTagsAllEquals(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-only-x", tc.User.ID).Content("only x").Tags("x", "x")) + tc.CreateMemo(NewMemoBuilder("memo-x-and-y", tc.User.ID).Content("x and y").Tags("x", "y")) + + memos := tc.ListWithFilter(`tags.all(t, t == "x")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-only-x", memos[0].UID) +} + +func TestMemoFilterContentStartsWithEscaping(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-pct", tc.User.ID).Content("100% complete")) + tc.CreateMemo(NewMemoBuilder("memo-plain", tc.User.ID).Content("100 things to do")) + + // The % must be treated literally, not as a LIKE wildcard. + memos := tc.ListWithFilter(`content.startsWith("100%")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-pct", memos[0].UID) +} + +func TestMemoFilterStartsWithCombinedAndNegated(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-pub", tc.User.ID).Content("Hello public").Visibility(store.Public)) + tc.CreateMemo(NewMemoBuilder("memo-priv", tc.User.ID).Content("Hello private").Visibility(store.Private)) + tc.CreateMemo(NewMemoBuilder("memo-bye", tc.User.ID).Content("Goodbye public").Visibility(store.Public)) + + memos := tc.ListWithFilter(`content.startsWith("Hello") && visibility == "PUBLIC"`) + require.Len(t, memos, 1) + require.Equal(t, "memo-pub", memos[0].UID) + + memos = tc.ListWithFilter(`!content.startsWith("Hello")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-bye", memos[0].UID) +} + +func TestMemoFilterContentMatchesAdvanced(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-release", tc.User.ID).Content("release v2024 notes")) + tc.CreateMemo(NewMemoBuilder("memo-draft", tc.User.ID).Content("draft document")) + tc.CreateMemo(NewMemoBuilder("memo-animals", tc.User.ID).Content("cat and dog")) + + // Anchor: starts with "release". + memos := tc.ListWithFilter(`content.matches("^release")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-release", memos[0].UID) + + // Character class + quantifier: a 4-digit run (portable across RE2/POSIX/ICU). + memos = tc.ListWithFilter(`content.matches("[0-9]{4}")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-release", memos[0].UID) + + // Alternation. + memos = tc.ListWithFilter(`content.matches("cat|mouse")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-animals", memos[0].UID) + + // No match. + memos = tc.ListWithFilter(`content.matches("^zzz")`) + require.Len(t, memos, 0) +} + +func TestMemoFilterTagsAllMorePredicates(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-all-done", tc.User.ID).Tags("a/done", "b/done")) + tc.CreateMemo(NewMemoBuilder("memo-mixed", tc.User.ID).Tags("a/done", "b/todo")) + tc.CreateMemo(NewMemoBuilder("memo-single", tc.User.ID).Tags("solo/done")) + + // endsWith: every tag ends with "done". + memos := tc.ListWithFilter(`tags.all(t, t.endsWith("done"))`) + require.ElementsMatch(t, []string{"memo-all-done", "memo-single"}, uids(memos)) + + // contains: every tag contains "/". + memos = tc.ListWithFilter(`tags.all(t, t.contains("/"))`) + require.ElementsMatch(t, []string{"memo-all-done", "memo-mixed", "memo-single"}, uids(memos)) +} + +func TestMemoFilterTagsAllNegatedAndCombined(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-work", tc.User.ID).Content("work memo").Tags("work/a", "work/b")) + tc.CreateMemo(NewMemoBuilder("memo-mixed", tc.User.ID).Content("mixed memo").Tags("work/a", "home")) + tc.CreateMemo(NewMemoBuilder("memo-untagged", tc.User.ID).Content("untagged memo")) + + // Negation: NOT all-work. The untagged memo's all() is false (non-empty guard), + // so !all() is true and it is included. + memos := tc.ListWithFilter(`!tags.all(t, t.startsWith("work/"))`) + require.ElementsMatch(t, []string{"memo-mixed", "memo-untagged"}, uids(memos)) + + // Combined with a content filter. + memos = tc.ListWithFilter(`tags.all(t, t.startsWith("work/")) && content.contains("work")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-work", memos[0].UID) +} + +func TestMemoFilterContentStartsWith(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-todo", tc.User.ID).Content("TODO: buy milk")) + tc.CreateMemo(NewMemoBuilder("memo-done", tc.User.ID).Content("Done with milk")) + + // Prefix match, case-insensitive (consistent with contains()). + memos := tc.ListWithFilter(`content.startsWith("todo")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-todo", memos[0].UID) + + memos = tc.ListWithFilter(`content.startsWith("nope")`) + require.Len(t, memos, 0) +} + +func TestMemoFilterContentEndsWith(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-md", tc.User.ID).Content("notes.md")) + tc.CreateMemo(NewMemoBuilder("memo-txt", tc.User.ID).Content("notes.txt")) + + memos := tc.ListWithFilter(`content.endsWith(".md")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-md", memos[0].UID) +} + +func TestMemoFilterContentMatches(t *testing.T) { + t.Parallel() + tc := NewMemoFilterTestContext(t) + defer tc.Close() + + tc.CreateMemo(NewMemoBuilder("memo-v1", tc.User.ID).Content("release v12 shipped")) + tc.CreateMemo(NewMemoBuilder("memo-plain", tc.User.ID).Content("no version here")) + + memos := tc.ListWithFilter(`content.matches("v[0-9]+")`) + require.Len(t, memos, 1) + require.Equal(t, "memo-v1", memos[0].UID) + + memos = tc.ListWithFilter(`content.matches("^xyz")`) + require.Len(t, memos, 0) +} + // ============================================================================= // Visibility Field Tests // Schema: visibility (string, ==, !=) diff --git a/web/src/pages/Shortcuts.tsx b/web/src/pages/Shortcuts.tsx index 996ddf745..5e9850ebe 100644 --- a/web/src/pages/Shortcuts.tsx +++ b/web/src/pages/Shortcuts.tsx @@ -85,14 +85,36 @@ const shortcutExamples = [ description: "Search text inside memo content.", icon: SearchIcon, }, + { + title: "Starts with", + filter: 'content.startsWith("TODO")', + description: "Memos whose content begins with text (also endsWith).", + icon: SearchIcon, + }, + { + title: "Regex match", + filter: 'content.matches("v[0-9]+")', + description: "Match content with a regular expression.", + icon: FilterIcon, + }, + { + title: "All tags match", + filter: 'tags.all(t, t.startsWith("work/"))', + description: "Every tag must satisfy the predicate (tagged memos only).", + icon: TagsIcon, + }, ]; const filterFields = [ "content.contains(...)", + "content.startsWith(...)", + "content.endsWith(...)", + "content.matches(...)", "visibility", "pinned", "tag in [...]", "tags.exists(...)", + "tags.all(...)", "has_task_list", "has_incomplete_tasks", "has_link",