Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Steven
|
e171fbc5b9 |
+1
-12
@@ -1,13 +1,2 @@
|
||||
web/node_modules
|
||||
web/dist
|
||||
.git
|
||||
.github
|
||||
build/
|
||||
tmp/
|
||||
memos
|
||||
*.md
|
||||
.gitignore
|
||||
.golangci.yaml
|
||||
.dockerignore
|
||||
docs/
|
||||
.DS_Store
|
||||
web/yarn.lock
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
github: usememos
|
||||
@@ -1,162 +0,0 @@
|
||||
name: Bug Report
|
||||
description: Report reproducible behavior that is broken in the current version
|
||||
type: Bug
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting a bug. Please help maintainers reproduce it on current code.
|
||||
|
||||
Before submitting:
|
||||
- Search [existing issues](https://github.com/usememos/memos/issues) to avoid duplicates.
|
||||
- Upgrade and retest with the [latest release](https://github.com/usememos/memos/releases/latest), current `main`, or the [demo site](https://demo.usememos.com/).
|
||||
- Use [Discussions](https://github.com/usememos/memos/discussions) for questions, setup help, or configuration troubleshooting.
|
||||
|
||||
Maintainers normally fix bugs in current code and future releases, not older released versions.
|
||||
|
||||
- type: checkboxes
|
||||
id: pre-check
|
||||
attributes:
|
||||
label: Pre-submission Checklist
|
||||
options:
|
||||
- label: I have searched existing issues and confirmed this has not already been reported
|
||||
required: true
|
||||
- label: I have reproduced this bug on the latest Memos release, current `main`, or the demo site, or this report is specifically about upgrading to the latest release
|
||||
required: true
|
||||
- label: This is a bug report, not a question or configuration request
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: area
|
||||
attributes:
|
||||
label: Affected Area
|
||||
description: Choose the main area where the bug appears.
|
||||
multiple: true
|
||||
options:
|
||||
- Web UI
|
||||
- API
|
||||
- Authentication / SSO
|
||||
- Database / migration
|
||||
- Resources / attachments
|
||||
- Search / filters
|
||||
- Markdown / editor
|
||||
- Deployment / startup
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: reproduced-on
|
||||
attributes:
|
||||
label: Reproduced On
|
||||
description: Where did you confirm the issue still happens?
|
||||
options:
|
||||
- Latest release
|
||||
- Current main branch
|
||||
- Demo site
|
||||
- Upgrade to latest release
|
||||
- I cannot test the latest version
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Memos Version Tested
|
||||
description: Report the exact version where you reproduced the bug. Use the version shown in **Settings > System**, the container image tag, the release binary version, or `memos --version`. Please do not write `latest` only.
|
||||
placeholder: "v0.29.x, ghcr.io/usememos/memos:v0.29.x, main@abcdef, or demo site"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: previous-version
|
||||
attributes:
|
||||
label: Previous Version
|
||||
description: If this happened after an upgrade, enter the version you upgraded from. Leave blank if not applicable.
|
||||
placeholder: "v0.28.x, unknown, or not applicable"
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: Environment Details
|
||||
description: Include the deployment, database, client, and network details that apply.
|
||||
placeholder: |
|
||||
- Deployment: Docker / binary / source / demo site
|
||||
- Database: SQLite / PostgreSQL / MySQL
|
||||
- Browser: Firefox 126 on macOS 15
|
||||
- Server OS: Ubuntu 24.04
|
||||
- Reverse proxy: nginx / Caddy / Traefik / none
|
||||
- Related settings: SSO / SMTP / storage / webhooks / none
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: frequency
|
||||
attributes:
|
||||
label: Reproduction Frequency
|
||||
description: How reliably can you reproduce this?
|
||||
options:
|
||||
- Always
|
||||
- Often
|
||||
- Sometimes
|
||||
- Once
|
||||
- Unknown
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: reproduction-steps
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: Provide minimal, numbered steps that start from a clean page/session when possible.
|
||||
placeholder: |
|
||||
1. Open ...
|
||||
2. Click ...
|
||||
3. Enter ...
|
||||
4. Wait for ...
|
||||
5. See ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: current-behavior
|
||||
attributes:
|
||||
label: Current Behavior
|
||||
description: What happened? Include exact error messages, status codes, or visible UI state if available.
|
||||
placeholder: The application ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: Expected Behavior
|
||||
description: What did you expect to happen instead?
|
||||
placeholder: I expected...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: regression
|
||||
attributes:
|
||||
label: Regression Information
|
||||
description: If this worked before, describe when it last worked and what changed before it broke.
|
||||
placeholder: |
|
||||
This last worked in ...
|
||||
It started after ...
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs
|
||||
description: Paste relevant server logs, browser console output, or network errors. Remove secrets before submitting.
|
||||
render: shell
|
||||
placeholder: |
|
||||
Paste logs here.
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: Additional Context
|
||||
description: Attach screenshots, screen recordings, sample data, exports, or links to related issues.
|
||||
placeholder: Drag and drop files here or add any extra context.
|
||||
@@ -1,8 +0,0 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Questions & Support
|
||||
url: https://github.com/usememos/memos/discussions
|
||||
about: Ask questions or get help in GitHub Discussions — please don't open issues for questions
|
||||
- name: Documentation
|
||||
url: https://www.usememos.com/docs
|
||||
about: Check the documentation before opening an issue
|
||||
@@ -1,109 +0,0 @@
|
||||
name: Feature Request
|
||||
description: Propose a product improvement with a clear use case
|
||||
type: Feature
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for suggesting an improvement. Please describe the problem and workflow first so maintainers can evaluate the request.
|
||||
|
||||
Before submitting:
|
||||
- Search [existing feature requests](https://github.com/usememos/memos/issues?q=label%3Aenhancement) to avoid duplicates.
|
||||
- Use [Discussions](https://github.com/usememos/memos/discussions) for broad ideas, setup help, or questions.
|
||||
- Keep one feature request focused on one user problem.
|
||||
|
||||
- type: checkboxes
|
||||
id: pre-check
|
||||
attributes:
|
||||
label: Pre-submission Checklist
|
||||
options:
|
||||
- label: I have searched existing issues and confirmed this has not already been requested
|
||||
required: true
|
||||
- label: This is a feature request, not a bug report or support question
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: area
|
||||
attributes:
|
||||
label: Area
|
||||
description: Choose the main area this request affects.
|
||||
multiple: true
|
||||
options:
|
||||
- Web UI
|
||||
- Mobile / responsive experience
|
||||
- API
|
||||
- Authentication / SSO
|
||||
- Administration
|
||||
- Memo editor
|
||||
- Search / filters
|
||||
- Tags / organization
|
||||
- Resources / attachments
|
||||
- Import / export
|
||||
- Deployment / operations
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem or Use Case
|
||||
description: What problem does this solve? Describe the user need, not just the proposed UI or API.
|
||||
placeholder: |
|
||||
I often need to ...
|
||||
Today, this is difficult because ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: users
|
||||
attributes:
|
||||
label: Who Would Use This?
|
||||
description: Describe the user, role, team, or workflow that would benefit.
|
||||
placeholder: |
|
||||
This helps ...
|
||||
They currently ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: Describe the behavior you would like Memos to support. Include UI, API, or configuration details if relevant.
|
||||
placeholder: |
|
||||
Memos should ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
description: Existing workarounds, related tools, or simpler alternatives you considered.
|
||||
placeholder: |
|
||||
I tried ...
|
||||
It does not work well because ...
|
||||
|
||||
- type: textarea
|
||||
id: compatibility
|
||||
attributes:
|
||||
label: Compatibility and Migration
|
||||
description: Note any expected impact on existing users, APIs, storage, permissions, or deployments.
|
||||
placeholder: |
|
||||
This should not affect existing memos because ...
|
||||
Existing API clients may need ...
|
||||
|
||||
- type: textarea
|
||||
id: examples
|
||||
attributes:
|
||||
label: Examples
|
||||
description: Link to screenshots, mockups, API sketches, related apps, or existing issues that show the desired behavior.
|
||||
placeholder: Drag and drop files here or add links/examples.
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: Additional Context
|
||||
description: Add any extra context that does not fit above.
|
||||
placeholder: Any other details.
|
||||
@@ -1,91 +0,0 @@
|
||||
name: Backend Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
- "**.go"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.26.2"
|
||||
|
||||
jobs:
|
||||
static-checks:
|
||||
name: Static Checks
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- name: Verify go.mod is tidy
|
||||
run: |
|
||||
go mod tidy -go=${{ env.GO_VERSION }}
|
||||
git diff --exit-code
|
||||
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.11.3
|
||||
args: --timeout=3m
|
||||
|
||||
tests:
|
||||
name: Tests (${{ matrix.test-group }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-group: [store, server, internal, other]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
case "${{ matrix.test-group }}" in
|
||||
store)
|
||||
# Run store tests for all drivers (sqlite, mysql, postgres)
|
||||
go test -v -coverprofile=coverage.out -covermode=atomic ./store/...
|
||||
;;
|
||||
server)
|
||||
go test -v -race -coverprofile=coverage.out -covermode=atomic ./server/...
|
||||
;;
|
||||
internal)
|
||||
go test -v -race -coverprofile=coverage.out -covermode=atomic ./internal/...
|
||||
;;
|
||||
other)
|
||||
go test -v -race -coverprofile=coverage.out -covermode=atomic \
|
||||
./cmd/... ./internal/... ./proto/...
|
||||
;;
|
||||
esac
|
||||
env:
|
||||
DRIVER: ${{ matrix.test-group == 'store' && '' || 'sqlite' }}
|
||||
|
||||
- name: Upload coverage
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
files: ./coverage.out
|
||||
flags: ${{ matrix.test-group }}
|
||||
fail_ci_if_error: false
|
||||
@@ -0,0 +1,44 @@
|
||||
name: build-and-push-release-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
# Run on pushing branches like `release/1.0.0`
|
||||
- "release/*.*.*"
|
||||
|
||||
jobs:
|
||||
build-and-push-release-image:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2
|
||||
|
||||
- name: Extract build args
|
||||
# Extract version from branch name
|
||||
# Example: branch name `release/1.0.0` sets up env.VERSION=1.0.0
|
||||
run: |
|
||||
echo "VERSION=${GITHUB_REF_NAME#release/}" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: neosmemo
|
||||
password: ${{ secrets.DOCKER_NEOSMEMO_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
with:
|
||||
install: true
|
||||
|
||||
- name: Build and Push
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: neosmemo/memos:latest, neosmemo/memos:${{ env.VERSION }}
|
||||
@@ -1,180 +0,0 @@
|
||||
name: Build Canary Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.repository }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: pnpm/action-setup@v4.2.0
|
||||
with:
|
||||
version: 11.0.1
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: "web/pnpm-lock.yaml"
|
||||
- name: Get pnpm store directory
|
||||
id: pnpm-cache
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
|
||||
- name: Setup pnpm cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }}
|
||||
restore-keys: ${{ runner.os }}-pnpm-store-
|
||||
- run: pnpm install --frozen-lockfile
|
||||
working-directory: web
|
||||
- name: Run frontend build
|
||||
run: pnpm release
|
||||
working-directory: web
|
||||
|
||||
- name: Upload frontend artifacts
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: server/router/frontend/dist
|
||||
retention-days: 1
|
||||
|
||||
build-push:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform:
|
||||
- linux/amd64
|
||||
- linux/arm64
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve version from latest tag
|
||||
id: version
|
||||
run: |
|
||||
version="$(git describe --tags --abbrev=0 2>/dev/null || echo dev)"
|
||||
echo "version=${version#v}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download frontend artifacts
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: server/router/frontend/dist
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./scripts/Dockerfile
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.version.outputs.version }}
|
||||
COMMIT=${{ github.sha }}
|
||||
cache-from: type=gha,scope=build-${{ matrix.platform }}
|
||||
cache-to: type=gha,mode=max,scope=build-${{ matrix.platform }}
|
||||
outputs: type=image,name=neosmemo/memos,push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: digests-${{ strategy.job-index }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
needs: build-push
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
neosmemo/memos
|
||||
ghcr.io/usememos/memos
|
||||
flavor: |
|
||||
latest=false
|
||||
tags: |
|
||||
type=raw,value=canary
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf 'neosmemo/memos@sha256:%s ' *)
|
||||
env:
|
||||
DOCKER_METADATA_OUTPUT_JSON: ${{ steps.meta.outputs.json }}
|
||||
|
||||
- name: Inspect images
|
||||
run: |
|
||||
docker buildx imagetools inspect neosmemo/memos:canary
|
||||
docker buildx imagetools inspect ghcr.io/usememos/memos:canary
|
||||
@@ -0,0 +1,70 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: "27 12 * * 0"
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: ["go", "javascript"]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
|
||||
# Learn more about CodeQL language support at https://git.io/codeql-language-support
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v1
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
@@ -1,17 +0,0 @@
|
||||
name: Demo Deploy
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy-demo:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Trigger Render Deploy
|
||||
run: |
|
||||
curl -X POST "${{ secrets.RENDER_DEPLOY_HOOK }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"trigger": "github_action"}'
|
||||
|
||||
- name: Deployment Status
|
||||
run: echo "Demo deployment triggered successfully on Render"
|
||||
@@ -1,76 +0,0 @@
|
||||
name: Frontend Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "web/**"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_VERSION: "24"
|
||||
PNPM_VERSION: "11.0.1"
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4.2.0
|
||||
with:
|
||||
version: ${{ env.PNPM_VERSION }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: pnpm
|
||||
cache-dependency-path: web/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: web
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Run lint
|
||||
working-directory: web
|
||||
run: pnpm lint
|
||||
|
||||
- name: Run unit tests
|
||||
working-directory: web
|
||||
run: pnpm test
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4.2.0
|
||||
with:
|
||||
version: ${{ env.PNPM_VERSION }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: pnpm
|
||||
cache-dependency-path: web/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: web
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build frontend
|
||||
working-directory: web
|
||||
run: pnpm build
|
||||
@@ -1,40 +0,0 @@
|
||||
name: Proto Linter
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "proto/**"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint Protos
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup buf
|
||||
uses: bufbuild/buf-setup-action@v1
|
||||
with:
|
||||
github_token: ${{ github.token }}
|
||||
|
||||
- name: Run buf lint
|
||||
uses: bufbuild/buf-lint-action@v1
|
||||
with:
|
||||
input: proto
|
||||
|
||||
- name: Check buf format
|
||||
run: |
|
||||
if [[ $(buf format -d) ]]; then
|
||||
echo "❌ Proto files are not formatted. Run 'buf format -w' to fix."
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,37 +0,0 @@
|
||||
name: Release Please
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
release-please:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check release token
|
||||
env:
|
||||
RELEASE_PLEASE_TOKEN: ${{ secrets.RELEASE_PLEASE_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$RELEASE_PLEASE_TOKEN" ]; then
|
||||
echo "RELEASE_PLEASE_TOKEN must be set to a fine-grained PAT so release-please tags can trigger release.yml." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run release-please
|
||||
uses: googleapis/release-please-action@v4
|
||||
with:
|
||||
# Use a fine-grained PAT so generated tags trigger release.yml.
|
||||
token: ${{ secrets.RELEASE_PLEASE_TOKEN }}
|
||||
config-file: release-please-config.json
|
||||
manifest-file: .release-please-manifest.json
|
||||
@@ -1,384 +0,0 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.26.2"
|
||||
NODE_VERSION: "24"
|
||||
PNPM_VERSION: "11.0.1"
|
||||
ARTIFACT_RETENTION_DAYS: 60
|
||||
ARTIFACT_PREFIX: memos
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Extract Version
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
major_minor: ${{ steps.version.outputs.major_minor }}
|
||||
is_prerelease: ${{ steps.version.outputs.is_prerelease }}
|
||||
steps:
|
||||
- name: Extract version
|
||||
id: version
|
||||
env:
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
run: |
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
echo "tag=" >> "$GITHUB_OUTPUT"
|
||||
echo "version=manual-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
echo "major_minor=" >> "$GITHUB_OUTPUT"
|
||||
echo "is_prerelease=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ! "$REF_NAME" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)(-rc\.[0-9]+)?$ ]]; then
|
||||
echo "Unsupported release tag format: $REF_NAME" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
version="${BASH_REMATCH[1]}${BASH_REMATCH[2]}"
|
||||
major_minor="${BASH_REMATCH[1]%.*}"
|
||||
is_prerelease=false
|
||||
if [ -n "${BASH_REMATCH[2]}" ]; then
|
||||
is_prerelease=true
|
||||
fi
|
||||
|
||||
echo "tag=${REF_NAME}" >> "$GITHUB_OUTPUT"
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
echo "major_minor=${major_minor}" >> "$GITHUB_OUTPUT"
|
||||
echo "is_prerelease=${is_prerelease}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build-frontend:
|
||||
name: Build Frontend
|
||||
needs: prepare
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4.2.0
|
||||
with:
|
||||
version: ${{ env.PNPM_VERSION }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: pnpm
|
||||
cache-dependency-path: web/pnpm-lock.yaml
|
||||
|
||||
- name: Get pnpm store directory
|
||||
id: pnpm-cache
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup pnpm cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }}
|
||||
restore-keys: ${{ runner.os }}-pnpm-store-
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: web
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build frontend release assets
|
||||
working-directory: web
|
||||
run: pnpm release
|
||||
|
||||
- name: Upload frontend artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: server/router/frontend/dist
|
||||
retention-days: 1
|
||||
|
||||
build-binaries:
|
||||
name: Build ${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.goarm && format('v{0}', matrix.goarm) || '' }}
|
||||
needs: [prepare, build-frontend]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
- goos: linux
|
||||
goarch: arm
|
||||
goarm: "7"
|
||||
- goos: darwin
|
||||
goarch: amd64
|
||||
- goos: darwin
|
||||
goarch: arm64
|
||||
- goos: windows
|
||||
goarch: amd64
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: true
|
||||
|
||||
- name: Download frontend artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: server/router/frontend/dist
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
GOARM: ${{ matrix.goarm }}
|
||||
CGO_ENABLED: "0"
|
||||
run: |
|
||||
output_name="memos"
|
||||
if [ "$GOOS" = "windows" ]; then
|
||||
output_name="memos.exe"
|
||||
fi
|
||||
|
||||
mkdir -p build
|
||||
|
||||
go build \
|
||||
-trimpath \
|
||||
-ldflags="-s -w -X github.com/usememos/memos/internal/version.Version=${{ needs.prepare.outputs.version }} -X github.com/usememos/memos/internal/version.Commit=${{ github.sha }} -extldflags '-static'" \
|
||||
-tags netgo,osusergo \
|
||||
-o "build/${output_name}" \
|
||||
./cmd/memos
|
||||
|
||||
- name: Package binary
|
||||
env:
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
GOARM: ${{ matrix.goarm }}
|
||||
run: |
|
||||
cd build
|
||||
|
||||
package_name="${ARTIFACT_PREFIX}_${VERSION}_${GOOS}_${GOARCH}"
|
||||
if [ -n "$GOARM" ]; then
|
||||
package_name="${package_name}v${GOARM}"
|
||||
fi
|
||||
|
||||
if [ "$GOOS" = "windows" ]; then
|
||||
artifact_name="${package_name}.zip"
|
||||
zip -q "${artifact_name}" memos.exe
|
||||
else
|
||||
artifact_name="${package_name}.tar.gz"
|
||||
tar czf "${artifact_name}" memos
|
||||
fi
|
||||
|
||||
echo "artifact_name=${artifact_name}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ env.artifact_name }}
|
||||
path: build/${{ env.artifact_name }}
|
||||
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
|
||||
|
||||
checksums:
|
||||
name: Generate Checksums
|
||||
needs: [prepare, build-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download binary artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
pattern: ${{ env.ARTIFACT_PREFIX }}_*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Generate checksums
|
||||
working-directory: artifacts
|
||||
run: sha256sum * > checksums.txt
|
||||
|
||||
- name: Upload checksum artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: checksums
|
||||
path: artifacts/checksums.txt
|
||||
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
|
||||
|
||||
release:
|
||||
name: Publish GitHub Release
|
||||
needs: [prepare, build-binaries, checksums]
|
||||
if: github.event_name != 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download binary artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
pattern: ${{ env.ARTIFACT_PREFIX }}_*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download checksum artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: checksums
|
||||
path: artifacts
|
||||
|
||||
- name: Publish release assets
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ needs.prepare.outputs.tag }}
|
||||
name: ${{ needs.prepare.outputs.tag }}
|
||||
prerelease: ${{ needs.prepare.outputs.is_prerelease == 'true' }}
|
||||
files: artifacts/*
|
||||
|
||||
build-push:
|
||||
name: Build Image ${{ matrix.platform }}
|
||||
needs: [prepare, build-frontend]
|
||||
if: github.event_name != 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform:
|
||||
- linux/amd64
|
||||
- linux/arm/v7
|
||||
- linux/arm64
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download frontend artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: server/router/frontend/dist
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./scripts/Dockerfile
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: |
|
||||
VERSION=${{ needs.prepare.outputs.version }}
|
||||
COMMIT=${{ github.sha }}
|
||||
cache-from: type=gha,scope=release-${{ matrix.platform }}
|
||||
cache-to: type=gha,mode=max,scope=release-${{ matrix.platform }}
|
||||
outputs: type=image,name=neosmemo/memos,push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ strategy.job-index }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge-images:
|
||||
name: Publish Release Image Tags
|
||||
needs: [prepare, build-push]
|
||||
if: github.event_name != 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
version="${{ needs.prepare.outputs.version }}"
|
||||
if [ "${{ needs.prepare.outputs.is_prerelease }}" = "true" ]; then
|
||||
docker buildx imagetools create \
|
||||
-t "neosmemo/memos:${version}" \
|
||||
-t "ghcr.io/usememos/memos:${version}" \
|
||||
$(printf 'neosmemo/memos@sha256:%s ' *)
|
||||
exit 0
|
||||
fi
|
||||
|
||||
docker buildx imagetools create \
|
||||
-t "neosmemo/memos:${version}" \
|
||||
-t "neosmemo/memos:${{ needs.prepare.outputs.major_minor }}" \
|
||||
-t "neosmemo/memos:stable" \
|
||||
-t "ghcr.io/usememos/memos:${version}" \
|
||||
-t "ghcr.io/usememos/memos:${{ needs.prepare.outputs.major_minor }}" \
|
||||
-t "ghcr.io/usememos/memos:stable" \
|
||||
$(printf 'neosmemo/memos@sha256:%s ' *)
|
||||
|
||||
- name: Inspect images
|
||||
run: |
|
||||
docker buildx imagetools inspect neosmemo/memos:${{ needs.prepare.outputs.version }}
|
||||
if [ "${{ needs.prepare.outputs.is_prerelease }}" = "true" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
docker buildx imagetools inspect neosmemo/memos:stable
|
||||
@@ -1,24 +0,0 @@
|
||||
name: Close Stale
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 */8 * * *" # Every 8 hours
|
||||
|
||||
jobs:
|
||||
close-stale:
|
||||
name: Close Stale Issues and PRs
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Mark and close stale issues and PRs
|
||||
uses: actions/stale@v10.1.1
|
||||
with:
|
||||
# Issues: mark stale after 14 days of inactivity, close after 3 more days
|
||||
days-before-issue-stale: 14
|
||||
days-before-issue-close: 3
|
||||
|
||||
# Pull requests: mark stale after 14 days of inactivity, close after 3 more days
|
||||
days-before-pr-stale: 14
|
||||
days-before-pr-close: 3
|
||||
@@ -0,0 +1,73 @@
|
||||
name: Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release/v*.*.*"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
go-static-checks:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.18
|
||||
check-latest: true
|
||||
cache: true
|
||||
- name: Verify go.mod is tidy
|
||||
run: |
|
||||
go mod tidy
|
||||
git diff --exit-code
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
args: -v
|
||||
skip-cache: true
|
||||
|
||||
eslint-checks:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: "16"
|
||||
cache: yarn
|
||||
cache-dependency-path: "web/yarn.lock"
|
||||
- run: yarn
|
||||
working-directory: web
|
||||
- name: Run eslint check
|
||||
run: yarn lint
|
||||
working-directory: web
|
||||
|
||||
frontend-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: "16"
|
||||
cache: yarn
|
||||
cache-dependency-path: "web/yarn.lock"
|
||||
- run: yarn
|
||||
working-directory: web
|
||||
- name: Run frontend build
|
||||
run: yarn build
|
||||
working-directory: web
|
||||
|
||||
go-tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.18
|
||||
check-latest: true
|
||||
cache: true
|
||||
- name: Run all tests
|
||||
run: go test -v ./... | tee test.log; exit ${PIPESTATUS[0]}
|
||||
- name: Pretty print tests running time
|
||||
run: grep --color=never -e '--- PASS:' -e '--- FAIL:' test.log | sed 's/[:()]//g' | awk '{print $2,$3,$4}' | sort -t' ' -nk3 -r | awk '{sum += $3; print $1,$2,$3,sum"s"}'
|
||||
+6
-26
@@ -1,33 +1,13 @@
|
||||
# Air (hot reload) generated
|
||||
.air
|
||||
|
||||
# temp folder
|
||||
tmp
|
||||
|
||||
# Frontend asset
|
||||
web/dist
|
||||
|
||||
# Build artifacts
|
||||
build/
|
||||
bin/
|
||||
memos
|
||||
# build folder
|
||||
build
|
||||
|
||||
.DS_Store
|
||||
|
||||
# Jetbrains
|
||||
.idea
|
||||
|
||||
# Docker Compose Environment File
|
||||
.env
|
||||
|
||||
dist
|
||||
|
||||
# VSCode settings
|
||||
.vscode
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Local pnpm store (project-scoped, created when --config.store-dir is set
|
||||
# without an existing store; contains a symlink back to the workspace).
|
||||
.pnpm-store/
|
||||
|
||||
# Frontend test coverage output (Vitest + @vitest/coverage-v8).
|
||||
web/coverage/
|
||||
.DS_Store
|
||||
+52
-89
@@ -1,7 +1,6 @@
|
||||
version: "2"
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- goimports
|
||||
- revive
|
||||
- govet
|
||||
- staticcheck
|
||||
@@ -11,91 +10,55 @@ linters:
|
||||
- rowserrcheck
|
||||
- nilerr
|
||||
- godot
|
||||
- forbidigo
|
||||
- mirror
|
||||
- bodyclose
|
||||
disable:
|
||||
- errcheck
|
||||
settings:
|
||||
exhaustive:
|
||||
explicit-exhaustive-switch: false
|
||||
staticcheck:
|
||||
checks:
|
||||
- all
|
||||
- -ST1000
|
||||
- -ST1003
|
||||
- -ST1021
|
||||
- -QF1003
|
||||
revive:
|
||||
# Default to run all linters so that new rules in the future could automatically be added to the static check.
|
||||
enable-all-rules: true
|
||||
rules:
|
||||
# The following rules are too strict and make coding harder. We do not enable them for now.
|
||||
- name: file-header
|
||||
disabled: true
|
||||
- name: line-length-limit
|
||||
disabled: true
|
||||
- name: function-length
|
||||
disabled: true
|
||||
- name: max-public-structs
|
||||
disabled: true
|
||||
- name: function-result-limit
|
||||
disabled: true
|
||||
- name: banned-characters
|
||||
disabled: true
|
||||
- name: argument-limit
|
||||
disabled: true
|
||||
- name: cognitive-complexity
|
||||
disabled: true
|
||||
- name: cyclomatic
|
||||
disabled: true
|
||||
- name: confusing-results
|
||||
disabled: true
|
||||
- name: add-constant
|
||||
disabled: true
|
||||
- name: flag-parameter
|
||||
disabled: true
|
||||
- name: nested-structs
|
||||
disabled: true
|
||||
- name: import-shadowing
|
||||
disabled: true
|
||||
- name: early-return
|
||||
disabled: true
|
||||
- name: use-any
|
||||
disabled: true
|
||||
- name: exported
|
||||
disabled: true
|
||||
- name: unhandled-error
|
||||
disabled: true
|
||||
- name: if-return
|
||||
disabled: true
|
||||
- name: max-control-nesting
|
||||
disabled: true
|
||||
- name: redefines-builtin-id
|
||||
disabled: true
|
||||
- name: package-comments
|
||||
disabled: true
|
||||
gocritic:
|
||||
disabled-checks:
|
||||
- ifElseChain
|
||||
govet:
|
||||
settings:
|
||||
printf: # The name of the analyzer, run `go tool vet help` to see the list of all analyzers
|
||||
funcs: # Run `go tool vet help printf` to see the full configuration of `printf`.
|
||||
- common.Errorf
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment
|
||||
- shadow
|
||||
forbidigo:
|
||||
forbid:
|
||||
- pattern: 'fmt\.Errorf(# Please use errors\.Wrap\|Wrapf\|Errorf instead)?'
|
||||
- pattern: 'ioutil\.ReadDir(# Please use os\.ReadDir)?'
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- goimports
|
||||
settings:
|
||||
goimports:
|
||||
local-prefixes:
|
||||
- github.com/usememos/memos
|
||||
issues:
|
||||
exclude:
|
||||
- Rollback
|
||||
- fmt.Printf
|
||||
- fmt.Print
|
||||
|
||||
linters-settings:
|
||||
revive:
|
||||
enable-all-rules: true
|
||||
rules:
|
||||
- name: file-header
|
||||
disabled: true
|
||||
- name: line-length-limit
|
||||
disabled: true
|
||||
- name: function-length
|
||||
disabled: true
|
||||
- name: max-public-structs
|
||||
disabled: true
|
||||
- name: function-result-limit
|
||||
disabled: true
|
||||
- name: banned-characters
|
||||
disabled: true
|
||||
- name: argument-limit
|
||||
disabled: true
|
||||
- name: cognitive-complexity
|
||||
disabled: true
|
||||
- name: cyclomatic
|
||||
disabled: true
|
||||
- name: confusing-results
|
||||
disabled: true
|
||||
- name: add-constant
|
||||
disabled: true
|
||||
- name: flag-parameter
|
||||
disabled: true
|
||||
- name: nested-structs
|
||||
disabled: true
|
||||
- name: import-shadowing
|
||||
disabled: true
|
||||
- name: early-return
|
||||
disabled: true
|
||||
gocritic:
|
||||
disabled-checks:
|
||||
- ifElseChain
|
||||
govet:
|
||||
settings:
|
||||
printf:
|
||||
funcs:
|
||||
- common.Errorf
|
||||
forbidigo:
|
||||
forbid:
|
||||
- 'fmt\.Errorf(# Please use errors\.Wrap\|Wrapf\|Errorf instead)?'
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
{
|
||||
".": "0.30.0-rc.2"
|
||||
}
|
||||
Vendored
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"go.lintOnSave": "workspace",
|
||||
"go.lintTool": "golangci-lint"
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
Repository instructions for AI coding agents. Keep this file short, concrete, and tied to commands that actually work in this
|
||||
repo. If a fact here conflicts with source files or CI config, trust the source file and update this guide.
|
||||
|
||||
## Project Snapshot
|
||||
|
||||
Memos is a self-hosted note-taking app.
|
||||
|
||||
- Backend: Go 1.26.2, Echo v5, Connect RPC, gRPC-Gateway, Protocol Buffers.
|
||||
- Frontend: React 19, TypeScript 6, Vite 8, Tailwind CSS v4, React Query v5.
|
||||
- Storage: SQLite, MySQL, PostgreSQL.
|
||||
- Generated API outputs: `proto/gen/` for Go/OpenAPI, `web/src/types/proto/` for TypeScript.
|
||||
|
||||
## Working Rules
|
||||
|
||||
- Read relevant code before editing; prefer local patterns over new abstractions.
|
||||
- Keep diffs scoped. Do not do repo-wide cleanup, dependency churn, or generated-file rewrites unless the task requires it.
|
||||
- Do not hand-edit generated proto outputs. Change `.proto` files, then run `buf generate`.
|
||||
- Add migrations for all database drivers when schema changes, and update each driver's `LATEST.sql`.
|
||||
- Add public API endpoints to `server/router/api/v1/acl_config.go`.
|
||||
- Ask before adding heavy dependencies, changing auth/token behavior, or altering Docker/release workflows.
|
||||
|
||||
## Commands
|
||||
|
||||
Run from the repository root unless a command starts with `cd`.
|
||||
|
||||
```bash
|
||||
# Backend
|
||||
go run ./cmd/memos --port 8081 # Start backend dev server
|
||||
go test ./... # Run all Go tests
|
||||
go test -v ./store/... # Store tests, including DB drivers via TestContainers
|
||||
go test -v -race ./server/... # Server tests with race detector
|
||||
go test -v -race ./internal/... # Internal package tests with race detector
|
||||
go test -v -run TestFoo ./pkg/... # Run matching Go tests
|
||||
go mod tidy -go=1.26.2 # Match CI tidy check
|
||||
golangci-lint run # Go lint, config: .golangci.yaml
|
||||
golangci-lint run --fix # Auto-fix lint, including goimports
|
||||
|
||||
# Frontend
|
||||
cd web && pnpm install # Install dependencies
|
||||
cd web && pnpm dev # Dev server on :3001, proxying API to :8081
|
||||
cd web && pnpm lint # Type check + Biome lint
|
||||
cd web && pnpm test # Vitest unit tests
|
||||
cd web && pnpm build # Production build
|
||||
cd web && pnpm release # Build SPA into server/router/frontend/dist
|
||||
|
||||
# Protocol Buffers
|
||||
cd proto && buf generate # Regenerate Go + TypeScript + OpenAPI
|
||||
cd proto && buf lint # Lint proto files
|
||||
cd proto && buf format -w # Format proto files
|
||||
```
|
||||
|
||||
## Code Map
|
||||
|
||||
| Path | Purpose |
|
||||
| --- | --- |
|
||||
| `cmd/memos/main.go` | Cobra/Viper CLI setup and server startup |
|
||||
| `server/server.go` | Echo HTTP server and background runner wiring |
|
||||
| `server/auth/` | JWT access tokens, refresh tokens, PAT handling |
|
||||
| `server/router/api/v1/` | Connect/gRPC-Gateway services, ACL config, SSE hub |
|
||||
| `server/router/frontend/` | Static SPA serving |
|
||||
| `server/router/fileserver/` | Native HTTP file serving, thumbnails, range requests |
|
||||
| `server/runner/` | Background memo processing and S3 presign refresh |
|
||||
| `store/` | Store facade, cache, migrations, driver interface |
|
||||
| `store/db/{sqlite,mysql,postgres}/` | Database-specific drivers and SQL |
|
||||
| `proto/api/v1/` | Public API service definitions |
|
||||
| `proto/store/` | Internal storage proto messages |
|
||||
| `internal/` | App-private packages: scheduler, cron, email, CEL filter, markdown, idp, S3 |
|
||||
| `web/src/connect.ts` | Connect RPC clients, auth interceptor, access-token refresh |
|
||||
| `web/src/auth-state.ts` | Token storage and BroadcastChannel cross-tab sync |
|
||||
| `web/src/hooks/` | React Query hooks for server state |
|
||||
| `web/src/contexts/` | React context for client/UI state |
|
||||
| `web/src/components/` | Radix/Tailwind UI components and feature components |
|
||||
| `web/src/themes/` | CSS themes using OKLch color tokens |
|
||||
|
||||
## Change Routing
|
||||
|
||||
| Change | Update | Verify |
|
||||
| --- | --- | --- |
|
||||
| Go service or router behavior | Service code under `server/`, tests near package | `go test -v -race ./server/...` |
|
||||
| Store or migration behavior | `store/`, all three DB driver migrations, `LATEST.sql` | `go test -v ./store/...` |
|
||||
| Internal package logic | Relevant `internal/` package tests | `go test -v -race ./internal/...` |
|
||||
| Frontend behavior | Components/hooks/contexts under `web/src/` | `cd web && pnpm lint && pnpm test` |
|
||||
| Frontend production output | Vite config or release-sensitive UI | `cd web && pnpm build` or `pnpm release` |
|
||||
| Proto API | `.proto` source plus generated outputs | `cd proto && buf generate && buf lint` |
|
||||
| Public unauthenticated route | `server/router/api/v1/acl_config.go` | Targeted server test or manual route check |
|
||||
|
||||
## Go Conventions
|
||||
|
||||
- Wrap errors with `errors.Wrap(err, "context")` from `github.com/pkg/errors`; do not use `fmt.Errorf`.
|
||||
- Return service errors with `status.Errorf(codes.X, "message")`.
|
||||
- Keep imports grouped as stdlib, third-party, then `github.com/usememos/memos`; goimports is run by golangci-lint.
|
||||
- Add doc comments for exported identifiers; godot enforces exported comment punctuation.
|
||||
- Avoid package-level mutable state unless the surrounding package already uses that pattern.
|
||||
|
||||
## Frontend Conventions
|
||||
|
||||
- Use `@/` for absolute imports.
|
||||
- Follow Biome formatting: 2-space indent, double quotes, semicolons, 140-character line width.
|
||||
- Put server data in React Query hooks under `web/src/hooks/`; keep UI-only state in contexts or component state.
|
||||
- Use Tailwind CSS v4 utilities, `cn()` for class merging, and CVA for variants.
|
||||
- Reuse Radix primitives and existing components before adding new UI primitives.
|
||||
- Keep generated proto TypeScript under `web/src/types/proto/` out of manual edits and Biome rewrites.
|
||||
|
||||
## Database And Proto Rules
|
||||
|
||||
- Schema changes require SQLite, MySQL, and PostgreSQL migrations plus `LATEST.sql` updates.
|
||||
- Fresh-install SQL and incremental migrations must stay equivalent.
|
||||
- Proto field changes must preserve compatibility unless the task explicitly allows a breaking API change.
|
||||
- Regenerate after proto edits and include both Go/OpenAPI and TypeScript generated outputs.
|
||||
|
||||
## Verification Policy
|
||||
|
||||
- Run the narrowest relevant checks while iterating.
|
||||
- Before finishing, run the checks that match the changed surface from "Change Routing".
|
||||
- For docs-only changes, `git diff --check` is sufficient unless the docs include runnable examples that should be tested.
|
||||
- If a required check cannot run locally, report the reason and the exact command that remains.
|
||||
|
||||
## CI Reference
|
||||
|
||||
- Backend CI: Go 1.26.2, `go mod tidy -go=1.26.2`, golangci-lint v2.11.3, test groups `store`, `server`, `internal`, `other`.
|
||||
- Frontend CI: Node 24, pnpm 11.0.1, `pnpm lint`, `pnpm test`, `pnpm build`.
|
||||
- Proto CI: `buf lint` and `buf format` check.
|
||||
- Docker: `scripts/Dockerfile`, Alpine 3.21 runtime, non-root user, port 5230, multi-arch amd64/arm64/arm/v7.
|
||||
-230
@@ -1,230 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
## [0.30.0-rc.2](https://github.com/usememos/memos/compare/v0.30.0-rc.1...v0.30.0-rc.2) (2026-07-20)
|
||||
|
||||
|
||||
### Highlights
|
||||
|
||||
* **Memo detail and navigation:** Redesigned the memo detail sidebar as a property rail with quick actions, editable visibility, memo metadata, attachments, relations, and a scroll-aware outline. Feed scroll position is now restored when navigating back from another page. ([d4b5a16](https://github.com/usememos/memos/commit/d4b5a1695b9acf4da3ef72f600789a4a0f90e4d5)) ([e497895](https://github.com/usememos/memos/commit/e497895bf3b72cdae92e5bab6cd5b691c8eedc15))
|
||||
* **Settings and access tokens:** Refreshed settings navigation and expanded the access-token page with practical guidance, a copyable API example, safety notes, and last-used timestamps. The About page and project documentation now surface the Memos Web Clipper. ([715306e](https://github.com/usememos/memos/commit/715306ea66306a91765d3f7c3ede8bc815d2def5)) ([06ecef3](https://github.com/usememos/memos/commit/06ecef33bc7e6bafc263b2dee71ffd290ab3dc42)) ([f946974](https://github.com/usememos/memos/commit/f94697496b60f5fe590c2ffa19e3f0c93b749af4))
|
||||
* **Faster feeds and Explore:** Reduced initial work and bandwidth by caching fingerprinted assets, deferring media and rich renderers until needed, rendering feed content near the viewport, and sharing user and memo lookups across creators, reactions, comments, mentions, and relations. ([88c6ee8](https://github.com/usememos/memos/commit/88c6ee8ebcc341aa1d2e2ae38638a518f4745de1)) ([6c1055f](https://github.com/usememos/memos/commit/6c1055f483c92c2a95193bea07110b6775359651)) ([249b331](https://github.com/usememos/memos/commit/249b331596fd1c9ceceb4e72da8e422499be683f))
|
||||
* **Demo experience:** Rebuilt the demo data around representative public and protected memos that showcase nested tags, references, comments, mentions, reactions, locations, attachments, and rich Markdown. ([469c995](https://github.com/usememos/memos/commit/469c995cc04b5e7de259156d28c58b948e85d111))
|
||||
|
||||
### Fixes and polish
|
||||
|
||||
* **API compatibility and errors:** Aligned new resource IDs and annotations with AIP conventions while preserving legacy UUID-based identifiers, corrected canonical user-stat resource names, and removed internal RPC prefixes from user-facing error messages. ([84776cc](https://github.com/usememos/memos/commit/84776cc106745278371e5a8c8d43abca6d38e8b5)) ([b7d5d09](https://github.com/usememos/memos/commit/b7d5d09f8a7445076811d6f53b98d3445ce3afac)) ([0bfeb91](https://github.com/usememos/memos/commit/0bfeb91d50fdd4bafd15c13bb02b3e2810d4b2e4))
|
||||
* **Interface:** Corrected radio-group indicator alignment and positioned tooltip arrows outside their content panels. ([#6097](https://github.com/usememos/memos/issues/6097)) ([2d01420](https://github.com/usememos/memos/commit/2d01420c23372d5708904ec78e03485edb44d49b)) ([#6095](https://github.com/usememos/memos/issues/6095)) ([773e56e](https://github.com/usememos/memos/commit/773e56e537657508af4e357f9c8845af4d4f15e8))
|
||||
|
||||
## [0.30.0-rc.1](https://github.com/usememos/memos/compare/v0.29.1...v0.30.0-rc.1) (2026-07-14)
|
||||
|
||||
|
||||
### ⚠ BREAKING CHANGES
|
||||
|
||||
* **Access control:** Instances without `--instance-url` (or `MEMOS_INSTANCE_URL`) now run in private mode. Anonymous visitors are sent to sign-in and anonymous API access is limited to setup, authentication, and shared-memo routes. Set the instance URL to retain the previous public behavior. ([d1cef7a](https://github.com/usememos/memos/commit/d1cef7a9ab23e8c93f76b2f32661c720c733536d))
|
||||
* **Filters:** `now()` has been replaced by the `now` timestamp variable, and time fields now use CEL timestamps. Update saved shortcuts to use expressions such as `created_ts >= now - duration("24h")` or `timestamp(<epoch>)` instead of comparing time fields with bare epoch values. ([26f4b73](https://github.com/usememos/memos/commit/26f4b73cb9a996f9232daad2b5daa34360742697))
|
||||
* **MCP:** The MCP server is now a stateless, tools-only endpoint generated from the OpenAPI schema. The previous prompts, resources, tool-filtering headers and route aliases, and unprefixed tool names have been removed. Clients that used them must switch to `/mcp` and the new service-prefixed tool names. ([#6026](https://github.com/usememos/memos/issues/6026))
|
||||
|
||||
### Highlights
|
||||
|
||||
* **Markdown editor:** Rebuilt the editor as a single CodeMirror 6 decorated-source editor. Markdown is preserved verbatim while headings, formatting, tags, and mentions are styled in place. The editor also adds tag completion, list indentation, and a toggleable formatting toolbar with headings, lists, task lists, code blocks, bold, italic, strikethrough, and inline code. ([5a73d7d](https://github.com/usememos/memos/commit/5a73d7d3e56f853a965207ac709e8f5e84918959)) ([e3c231f](https://github.com/usememos/memos/commit/e3c231fcac5691f9590a5d685bdebec23b73be35))
|
||||
* **MCP:** Replaced the hand-written MCP implementation with a curated OpenAPI-driven tool surface that reuses the public API's authentication and authorization. The final toolset includes memo, comment, relation, reaction, shortcut, identity, and attachment operations, including attachment uploads. ([777d227](https://github.com/usememos/memos/commit/777d227eb992c4feccd5d78c781cf8e9094e38de)) ([0e1d821](https://github.com/usememos/memos/commit/0e1d821fb84310feff40b90d899403d45bced596))
|
||||
* **Deployment configuration:** Identity providers and supported instance settings can be supplied as validated JSON files under `/etc/secrets`. File-backed settings act as runtime overrides and cannot be changed through the UI or API until the files are removed. ([0038295](https://github.com/usememos/memos/commit/0038295bbc772b38425b6c7f9ca814e4d1e44260))
|
||||
* **Memo feeds:** Added persisted controls for one, two, three, or automatically fitted columns; compact mode; and link previews. Multi-column feeds stay balanced as cards resize, while newly created memos remain easy to find above pinned content. ([177d65a](https://github.com/usememos/memos/commit/177d65a90e321fa975a7cf19bd3c075143af5c10)) ([e3e4ae1](https://github.com/usememos/memos/commit/e3e4ae10512f514f71729779b5096d0d591c8cf4))
|
||||
* **Webhooks:** Added Standard Webhooks HMAC-SHA256 signing secrets, webhook editing, and a signing-status indicator. Secrets are generated server-side, shown after creation, and can be revealed later from the edit dialog; malformed secrets now fail validation. ([#6013](https://github.com/usememos/memos/issues/6013)) ([#6027](https://github.com/usememos/memos/issues/6027))
|
||||
* **Filters:** Expanded CEL shortcuts with string matching, regular expressions, collection predicates, timestamp accessors, set operations, and an untagged-memos example. ([f0e4a56](https://github.com/usememos/memos/commit/f0e4a5624f4371e08fb0c41510891c9f8cce6ba0)) ([b787bfa](https://github.com/usememos/memos/commit/b787bfa75f74d8ee5ec271bd308c2e5a0389ed35))
|
||||
* **Tag settings:** Tag colors and content-blur rules are now per-user. Existing instance tag settings are copied to current users during migration. ([#6017](https://github.com/usememos/memos/issues/6017))
|
||||
* **Internationalization:** Added a searchable locale picker, expanded European locale coverage, and completed missing Japanese and Simplified Chinese translations. ([4183985](https://github.com/usememos/memos/commit/418398587cef90151745ba0dbc51cef4762045ca)) ([a47d049](https://github.com/usememos/memos/commit/a47d04954e75c54491138601c72f1c0cbe140f3c))
|
||||
* **Interface:** Refreshed the sign-in, sign-up, and About pages and aligned common controls across the app. Compact rendering is now opt-in in single-column feeds, so full memo content is shown by default. ([564da94](https://github.com/usememos/memos/commit/564da949cb87491ca8e493d2f7aedcd2ef9545a1)) ([cafa56f](https://github.com/usememos/memos/commit/cafa56f1a8c9091062a4b220284e8ca697b050e8))
|
||||
* **Markdown and storage:** Added GFM footnote rendering and navigation, plus an `insecure_skip_tls_verify` option for S3-compatible storage with self-signed certificates. ([1020060](https://github.com/usememos/memos/commit/10200606db24e3d70fb8efefee99c7b0a369ddea)) ([#6039](https://github.com/usememos/memos/issues/6039))
|
||||
|
||||
### Fixes and polish
|
||||
|
||||
* **Authentication and setup:** Stabilized SSO option loading, added OAuth client-auth auto-detection, and distinguished fresh instances from populated instances that no longer have an admin. ([6c17e87](https://github.com/usememos/memos/commit/6c17e87cf61cd9c62ea4ef4e25d4adb5603ffc88)) ([96cb653](https://github.com/usememos/memos/commit/96cb65320b714ef8d7993ddb7c9182350ee9da4c))
|
||||
* **Editor:** Improved formatting toggles and keyboard shortcuts, file paste and drop handling, focus-mode layout, caret behavior in empty space, and CodeMirror spacing. ([c349c15](https://github.com/usememos/memos/commit/c349c1549e49950e3171b7de28faba5d89249cfa)) ([#6076](https://github.com/usememos/memos/issues/6076)) ([#6093](https://github.com/usememos/memos/issues/6093))
|
||||
* **Memos and comments:** Preserved expanded task-list state, loaded every comment page, and included the parent memo relation in comment webhook payloads. ([ecbe2ab](https://github.com/usememos/memos/commit/ecbe2ab7977fcd3521aeae1226e816a9dc2a6a40)) ([#6083](https://github.com/usememos/memos/issues/6083))
|
||||
* **Tags:** Tags inside links are no longer parsed as memo tags, literal tags can be escaped with a backslash, and tag names support Unicode combining marks. ([a50ce09](https://github.com/usememos/memos/commit/a50ce09e8159836aed1a97fa717488283378200f)) ([#6051](https://github.com/usememos/memos/issues/6051))
|
||||
* **API and security:** Cross-origin API requests now work with bearer tokens while cookie authentication remains same-origin, and `ListUsers` now returns consistent paginated results. ([385fa22](https://github.com/usememos/memos/commit/385fa22056c51a42a0eb99fd08a8876009c2c52d)) ([4bc3928](https://github.com/usememos/memos/commit/4bc39280290b0f2612ff52285689854b6cf1344b))
|
||||
* **Interface:** Replaced mismatched memo skeletons with a delayed spinner, improved mobile control spacing, and truncated long location labels without hiding the full address. ([#6047](https://github.com/usememos/memos/issues/6047)) ([3b601b8](https://github.com/usememos/memos/commit/3b601b841670ece4a2832341a99ccc74124a745e))
|
||||
* **Container:** Prevented the entrypoint from restarting indefinitely when `MEMOS_UID=0`. ([#6061](https://github.com/usememos/memos/issues/6061))
|
||||
|
||||
## [0.29.1](https://github.com/usememos/memos/compare/v0.29.0...v0.29.1) (2026-06-04)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **markdown:** keep task item content in one grid column ([7c3bff4](https://github.com/usememos/memos/commit/7c3bff4e98223fb99a5d49f093e3cd57dd140ba4))
|
||||
* support <meta name=description> in link previews ([#6000](https://github.com/usememos/memos/issues/6000)) ([e8d32e8](https://github.com/usememos/memos/commit/e8d32e87d1d6e4927250ad5794ba5965e0260153))
|
||||
* **web:** render video attachment posters on mobile ([0e2a9a9](https://github.com/usememos/memos/commit/0e2a9a9c0ce0e2da63535210c553b3ae2f79b239))
|
||||
|
||||
## [0.29.0](https://github.com/usememos/memos/compare/v0.28.0...v0.29.0) (2026-05-27)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **about:** add about page with bird sprites ([411ba7b](https://github.com/usememos/memos/commit/411ba7b34c0c23ad217a7897835097887a28a036))
|
||||
* **activity-calendar:** aggregate by ViewContext.timeBasis ([8daef1d](https://github.com/usememos/memos/commit/8daef1dc89544512fff78fd64d8216b2babc2e42))
|
||||
* add <Placeholder> component with ASCII bird states ([#5949](https://github.com/usememos/memos/issues/5949)) ([8c16ffa](https://github.com/usememos/memos/commit/8c16ffa1f1410f9413cb57ebe251b4b604f9aeff))
|
||||
* add configurable `--log-level` flag ([#5934](https://github.com/usememos/memos/issues/5934)) ([f1e2a06](https://github.com/usememos/memos/commit/f1e2a06b46c7f45ba6562c308d2703fdfb4067b0))
|
||||
* add dedicated shortcuts page ([#5942](https://github.com/usememos/memos/issues/5942)) ([1df6479](https://github.com/usememos/memos/commit/1df6479443c334732b862d60a0f76e61f00ebb89))
|
||||
* add link metadata endpoints ([9c5c604](https://github.com/usememos/memos/commit/9c5c604944428d3f682f35c9d05a87c1d4c3152f))
|
||||
* **frontend:** add pixel bird tilemaps ([cf55f11](https://github.com/usememos/memos/commit/cf55f1107273c928e7c2d6bc26df5f3efea26458))
|
||||
* **memo:** add task list quick actions ([#5983](https://github.com/usememos/memos/issues/5983)) ([648b3bd](https://github.com/usememos/memos/commit/648b3bd812e5bf2b1abb9f8b3529efefe4734e7c))
|
||||
* **memo:** create memos on the selected calendar date ([#5925](https://github.com/usememos/memos/issues/5925)) ([ef55013](https://github.com/usememos/memos/commit/ef55013418d68e2d6a24a6d02a0365c1ffff03d7))
|
||||
* **notification:** add smtp email settings ([cd4f28a](https://github.com/usememos/memos/commit/cd4f28ae1058a125e245ea0a6eae16aea71b8f9d))
|
||||
* **placeholder:** add woodpecker tilemap ([638e4f3](https://github.com/usememos/memos/commit/638e4f398e90c556f70af150a79538312c8fc760))
|
||||
* render link metadata cards ([0bc5669](https://github.com/usememos/memos/commit/0bc56694b0ca347ab1eb083f62997a22007b763d))
|
||||
* **stats:** admin instance resource statistics ([ea0625d](https://github.com/usememos/memos/commit/ea0625da45a419e08158ab4102051f3fe5e59b87))
|
||||
* **stats:** support filtered all-user stats ([88ac3ec](https://github.com/usememos/memos/commit/88ac3ec31ee3e808db82663c04a31cf730d1221e))
|
||||
* **transcription:** explicit STT settings with provider, model, prompt ([#5926](https://github.com/usememos/memos/issues/5926)) ([238f27d](https://github.com/usememos/memos/commit/238f27dea149492a78a8994470b0aac55fece78d))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* avoid update event on memo create attachments ([#5961](https://github.com/usememos/memos/issues/5961)) ([3c3382a](https://github.com/usememos/memos/commit/3c3382a3c652bed3b58058931ff3cbf1d875b9a8))
|
||||
* delete user cleanup ([#5981](https://github.com/usememos/memos/issues/5981)) ([e53b7d9](https://github.com/usememos/memos/commit/e53b7d96e70965529ba3b04ea720ea6966f77b60))
|
||||
* **editor:** wrap selected text when pasting URL ([e0bb3a2](https://github.com/usememos/memos/commit/e0bb3a2e684e932309df2f9fb0ff774ce8e6b692))
|
||||
* **fileserver:** preserve HDR image metadata in thumbnails ([c724232](https://github.com/usememos/memos/commit/c7242324a18962e453e7f2a0309a7c07b358bdbe))
|
||||
* **frontend:** correct static cache headers ([084f40b](https://github.com/usememos/memos/commit/084f40bc9e5922ff2d41c08dce837b96696437aa))
|
||||
* **frontend:** use correct url path for memos in sitemap.xml ([#5921](https://github.com/usememos/memos/issues/5921)) ([603781f](https://github.com/usememos/memos/commit/603781f792b2603fbda93146988a64a93c92d760))
|
||||
* **httpgetter:** prevent DNS rebinding in link metadata fetch ([078488c](https://github.com/usememos/memos/commit/078488ca818626fbcab69bdfb4b93b58dca6b6eb))
|
||||
* **markdown:** align list items with checkboxes ([e008b1a](https://github.com/usememos/memos/commit/e008b1a23c77945eff87707eb15578d4d6d74e4c))
|
||||
* **memo:** enforce parent visibility for comments ([4a1e401](https://github.com/usememos/memos/commit/4a1e401bd99c7850ac48e9693b25c9ebeecfb042))
|
||||
* **s3presign:** preserve motion media payload ([7f1f53f](https://github.com/usememos/memos/commit/7f1f53ffc417962f6930d1e6e3783eacd911003d))
|
||||
* **security:** enforce attachment ownership on memo updates ([35bf761](https://github.com/usememos/memos/commit/35bf761b8c2c3f155bdc06e9b373b6076116a725))
|
||||
* **sse:** stream initial response and refresh tokens ([21303e8](https://github.com/usememos/memos/commit/21303e879d2c7a857306206d2cbdae83f61983e6))
|
||||
* **user:** omit internal settings from list responses ([#5917](https://github.com/usememos/memos/issues/5917)) ([1df3fe7](https://github.com/usememos/memos/commit/1df3fe79559ccf94b6c71e9ffb58e870ed43820d))
|
||||
* **web:** sync avatar changes immediately after profile update ([#5903](https://github.com/usememos/memos/issues/5903)) ([328396a](https://github.com/usememos/memos/commit/328396a97f2219592032e8418c383e0ab7edaca6))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* lazy load heavy first-screen dependencies ([#5947](https://github.com/usememos/memos/issues/5947)) ([a6024ee](https://github.com/usememos/memos/commit/a6024eebf129c4b8fb65ee0cba09c6657274bfde))
|
||||
|
||||
## [0.28.0](https://github.com/usememos/memos/compare/v0.27.1...v0.28.0) (2026-04-27)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **auth:** add SSO user identity linkage ([#5883](https://github.com/usememos/memos/issues/5883)) ([d688914](https://github.com/usememos/memos/commit/d688914b2864791eeadbf21c882608632875f17c))
|
||||
* **memos:** choose created or updated time for memos ([#5894](https://github.com/usememos/memos/issues/5894)) ([c268551](https://github.com/usememos/memos/commit/c268551a16929a2cbea6891951feff91926bba59))
|
||||
* redesign account and SSO management ([#5886](https://github.com/usememos/memos/issues/5886)) ([ee17998](https://github.com/usememos/memos/commit/ee1799851e88674a6920c7a56d93428fcf95e662))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **auth:** harden authorization and username validation ([#5890](https://github.com/usememos/memos/issues/5890)) ([0fb83a7](https://github.com/usememos/memos/commit/0fb83a745dd5057ade45a3caad2c444af2239113))
|
||||
* disable modal prop on DropdownMenu to prevent scroll disappearing ([#5861](https://github.com/usememos/memos/issues/5861)) ([d98f665](https://github.com/usememos/memos/commit/d98f6659190b8d1a8252e64549d9120d85e05d33))
|
||||
* fix legacy username auth flows ([#5885](https://github.com/usememos/memos/issues/5885)) ([30c0611](https://github.com/usememos/memos/commit/30c0611a82f9254952a74650095105254f2940e4))
|
||||
* **markdown:** split mixed task and bullet lists ([e2c6084](https://github.com/usememos/memos/commit/e2c60845eaff9a78b8d8eb3ccc9a067ef5690976))
|
||||
* reduce list memo query overhead ([#5880](https://github.com/usememos/memos/issues/5880)) ([5063804](https://github.com/usememos/memos/commit/50638040f618b02b0c6d010e1d41554c75067517))
|
||||
* **web:** preserve task checkbox state ([#5867](https://github.com/usememos/memos/issues/5867)) ([b5863d7](https://github.com/usememos/memos/commit/b5863d76be3cfbf3e0f8237d8e762122b5a0a679))
|
||||
|
||||
## [0.27.1](https://github.com/usememos/memos/compare/v0.27.0...v0.27.1) (2026-04-19)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* mixed-case user resource names ([#5853](https://github.com/usememos/memos/issues/5853)) ([01be01f](https://github.com/usememos/memos/commit/01be01f4b7676af41bdd1758b1e9b096aa922546))
|
||||
* **release:** inject build version into artifacts ([f8a304b](https://github.com/usememos/memos/commit/f8a304bae33086320b39095d631288156eec4249))
|
||||
* user resource names can be uuidv4 from idp sub claim ([#5856](https://github.com/usememos/memos/issues/5856)) ([bbded58](https://github.com/usememos/memos/commit/bbded584ce85a856d863485768e08b53adec7244))
|
||||
|
||||
## [0.27.0](https://github.com/usememos/memos/compare/v0.26.2...v0.27.0) (2026-04-18)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --allow-private-webhooks flag to bypass SSRF protection ([#5694](https://github.com/usememos/memos/issues/5694)) ([cd5816c](https://github.com/usememos/memos/commit/cd5816c428931f56561f7d48b649a33858812539))
|
||||
* add blur_content attribute to tag metadata settings ([#5767](https://github.com/usememos/memos/issues/5767)) ([45b2153](https://github.com/usememos/memos/commit/45b21530d9ebbc730e77bcbcf407f4c295b52e93))
|
||||
* add Gemini transcription provider ([#5830](https://github.com/usememos/memos/issues/5830)) ([d87539a](https://github.com/usememos/memos/commit/d87539a1e1261590ab355220e33fa0ae00bd9431))
|
||||
* add GitHub release installer and release workflow ([1ed542c](https://github.com/usememos/memos/commit/1ed542c21db421e9aa0cc47417a92c6bd103e257))
|
||||
* add live refresh via Server-Sent Events (SSE) with visual indicator ([#5638](https://github.com/usememos/memos/issues/5638)) ([ea0892a](https://github.com/usememos/memos/commit/ea0892a8b26d8806a093e3a942562dfc22794f68))
|
||||
* add MCP server with PAT authentication ([47d9414](https://github.com/usememos/memos/commit/47d9414702dc18966af385352b960bfe451511b7))
|
||||
* add outline navigation to memo detail sidebar ([#5771](https://github.com/usememos/memos/issues/5771)) ([6b30579](https://github.com/usememos/memos/commit/6b3057990396f9cb6f21706c2e99dddebc35ffd5))
|
||||
* **ai:** add BYOK audio transcription ([#5832](https://github.com/usememos/memos/issues/5832)) ([101704c](https://github.com/usememos/memos/commit/101704c8eac17c7f34508d8db6c53bc972061cdb))
|
||||
* **ai:** add instance AI providers and transcription ([#5829](https://github.com/usememos/memos/issues/5829)) ([83ed32f](https://github.com/usememos/memos/commit/83ed32f1195841d2d6c057c3e00086a3147879a2))
|
||||
* **attachments:** add Live Photo and Motion Photo support ([#5810](https://github.com/usememos/memos/issues/5810)) ([4b4e719](https://github.com/usememos/memos/commit/4b4e719470184e49cd62084b1aa53c9a777a9fec))
|
||||
* **cli:** add version subcommand ([#5731](https://github.com/usememos/memos/issues/5731)) ([0ba4c0f](https://github.com/usememos/memos/commit/0ba4c0f397c532a5a314f35ddeb3ce3633e84d42))
|
||||
* **editor:** add voice note recording to the memo composer ([#5801](https://github.com/usememos/memos/issues/5801)) ([c0d5854](https://github.com/usememos/memos/commit/c0d5854f678f357cf4054d448e0311f2ee90f8ac))
|
||||
* extract title from first H1 heading into memo property ([#5726](https://github.com/usememos/memos/issues/5726)) ([1e688b2](https://github.com/usememos/memos/commit/1e688b2a5d78c2972f4b0078a7b033dae5d1cbf3))
|
||||
* **i18n:** update sse connect label in Chinese ([#5732](https://github.com/usememos/memos/issues/5732)) ([89c6902](https://github.com/usememos/memos/commit/89c69028953fa33dbb6e0e257d764636c0fc2d09))
|
||||
* **instance:** add canonical tag metadata setting ([#5736](https://github.com/usememos/memos/issues/5736)) ([65d14fb](https://github.com/usememos/memos/commit/65d14fbb63053fc1fbf1f63881c81adbc9d01671))
|
||||
* **instance:** add notification transport setting ([#5737](https://github.com/usememos/memos/issues/5737)) ([a249d06](https://github.com/usememos/memos/commit/a249d06e2e0f33e0e3fea34471a891352dacc466))
|
||||
* **mcp:** enhance MCP server with full capabilities and new tools ([#5720](https://github.com/usememos/memos/issues/5720)) ([b8e9ee2](https://github.com/usememos/memos/commit/b8e9ee2b26a59e1b19aab6db3ca39656fc18785a))
|
||||
* **mcp:** harden tool exposure and side effects ([#5850](https://github.com/usememos/memos/issues/5850)) ([583c3d2](https://github.com/usememos/memos/commit/583c3d24f4d785faa5e034c1b88ed90eda119baa))
|
||||
* **mcp:** refactor MCP server to standard protocol structure ([803d488](https://github.com/usememos/memos/commit/803d488a5f8f55477cd3ad4cc4cf0fac98901dd3))
|
||||
* **memo-editor:** add compact live waveform recorder panel ([#5817](https://github.com/usememos/memos/issues/5817)) ([e51985a](https://github.com/usememos/memos/commit/e51985a29ffeeecd318b2fb793358ed45bb0eff4))
|
||||
* **memo-preview:** support comment metadata in previews ([#5768](https://github.com/usememos/memos/issues/5768)) ([e176b28](https://github.com/usememos/memos/commit/e176b28c801cc3bb3718208a96a7d64c36620d0f))
|
||||
* **memo:** add image sharing in detail view ([38fc22b](https://github.com/usememos/memos/commit/38fc22b7541b8a9ddcd848cf77054fcf844eb87f))
|
||||
* **memo:** add share links for private memos ([#5742](https://github.com/usememos/memos/issues/5742)) ([3f3133d](https://github.com/usememos/memos/commit/3f3133d6e2f404061e147f9dd2424680dc0303a3))
|
||||
* **mentions:** add memo mention parsing, notifications, and rendering ([#5811](https://github.com/usememos/memos/issues/5811)) ([24fc8ab](https://github.com/usememos/memos/commit/24fc8ab8ca68d092e7b12a5d51c48ca9420f72b5))
|
||||
* replace auto-increment ID with UID for identity provider resource names ([#5687](https://github.com/usememos/memos/issues/5687)) ([92d937b](https://github.com/usememos/memos/commit/92d937b1aa87365152edc005a3b736e6bf9b9e45))
|
||||
* show inline comment preview in list view ([3a5d3c8](https://github.com/usememos/memos/commit/3a5d3c8ff92ae5c24559dc4574300d6475820a87))
|
||||
* **store:** change default storage type to local filesystem ([78efa68](https://github.com/usememos/memos/commit/78efa6802e2fd96c981e86c5b658fb4b5c05091f))
|
||||
* treat tag setting keys as anchored regex patterns ([#5759](https://github.com/usememos/memos/issues/5759)) ([9e04049](https://github.com/usememos/memos/commit/9e04049632e63f1cf53535773f170c1aa5af7168))
|
||||
* **ui:** allow navigating between images with arrows in preview dialog ([#5669](https://github.com/usememos/memos/issues/5669)) ([104d2ec](https://github.com/usememos/memos/commit/104d2ec0a6973e983c46724165fdc0916515ad16))
|
||||
* **user:** add per-user tag metadata settings ([#5735](https://github.com/usememos/memos/issues/5735)) ([330291d](https://github.com/usememos/memos/commit/330291d4d9fa95649e940001898929b4f6f0367e))
|
||||
* **web:** add demo mode banner ([#5836](https://github.com/usememos/memos/issues/5836)) ([35504cc](https://github.com/usememos/memos/commit/35504cc8bd6f9921291fb565633d549388a3f948))
|
||||
* **webhook:** dispatch webhook on memo comment creation ([7c1defb](https://github.com/usememos/memos/commit/7c1defba01fbc91cc81e1a0841cecb6738056db7))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* access token refresh on web app ([#5681](https://github.com/usememos/memos/issues/5681)) ([3010f10](https://github.com/usememos/memos/commit/3010f10eafb49af5aadd71a94b3ef7a6ec71f617))
|
||||
* add unix socket file permission setting (755 -> 660) ([#5849](https://github.com/usememos/memos/issues/5849)) ([0fc1dab](https://github.com/usememos/memos/commit/0fc1dab28b33f7fdbe9a21cd3bd3affe75dd7f19))
|
||||
* **api:** appease image size lint ([ff6389a](https://github.com/usememos/memos/commit/ff6389a5ef73772fcb5c132bf1cb169fb4744c00))
|
||||
* **api:** improve SSE hub design and fix double-broadcast on comments ([c53677f](https://github.com/usememos/memos/commit/c53677fcba202a8eed35c8ecdfaacea5152b8691))
|
||||
* **api:** make credentials write-only and restrict sensitive settings to admins ([9d3a74b](https://github.com/usememos/memos/commit/9d3a74bcccf934aa2fa95ee240b56cc3b3a25776))
|
||||
* **api:** reduce memory pressure in backend paths ([c456637](https://github.com/usememos/memos/commit/c45663761d148e94d3e1b30f4810922e7c2571ab))
|
||||
* **api:** remove public activity service ([#5734](https://github.com/usememos/memos/issues/5734)) ([04f239a](https://github.com/usememos/memos/commit/04f239a2fc74f7090c779a6a7b8a08238fd31b88))
|
||||
* **api:** restrict user email exposure to self and admins ([#5784](https://github.com/usememos/memos/issues/5784)) ([a24d420](https://github.com/usememos/memos/commit/a24d4209222814e4a8bedfc0392e91462bc855ad))
|
||||
* **api:** switch user resource names to usernames ([#5779](https://github.com/usememos/memos/issues/5779)) ([acddef1](https://github.com/usememos/memos/commit/acddef1f3dcca44806bcbb85e9dc5cae7daa5285))
|
||||
* **api:** tolerate missing related users in memo conversions ([#5809](https://github.com/usememos/memos/issues/5809)) ([25feef3](https://github.com/usememos/memos/commit/25feef3aadd34bfd474d7e3b685815a6509bc4c6))
|
||||
* **auth:** recover session via refresh cookie when localStorage is empty ([#5748](https://github.com/usememos/memos/issues/5748)) ([551ee1d](https://github.com/usememos/memos/commit/551ee1d81f398abdcaf73a0c2b782c8379b0e3f2))
|
||||
* backend tests action ([065e817](https://github.com/usememos/memos/commit/065e817470b1d3b6b00222525791b85d825c1492))
|
||||
* clear content search filter when selecting shortcut ([#5499](https://github.com/usememos/memos/issues/5499)) ([2c3f9e3](https://github.com/usememos/memos/commit/2c3f9e3bfbe09e5d0028d1525c1df83c03f547c9))
|
||||
* correct typos in comments, error messages, and identifiers ([#5704](https://github.com/usememos/memos/issues/5704)) ([8f43e80](https://github.com/usememos/memos/commit/8f43e8075b62706f99da4005ab04b165b35215f6))
|
||||
* detect legacy installations with empty schema version ([9628d3d](https://github.com/usememos/memos/commit/9628d3de21289e887ffeb6cc4b6710913a0de44e))
|
||||
* **editor:** show newly linked memos when editing a memo with attachments ([026ea92](https://github.com/usememos/memos/commit/026ea92f7b4005c30ca84340c22018740659ab51))
|
||||
* ensure comment divs span full width in MemoDetail ([ce44164](https://github.com/usememos/memos/commit/ce441644af34ffff156f1793c40011c2b82e8d5f))
|
||||
* **fileserver:** render SVG attachment previews ([40fd700](https://github.com/usememos/memos/commit/40fd700fb8be27a8d7a300a2636cf4f366572176))
|
||||
* **filter:** enforce CEL syntax semantics ([0e89407](https://github.com/usememos/memos/commit/0e89407ee91deda87e0df7464a89d26d1e9a88b3))
|
||||
* **frontend:** restore sitemap and robots routes ([fee7fcd](https://github.com/usememos/memos/commit/fee7fcd6608b9d07da1f146c33c9bb8f898f708f))
|
||||
* handle chunk load errors after redeployment with auto-reload ([#5703](https://github.com/usememos/memos/issues/5703)) ([bdd3554](https://github.com/usememos/memos/commit/bdd3554b897246bac5c1e6fd801495c24b6a656e))
|
||||
* harden memo content iframe and HTML sanitization ([7e21b72](https://github.com/usememos/memos/commit/7e21b728b346e80023c93e8986130f83e70584f9))
|
||||
* hide transcribe button without AI provider ([ab53329](https://github.com/usememos/memos/commit/ab5332901fd6626f0a2ae6b99671af9663f68078))
|
||||
* improve image preview dialog and live photo trigger ([aafcc21](https://github.com/usememos/memos/commit/aafcc21ae6f96a313fb372365dd1af8a1a69d1ba))
|
||||
* improve installer compatibility and docs ([f90d9a4](https://github.com/usememos/memos/commit/f90d9a49a78d0764a6c5cf8f6720e69ea1164b25))
|
||||
* improve KaTeX and Mermaid error handling and overflow ([6b37fcc](https://github.com/usememos/memos/commit/6b37fcc01b5dcded6a2ec254fbaf895f7812d072))
|
||||
* include plain URLs and tags in memo snippet generation ([#5688](https://github.com/usememos/memos/issues/5688)) ([3d4f793](https://github.com/usememos/memos/commit/3d4f793f97b567bda823a60b7fe10e09e14499ce))
|
||||
* **lint:** correct goimports struct literal alignment after removing write-only credential fields ([#5794](https://github.com/usememos/memos/issues/5794)) ([9610ed8](https://github.com/usememos/memos/commit/9610ed8fc809b3e7aa4b93e4909634957fa94cf7))
|
||||
* **map:** align dark mode map styling ([7ac9989](https://github.com/usememos/memos/commit/7ac9989d43f3967b466f340da3841290d86b20f1))
|
||||
* **map:** refine Leaflet controls and memo map styling ([894b3eb](https://github.com/usememos/memos/commit/894b3eb045c7d2c2acce847c34a85295c909dcf7))
|
||||
* **markdown:** support height/width attributes on img elements ([737acbb](https://github.com/usememos/memos/commit/737acbba2f5e3dd911da3a77d2d5d1cde1bf8ba1))
|
||||
* **memo-editor:** scope Cmd+Enter save to the active editor ([#5745](https://github.com/usememos/memos/issues/5745)) ([05810e7](https://github.com/usememos/memos/commit/05810e7882cb03d448d9d8fa84bc6ca54eafbed0))
|
||||
* **mysql:** handle CreateMemo custom timestamps with FROM_UNIXTIME ([#5673](https://github.com/usememos/memos/issues/5673)) ([09d73e8](https://github.com/usememos/memos/commit/09d73e8b6e16c43a61767817909a472916c83daa))
|
||||
* normalize attachment MIME types before validation ([c3e7e2c](https://github.com/usememos/memos/commit/c3e7e2c316d01bf857931c38303efc87307f2a19))
|
||||
* preserve draft content when tab is suspended or editor remounts ([9ca7122](https://github.com/usememos/memos/commit/9ca71229a6c0dfad44b76ed06fced46c5bef49c7))
|
||||
* prevent local attachment uploads from overwriting files ([4add9b0](https://github.com/usememos/memos/commit/4add9b04ad5d0b7608c985281f5975dc39f9a572))
|
||||
* prevent stale comment drafts from being restored ([e520b63](https://github.com/usememos/memos/commit/e520b637fd8d0f6331674003abe72f9dfbae8231))
|
||||
* remove duplicate Japanese locale keys ([efeb28c](https://github.com/usememos/memos/commit/efeb28c872a2bd6edd28fa3b68bd0b15af923d50))
|
||||
* render audio attachments as inline players ([#5699](https://github.com/usememos/memos/issues/5699)) ([2ccb98a](https://github.com/usememos/memos/commit/2ccb98a6cbc40544e81eeeca2a2cbbaf8348f45e))
|
||||
* restrict archived memo access to creator only ([#5707](https://github.com/usememos/memos/issues/5707)) ([f4154d0](https://github.com/usememos/memos/commit/f4154d090be3896c8e0dd83440f24953bbeb308c))
|
||||
* **routing:** redirect unauthenticated users to /explore when public visibility is allowed ([98859eb](https://github.com/usememos/memos/commit/98859eb5e5065a7c12ada983b7c38c5af5035bb6))
|
||||
* **server:** close SSE clients during shutdown ([a5ddd5a](https://github.com/usememos/memos/commit/a5ddd5adafef282f00b23e00e145119d4316f24f))
|
||||
* sync html lang attribute with active locale ([#5753](https://github.com/usememos/memos/issues/5753)) ([be00abe](https://github.com/usememos/memos/commit/be00abe852479820b7c574e15e276d70961ab7bd))
|
||||
* tag parsing truncates emojis with variation selectors ([#5496](https://github.com/usememos/memos/issues/5496)) ([3ea6ea3](https://github.com/usememos/memos/commit/3ea6ea3108f58dc34f39fa6f21d761bd2e97732b))
|
||||
* **tags:** allow blur-only tag metadata ([#5800](https://github.com/usememos/memos/issues/5800)) ([1921b57](https://github.com/usememos/memos/commit/1921b57662c2129d179930f71e5c42caf9070a19))
|
||||
* toggle focus mode do not reset editor height ([#5504](https://github.com/usememos/memos/issues/5504)) ([0729779](https://github.com/usememos/memos/commit/0729779e0427c68bdbc5b6b711a5683f66ba9bbc))
|
||||
* **ui:** show comment editor above the comment list ([#5662](https://github.com/usememos/memos/issues/5662)) ([6b0736b](https://github.com/usememos/memos/commit/6b0736b29325ff7698b7395dc761f2ca41e6521c))
|
||||
* **ui:** unify metadata component styles across MemoView and MemoEditor ([664b8c5](https://github.com/usememos/memos/commit/664b8c56290026ba2beb7b75db24c684824b8670))
|
||||
* unify live photo previews around LIVE badge playback ([6b0487d](https://github.com/usememos/memos/commit/6b0487dcd857ba72189ca0624cb4985961f08f25))
|
||||
* **webhooks:** trigger memo updates for attachment and relation changes ([#5795](https://github.com/usememos/memos/issues/5795)) ([acbc914](https://github.com/usememos/memos/commit/acbc914dea8d4e533028b03929193a4af1ac32b4))
|
||||
* **web:** prevent MemoContent prop leaks ([22519b5](https://github.com/usememos/memos/commit/22519b57a0a971ab927a03015c994c63b8c3f1da))
|
||||
* **web:** refine attachment media layout ([a0d83e1](https://github.com/usememos/memos/commit/a0d83e1a9e9d0ba50a0103e594adb011728faf8c))
|
||||
* **web:** refresh memo detail cache after editor save ([333c9df](https://github.com/usememos/memos/commit/333c9df233240a843a9e62d17c6ff464690636f8))
|
||||
* **web:** use BroadcastChannel to sync token refreshes across tabs ([bbdc998](https://github.com/usememos/memos/commit/bbdc998646e8093223a448378a2c29690f0d8612))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* batch load memo relations when listing memos ([#5692](https://github.com/usememos/memos/issues/5692)) ([1e82714](https://github.com/usememos/memos/commit/1e82714a52c72455a6ba02605cc641d7ad4fbf1a))
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
# These owners will be the default owners for everything in the repo.
|
||||
* @usememos/moderators
|
||||
* @boojack @lqwakeup
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
# Build frontend dist.
|
||||
FROM node:16.15.0-alpine AS frontend
|
||||
WORKDIR /frontend-build
|
||||
|
||||
COPY ./web/ .
|
||||
|
||||
RUN yarn
|
||||
RUN yarn build
|
||||
|
||||
# Build backend exec file.
|
||||
FROM golang:1.18.3-alpine3.16 AS backend
|
||||
WORKDIR /backend-build
|
||||
|
||||
RUN apk update
|
||||
RUN apk --no-cache add gcc musl-dev
|
||||
|
||||
COPY . .
|
||||
COPY --from=frontend /frontend-build/dist ./server/dist
|
||||
|
||||
RUN go build -o memos ./bin/server/main.go
|
||||
|
||||
# Make workspace with above generated files.
|
||||
FROM alpine:3.16.0 AS monolithic
|
||||
WORKDIR /usr/local/memos
|
||||
|
||||
COPY --from=backend /backend-build/memos /usr/local/memos/
|
||||
|
||||
# Directory to store the data, which can be referenced as the mounting point.
|
||||
RUN mkdir -p /var/opt/memos
|
||||
|
||||
ENTRYPOINT ["./memos", "--mode", "prod", "--port", "5230"]
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Memos
|
||||
Copyright (c) 2022 Memos
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,81 +1,49 @@
|
||||
> ✨ Featured Sponsor: [CodeRabbit](https://coderabbit.link/usememos) — Cut code review time & bugs in half, instantly.
|
||||
<p align="center"><a href="https://usememos.com"><img height="64px" src="https://raw.githubusercontent.com/usememos/memos/main/resources/logo-full.webp" alt="✍️ memos" /></a></p>
|
||||
|
||||
# Memos
|
||||
<p align="center">An open source, self-hosted knowledge base that works with a SQLite db file.</p>
|
||||
|
||||
<img align="right" height="96px" src="https://raw.githubusercontent.com/usememos/.github/refs/heads/main/assets/logo-rounded.png" alt="Memos" />
|
||||
<p align="center">
|
||||
<a href="https://github.com/usememos/memos/stargazers"><img alt="GitHub stars" src="https://img.shields.io/github/stars/usememos/memos" /></a>
|
||||
<a href="https://hub.docker.com/r/neosmemo/memos"><img alt="Docker pull" src="https://img.shields.io/docker/pulls/neosmemo/memos.svg" /></a>
|
||||
<img alt="Go report" src="https://goreportcard.com/badge/github.com/usememos/memos" />
|
||||
</p>
|
||||
|
||||
Memos is an open-source, self-hosted note-taking app built for quick capture. It is Markdown-native, lightweight, and keeps your data under your control.
|
||||
<p align="center">
|
||||
<a href="https://demo.usememos.com/">Live Demo</a> •
|
||||
<a href="https://t.me/+-_tNF1k70UU4ZTc9">Discuss in Telegram 👾</a>
|
||||
</p>
|
||||
|
||||
[](https://usememos.com)
|
||||
[](https://demo.usememos.com/)
|
||||
[](https://usememos.com/docs)
|
||||
[](https://discord.gg/tfPJa4UmAv)
|
||||
[](https://hub.docker.com/r/neosmemo/memos)
|
||||
|
||||
<img src="https://raw.githubusercontent.com/usememos/.github/refs/heads/main/assets/demo.png" alt="Memos Demo Screenshot" height="512" />
|
||||
|
||||
|
||||
## Features
|
||||
|
||||
- **Capture quickly** — A timeline-first interface keeps note-taking simple: open, write, and move on.
|
||||
- **Own your data** — Self-host Memos on your infrastructure with no telemetry.
|
||||
- **Deploy anywhere** — Run a single Go binary or Docker container with SQLite, MySQL, or PostgreSQL.
|
||||
- **Integrate freely** — Build on the REST and gRPC APIs or adapt the MIT-licensed source to your needs.
|
||||
- 🦄 Fully open source;
|
||||
- 📜 Writing in plain textarea without any burden,
|
||||
- and support some useful markdown syntax 💪.
|
||||
- 🌄 Share the memo in a pretty image or personal page like Twitter;
|
||||
- 🚀 Fast self-hosting with `Docker`;
|
||||
- 🤠 Pleasant UI and UX;
|
||||
|
||||
## Quick Start
|
||||
## Deploy with Docker
|
||||
|
||||
Want to explore Memos first? Open the [live demo](https://demo.usememos.com/).
|
||||
### Docker Run
|
||||
|
||||
### Docker (Recommended)
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name memos \
|
||||
-p 5230:5230 \
|
||||
-v ~/.memos:/var/opt/memos \
|
||||
neosmemo/memos:stable
|
||||
```docker
|
||||
docker run -d --name memos -p 5230:5230 -v ~/.memos/:/var/opt/memos neosmemo/memos:latest
|
||||
```
|
||||
|
||||
Open `http://localhost:5230` and start writing.
|
||||
Memos should be running at [http://localhost:5230](http://localhost:5230). If the `~/.memos/` does not have a `memos_prod.db` file, then memos will auto generate it.
|
||||
|
||||
### Native Binary
|
||||
### Docker Compose
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/usememos/memos/main/scripts/install.sh | sh
|
||||
```
|
||||
|
||||
### Other Installation Methods
|
||||
|
||||
- **Docker Compose** — Recommended for production deployments.
|
||||
- **Kubernetes** — Helm charts and manifests are available.
|
||||
- **Build from source** — Best for development and customization.
|
||||
|
||||
See the [deployment guide](https://usememos.com/docs/deploy) for detailed instructions.
|
||||
|
||||
## Web Clipper
|
||||
|
||||
Save pages, selected text, and images directly to your Memos instance with the official [Memos Web Clipper](https://github.com/usememos/web-clipper). The extension is available for [Chrome](https://chromewebstore.google.com/detail/memos-web-clipper/nebaoebnljalfegiidibihhkebeiklbl) and [Firefox](https://addons.mozilla.org/en-US/firefox/addon/memos-web-clipper/), and lets you review each clip, choose its visibility, and customize its Markdown format before saving.
|
||||
See more in the example [`docker-compose.yaml`](./docker-compose.yaml) file.
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions of all kinds are welcome, including bug reports, feature suggestions, pull requests, documentation, and translations.
|
||||
Contributions are what make the open source community such an amazing place to be learn, inspire, and create. Any contributions you make are greatly appreciated. 🥰
|
||||
|
||||
- [Report bugs](https://github.com/usememos/memos/issues/new?template=bug_report.md)
|
||||
- [Suggest features](https://github.com/usememos/memos/issues/new?template=feature_request.md)
|
||||
- [Submit pull requests](https://github.com/usememos/memos/pulls)
|
||||
- [Improve documentation](https://github.com/usememos/dotcom)
|
||||
- [Help with translations](https://github.com/usememos/memos/tree/main/web/src/locales)
|
||||
Gets more about [development guide](https://github.com/usememos/memos/tree/main/docs/development.md).
|
||||
|
||||
## Sponsors
|
||||
## Star history
|
||||
|
||||
- [**CodeRabbit** — Cut code review time and bugs in half](https://coderabbit.link/usememos)
|
||||
- [**SSD Nodes** — Affordable VPS hosting for self-hosters](https://ssdnodes.com/?utm_source=memos&utm_medium=sponsor)
|
||||
|
||||
Love Memos? [Sponsor us on GitHub](https://github.com/sponsors/usememos) to help keep the project growing!
|
||||
|
||||
## License
|
||||
|
||||
Memos is open-source software licensed under the [MIT License](LICENSE). See our [Privacy Policy](https://usememos.com/privacy) for details on data handling.
|
||||
|
||||
---
|
||||
|
||||
**[Website](https://usememos.com)** • **[Documentation](https://usememos.com/docs)** • **[Demo](https://demo.usememos.com/)** • **[Discord](https://discord.gg/tfPJa4UmAv)** • **[X/Twitter](https://x.com/usememos)**
|
||||
[](https://star-history.com/#usememos/memos&Date)
|
||||
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Memos is currently a `0.x` project. Security fixes are only provided for the latest release. Older releases are not supported for security updates, and fixes are not backported.
|
||||
|
||||
If you run Memos in production, keep your instance updated to the latest release.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report security issues privately by email: `dev@usememos.com`
|
||||
|
||||
Do not open public GitHub issues, discussions, or pull requests for suspected vulnerabilities.
|
||||
|
||||
Please include:
|
||||
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce
|
||||
- Affected version or commit
|
||||
- Deployment details that matter to reproduction
|
||||
- Your assessment of impact
|
||||
|
||||
We will review reports as time permits and fix valid issues in regular releases.
|
||||
|
||||
## Disclosure and CVEs
|
||||
|
||||
Memos is self-hosted software and is still in the `0.x` stage. At this stage, we do not run a formal disclosure program, publish separate security advisories for every issue, or request CVE IDs.
|
||||
|
||||
Security fixes may be shipped directly in normal releases or noted briefly in release notes and changelogs.
|
||||
|
||||
## Self-Hosted Deployment Notes
|
||||
|
||||
The security posture of a Memos instance depends heavily on how it is deployed and operated. In particular:
|
||||
|
||||
- Keep Memos updated
|
||||
- Put it behind a properly configured reverse proxy when exposed to the internet
|
||||
- Require authentication for any non-public deployment
|
||||
- Use TLS in production
|
||||
- Limit access to trusted users and administrators
|
||||
|
||||
Reports that depend entirely on intentionally unsafe deployment choices, unsupported local patches, or administrator actions may be treated as deployment issues rather than product vulnerabilities.
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package api
|
||||
|
||||
// RowStatus is the status for a row.
|
||||
type RowStatus string
|
||||
|
||||
const (
|
||||
// Normal is the status for a normal row.
|
||||
Normal RowStatus = "NORMAL"
|
||||
// Archived is the status for an archived row.
|
||||
Archived RowStatus = "ARCHIVED"
|
||||
)
|
||||
|
||||
func (e RowStatus) String() string {
|
||||
switch e {
|
||||
case Normal:
|
||||
return "NORMAL"
|
||||
case Archived:
|
||||
return "ARCHIVED"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package api
|
||||
|
||||
type Signin struct {
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
type Signup struct {
|
||||
Email string `json:"email"`
|
||||
Role Role `json:"role"`
|
||||
Name string `json:"name"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package api
|
||||
|
||||
// CacheNamespace is the type of a cache.
|
||||
type CacheNamespace string
|
||||
|
||||
const (
|
||||
// UserCache is the cache type of users.
|
||||
UserCache CacheNamespace = "u"
|
||||
// MemoCache is the cache type of memos.
|
||||
MemoCache CacheNamespace = "m"
|
||||
// ShortcutCache is the cache type of shortcuts.
|
||||
ShortcutCache CacheNamespace = "s"
|
||||
// ResourceCache is the cache type of resources.
|
||||
ResourceCache CacheNamespace = "r"
|
||||
)
|
||||
|
||||
// CacheService is the service for caches.
|
||||
type CacheService interface {
|
||||
FindCache(namespace CacheNamespace, id int, entry interface{}) (bool, error)
|
||||
UpsertCache(namespace CacheNamespace, id int, entry interface{}) error
|
||||
DeleteCache(namespace CacheNamespace, id int)
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package api
|
||||
|
||||
// Visibility is the type of a visibility.
|
||||
type Visibility string
|
||||
|
||||
const (
|
||||
// Public is the PUBLIC visibility.
|
||||
Public Visibility = "PUBLIC"
|
||||
// Protected is the PROTECTED visibility.
|
||||
Protected Visibility = "PROTECTED"
|
||||
// Privite is the PRIVATE visibility.
|
||||
Privite Visibility = "PRIVATE"
|
||||
)
|
||||
|
||||
func (e Visibility) String() string {
|
||||
switch e {
|
||||
case Public:
|
||||
return "PUBLIC"
|
||||
case Protected:
|
||||
return "PROTECTED"
|
||||
case Privite:
|
||||
return "PRIVATE"
|
||||
}
|
||||
return "PRIVATE"
|
||||
}
|
||||
|
||||
type Memo struct {
|
||||
ID int `json:"id"`
|
||||
|
||||
// Standard fields
|
||||
RowStatus RowStatus `json:"rowStatus"`
|
||||
CreatorID int `json:"creatorId"`
|
||||
CreatedTs int64 `json:"createdTs"`
|
||||
UpdatedTs int64 `json:"updatedTs"`
|
||||
|
||||
// Domain specific fields
|
||||
Content string `json:"content"`
|
||||
Visibility Visibility `json:"visibility"`
|
||||
Pinned bool `json:"pinned"`
|
||||
|
||||
// Related fields
|
||||
Creator *User `json:"creator"`
|
||||
}
|
||||
|
||||
type MemoCreate struct {
|
||||
// Standard fields
|
||||
CreatorID int
|
||||
|
||||
// Domain specific fields
|
||||
Visibility Visibility `json:"visibility"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type MemoPatch struct {
|
||||
ID int
|
||||
|
||||
// Standard fields
|
||||
CreatedTs *int64 `json:"createdTs"`
|
||||
RowStatus *RowStatus `json:"rowStatus"`
|
||||
|
||||
// Domain specific fields
|
||||
Content *string `json:"content"`
|
||||
Visibility *Visibility `json:"visibility"`
|
||||
}
|
||||
|
||||
type MemoFind struct {
|
||||
ID *int `json:"id"`
|
||||
|
||||
// Standard fields
|
||||
RowStatus *RowStatus `json:"rowStatus"`
|
||||
CreatorID *int `json:"creatorId"`
|
||||
|
||||
// Domain specific fields
|
||||
Pinned *bool
|
||||
ContentSearch *string
|
||||
VisibilityList []Visibility
|
||||
|
||||
// Pagination
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type MemoDelete struct {
|
||||
ID int `json:"id"`
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package api
|
||||
|
||||
type MemoOrganizer struct {
|
||||
ID int
|
||||
|
||||
// Domain specific fields
|
||||
MemoID int
|
||||
UserID int
|
||||
Pinned bool
|
||||
}
|
||||
|
||||
type MemoOrganizerFind struct {
|
||||
MemoID int
|
||||
UserID int
|
||||
}
|
||||
|
||||
type MemoOrganizerUpsert struct {
|
||||
MemoID int
|
||||
UserID int
|
||||
Pinned bool `json:"pinned"`
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package api
|
||||
|
||||
type Resource struct {
|
||||
ID int `json:"id"`
|
||||
|
||||
// Standard fields
|
||||
CreatorID int `json:"creatorId"`
|
||||
CreatedTs int64 `json:"createdTs"`
|
||||
UpdatedTs int64 `json:"updatedTs"`
|
||||
|
||||
// Domain specific fields
|
||||
Filename string `json:"filename"`
|
||||
Blob []byte `json:"-"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type ResourceCreate struct {
|
||||
// Standard fields
|
||||
CreatorID int
|
||||
|
||||
// Domain specific fields
|
||||
Filename string `json:"filename"`
|
||||
Blob []byte `json:"blob"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type ResourceFind struct {
|
||||
ID *int `json:"id"`
|
||||
|
||||
// Standard fields
|
||||
CreatorID *int `json:"creatorId"`
|
||||
|
||||
// Domain specific fields
|
||||
Filename *string `json:"filename"`
|
||||
}
|
||||
|
||||
type ResourceDelete struct {
|
||||
ID int
|
||||
|
||||
// Standard fields
|
||||
CreatorID int
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package api
|
||||
|
||||
type Shortcut struct {
|
||||
ID int `json:"id"`
|
||||
|
||||
// Standard fields
|
||||
RowStatus RowStatus `json:"rowStatus"`
|
||||
CreatorID int `json:"creatorId"`
|
||||
CreatedTs int64 `json:"createdTs"`
|
||||
UpdatedTs int64 `json:"updatedTs"`
|
||||
|
||||
// Domain specific fields
|
||||
Title string `json:"title"`
|
||||
Payload string `json:"payload"`
|
||||
}
|
||||
|
||||
type ShortcutCreate struct {
|
||||
// Standard fields
|
||||
CreatorID int
|
||||
|
||||
// Domain specific fields
|
||||
Title string `json:"title"`
|
||||
Payload string `json:"payload"`
|
||||
}
|
||||
|
||||
type ShortcutPatch struct {
|
||||
ID int
|
||||
|
||||
// Standard fields
|
||||
RowStatus *RowStatus `json:"rowStatus"`
|
||||
|
||||
// Domain specific fields
|
||||
Title *string `json:"title"`
|
||||
Payload *string `json:"payload"`
|
||||
}
|
||||
|
||||
type ShortcutFind struct {
|
||||
ID *int
|
||||
|
||||
// Standard fields
|
||||
CreatorID *int
|
||||
|
||||
// Domain specific fields
|
||||
Title *string `json:"title"`
|
||||
}
|
||||
|
||||
type ShortcutDelete struct {
|
||||
ID int
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package api
|
||||
|
||||
import "github.com/usememos/memos/server/profile"
|
||||
|
||||
type SystemStatus struct {
|
||||
Host *User `json:"host"`
|
||||
Profile *profile.Profile `json:"profile"`
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/usememos/memos/common"
|
||||
)
|
||||
|
||||
// Role is the type of a role.
|
||||
type Role string
|
||||
|
||||
const (
|
||||
// Host is the HOST role.
|
||||
Host Role = "HOST"
|
||||
// NormalUser is the USER role.
|
||||
NormalUser Role = "USER"
|
||||
)
|
||||
|
||||
func (e Role) String() string {
|
||||
switch e {
|
||||
case Host:
|
||||
return "HOST"
|
||||
case NormalUser:
|
||||
return "USER"
|
||||
}
|
||||
return "USER"
|
||||
}
|
||||
|
||||
type User struct {
|
||||
ID int `json:"id"`
|
||||
|
||||
// Standard fields
|
||||
RowStatus RowStatus `json:"rowStatus"`
|
||||
CreatedTs int64 `json:"createdTs"`
|
||||
UpdatedTs int64 `json:"updatedTs"`
|
||||
|
||||
// Domain specific fields
|
||||
Email string `json:"email"`
|
||||
Role Role `json:"role"`
|
||||
Name string `json:"name"`
|
||||
PasswordHash string `json:"-"`
|
||||
OpenID string `json:"openId"`
|
||||
UserSettingList []*UserSetting `json:"userSettingList"`
|
||||
}
|
||||
|
||||
type UserCreate struct {
|
||||
// Domain specific fields
|
||||
Email string `json:"email"`
|
||||
Role Role `json:"role"`
|
||||
Name string `json:"name"`
|
||||
Password string `json:"password"`
|
||||
PasswordHash string
|
||||
OpenID string
|
||||
}
|
||||
|
||||
func (create UserCreate) Validate() error {
|
||||
if !common.ValidateEmail(create.Email) {
|
||||
return fmt.Errorf("invalid email format")
|
||||
}
|
||||
if len(create.Email) < 6 {
|
||||
return fmt.Errorf("email is too short, minimum length is 6")
|
||||
}
|
||||
if len(create.Password) < 6 {
|
||||
return fmt.Errorf("password is too short, minimum length is 6")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type UserPatch struct {
|
||||
ID int
|
||||
|
||||
// Standard fields
|
||||
RowStatus *RowStatus `json:"rowStatus"`
|
||||
|
||||
// Domain specific fields
|
||||
Email *string `json:"email"`
|
||||
Name *string `json:"name"`
|
||||
Password *string `json:"password"`
|
||||
ResetOpenID *bool `json:"resetOpenId"`
|
||||
PasswordHash *string
|
||||
OpenID *string
|
||||
}
|
||||
|
||||
type UserFind struct {
|
||||
ID *int `json:"id"`
|
||||
|
||||
// Standard fields
|
||||
RowStatus *RowStatus `json:"rowStatus"`
|
||||
|
||||
// Domain specific fields
|
||||
Email *string `json:"email"`
|
||||
Role *Role
|
||||
Name *string `json:"name"`
|
||||
OpenID *string
|
||||
}
|
||||
|
||||
type UserDelete struct {
|
||||
ID int
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type UserSettingKey string
|
||||
|
||||
const (
|
||||
// UserSettingLocaleKey is the key type for user locale.
|
||||
UserSettingLocaleKey UserSettingKey = "locale"
|
||||
// UserSettingMemoVisibilityKey is the key type for user preference memo default visibility.
|
||||
UserSettingMemoVisibilityKey UserSettingKey = "memoVisibility"
|
||||
// UserSettingEditorFontStyleKey is the key type for editor font style.
|
||||
UserSettingEditorFontStyleKey UserSettingKey = "editorFontStyle"
|
||||
// UserSettingEditorFontStyleKey is the key type for mobile editor style.
|
||||
UserSettingMobileEditorStyleKey UserSettingKey = "mobileEditorStyle"
|
||||
)
|
||||
|
||||
// String returns the string format of UserSettingKey type.
|
||||
func (key UserSettingKey) String() string {
|
||||
switch key {
|
||||
case UserSettingLocaleKey:
|
||||
return "locale"
|
||||
case UserSettingMemoVisibilityKey:
|
||||
return "memoVisibility"
|
||||
case UserSettingEditorFontStyleKey:
|
||||
return "editorFontFamily"
|
||||
case UserSettingMobileEditorStyleKey:
|
||||
return "mobileEditorStyle"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var (
|
||||
UserSettingLocaleValue = []string{"en", "zh", "vi"}
|
||||
UserSettingMemoVisibilityValue = []Visibility{Privite, Protected, Public}
|
||||
UserSettingEditorFontStyleValue = []string{"normal", "mono"}
|
||||
UserSettingMobileEditorStyleValue = []string{"normal", "float"}
|
||||
)
|
||||
|
||||
type UserSetting struct {
|
||||
UserID int
|
||||
Key UserSettingKey `json:"key"`
|
||||
// Value is a JSON string with basic value
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type UserSettingUpsert struct {
|
||||
UserID int
|
||||
Key UserSettingKey `json:"key"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
func (upsert UserSettingUpsert) Validate() error {
|
||||
if upsert.Key == UserSettingLocaleKey {
|
||||
localeValue := "en"
|
||||
err := json.Unmarshal([]byte(upsert.Value), &localeValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to unmarshal user setting locale value")
|
||||
}
|
||||
|
||||
invalid := true
|
||||
for _, value := range UserSettingLocaleValue {
|
||||
if localeValue == value {
|
||||
invalid = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
return fmt.Errorf("invalid user setting locale value")
|
||||
}
|
||||
} else if upsert.Key == UserSettingMemoVisibilityKey {
|
||||
memoVisibilityValue := Privite
|
||||
err := json.Unmarshal([]byte(upsert.Value), &memoVisibilityValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to unmarshal user setting memo visibility value")
|
||||
}
|
||||
|
||||
invalid := true
|
||||
for _, value := range UserSettingMemoVisibilityValue {
|
||||
if memoVisibilityValue == value {
|
||||
invalid = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
return fmt.Errorf("invalid user setting memo visibility value")
|
||||
}
|
||||
} else if upsert.Key == UserSettingEditorFontStyleKey {
|
||||
editorFontStyleValue := "normal"
|
||||
err := json.Unmarshal([]byte(upsert.Value), &editorFontStyleValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to unmarshal user setting editor font style")
|
||||
}
|
||||
|
||||
invalid := true
|
||||
for _, value := range UserSettingEditorFontStyleValue {
|
||||
if editorFontStyleValue == value {
|
||||
invalid = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
return fmt.Errorf("invalid user setting editor font style value")
|
||||
}
|
||||
} else if upsert.Key == UserSettingMobileEditorStyleKey {
|
||||
mobileEditorStyleValue := "normal"
|
||||
err := json.Unmarshal([]byte(upsert.Value), &mobileEditorStyleValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to unmarshal user setting mobile editor style")
|
||||
}
|
||||
|
||||
invalid := true
|
||||
for _, value := range UserSettingMobileEditorStyleValue {
|
||||
if mobileEditorStyleValue == value {
|
||||
invalid = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
return fmt.Errorf("invalid user setting mobile editor style value")
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("invalid user setting key")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type UserSettingFind struct {
|
||||
UserID int
|
||||
|
||||
Key *UserSettingKey `json:"key"`
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/usememos/memos/server"
|
||||
"github.com/usememos/memos/server/profile"
|
||||
"github.com/usememos/memos/store"
|
||||
|
||||
DB "github.com/usememos/memos/store/db"
|
||||
)
|
||||
|
||||
const (
|
||||
greetingBanner = `
|
||||
███╗ ███╗███████╗███╗ ███╗ ██████╗ ███████╗
|
||||
████╗ ████║██╔════╝████╗ ████║██╔═══██╗██╔════╝
|
||||
██╔████╔██║█████╗ ██╔████╔██║██║ ██║███████╗
|
||||
██║╚██╔╝██║██╔══╝ ██║╚██╔╝██║██║ ██║╚════██║
|
||||
██║ ╚═╝ ██║███████╗██║ ╚═╝ ██║╚██████╔╝███████║
|
||||
╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝
|
||||
`
|
||||
)
|
||||
|
||||
func run(profile *profile.Profile) error {
|
||||
ctx := context.Background()
|
||||
|
||||
db := DB.NewDB(profile)
|
||||
if err := db.Open(ctx); err != nil {
|
||||
return fmt.Errorf("cannot open db: %w", err)
|
||||
}
|
||||
|
||||
s := server.NewServer(profile)
|
||||
|
||||
storeInstance := store.New(db.Db, profile)
|
||||
s.Store = storeInstance
|
||||
|
||||
println(greetingBanner)
|
||||
fmt.Printf("Version %s has started at :%d\n", profile.Version, profile.Port)
|
||||
|
||||
return s.Run()
|
||||
}
|
||||
|
||||
func execute() error {
|
||||
profile, err := profile.GetProfile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
println("---")
|
||||
println("profile")
|
||||
println("mode:", profile.Mode)
|
||||
println("port:", profile.Port)
|
||||
println("dsn:", profile.DSN)
|
||||
println("version:", profile.Version)
|
||||
println("---")
|
||||
|
||||
if err := run(profile); err != nil {
|
||||
fmt.Printf("error: %+v\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := execute(); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func parseSlogLevel(s string) (slog.Level, error) {
|
||||
switch strings.ToLower(s) {
|
||||
case "debug":
|
||||
return slog.LevelDebug, nil
|
||||
case "info":
|
||||
return slog.LevelInfo, nil
|
||||
case "warn":
|
||||
return slog.LevelWarn, nil
|
||||
case "error":
|
||||
return slog.LevelError, nil
|
||||
default:
|
||||
return slog.LevelInfo, errors.Errorf("unknown log level %q: must be debug, info, warn, or error", s)
|
||||
}
|
||||
}
|
||||
|
||||
func newLogger(level slog.Level, w io.Writer) *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(w, &slog.HandlerOptions{Level: level}))
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseSlogLevel(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
wantLevel slog.Level
|
||||
wantErr bool
|
||||
}{
|
||||
{"debug", slog.LevelDebug, false},
|
||||
{"info", slog.LevelInfo, false},
|
||||
{"warn", slog.LevelWarn, false},
|
||||
{"error", slog.LevelError, false},
|
||||
{"DEBUG", slog.LevelDebug, false},
|
||||
{"INFO", slog.LevelInfo, false},
|
||||
{"WARN", slog.LevelWarn, false},
|
||||
{"ERROR", slog.LevelError, false},
|
||||
{"invalid", slog.LevelInfo, true},
|
||||
{"", slog.LevelInfo, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got, err := parseSlogLevel(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseSlogLevel(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
if got != tt.wantLevel {
|
||||
t.Errorf("parseSlogLevel(%q) = %v, want %v", tt.input, got, tt.wantLevel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLoggerLevelFiltering(t *testing.T) {
|
||||
tests := []struct {
|
||||
level slog.Level
|
||||
logAt slog.Level
|
||||
msg string
|
||||
shouldAppear bool
|
||||
}{
|
||||
// debug passes all
|
||||
{slog.LevelDebug, slog.LevelDebug, "debug-msg", true},
|
||||
{slog.LevelDebug, slog.LevelInfo, "info-msg", true},
|
||||
{slog.LevelDebug, slog.LevelWarn, "warn-msg", true},
|
||||
{slog.LevelDebug, slog.LevelError, "error-msg", true},
|
||||
// info suppresses debug
|
||||
{slog.LevelInfo, slog.LevelDebug, "debug-suppressed", false},
|
||||
{slog.LevelInfo, slog.LevelInfo, "info-visible", true},
|
||||
{slog.LevelInfo, slog.LevelWarn, "warn-visible", true},
|
||||
// warn suppresses debug+info
|
||||
{slog.LevelWarn, slog.LevelDebug, "debug-suppressed", false},
|
||||
{slog.LevelWarn, slog.LevelInfo, "info-suppressed", false},
|
||||
{slog.LevelWarn, slog.LevelWarn, "warn-visible", true},
|
||||
{slog.LevelWarn, slog.LevelError, "error-visible", true},
|
||||
// error suppresses everything below
|
||||
{slog.LevelError, slog.LevelDebug, "debug-suppressed", false},
|
||||
{slog.LevelError, slog.LevelInfo, "info-suppressed", false},
|
||||
{slog.LevelError, slog.LevelWarn, "warn-suppressed", false},
|
||||
{slog.LevelError, slog.LevelError, "error-visible", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
var buf bytes.Buffer
|
||||
logger := newLogger(tt.level, &buf)
|
||||
logger.Log(context.TODO(), tt.logAt, tt.msg)
|
||||
|
||||
appeared := strings.Contains(buf.String(), tt.msg)
|
||||
if appeared != tt.shouldAppear {
|
||||
t.Errorf("level=%s logAt=%s msg=%q: appeared=%v want=%v",
|
||||
tt.level, tt.logAt, tt.msg, appeared, tt.shouldAppear)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLoggerOutputFormat(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
logger := newLogger(slog.LevelDebug, &buf)
|
||||
logger.Info("hello-world", "key", "value")
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "hello-world") {
|
||||
t.Errorf("expected message in output, got: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "key=value") {
|
||||
t.Errorf("expected key=value attr in output, got: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "INFO") {
|
||||
t.Errorf("expected level in output, got: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLoggerDoesNotMutateGlobalDefault(t *testing.T) {
|
||||
original := slog.Default()
|
||||
var buf bytes.Buffer
|
||||
_ = newLogger(slog.LevelError, &buf)
|
||||
if slog.Default() != original {
|
||||
t.Error("newLogger must not change slog.Default()")
|
||||
}
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/usememos/memos/internal/profile"
|
||||
"github.com/usememos/memos/internal/version"
|
||||
"github.com/usememos/memos/internal/webhook"
|
||||
"github.com/usememos/memos/server"
|
||||
"github.com/usememos/memos/store"
|
||||
"github.com/usememos/memos/store/db"
|
||||
)
|
||||
|
||||
func initSlogDefault() {
|
||||
level, err := parseSlogLevel(viper.GetString("log-level"))
|
||||
if err != nil {
|
||||
slog.Warn("invalid log-level value, defaulting to info", "error", err)
|
||||
}
|
||||
slog.SetDefault(newLogger(level, os.Stderr))
|
||||
}
|
||||
|
||||
var (
|
||||
rootCmd = &cobra.Command{
|
||||
Use: "memos",
|
||||
Short: `An open source, lightweight note-taking service. Easily capture and share your great thoughts.`,
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
instanceProfile := &profile.Profile{
|
||||
Demo: viper.GetBool("demo"),
|
||||
Addr: viper.GetString("addr"),
|
||||
Port: viper.GetInt("port"),
|
||||
UNIXSock: viper.GetString("unix-sock"),
|
||||
Data: viper.GetString("data"),
|
||||
Driver: viper.GetString("driver"),
|
||||
DSN: viper.GetString("dsn"),
|
||||
InstanceURL: viper.GetString("instance-url"),
|
||||
}
|
||||
instanceProfile.Version = version.GetCurrentVersion()
|
||||
instanceProfile.Commit = version.Commit
|
||||
webhook.AllowPrivateIPs = viper.GetBool("allow-private-webhooks")
|
||||
|
||||
if err := instanceProfile.Validate(); err != nil {
|
||||
slog.Error("failed to validate profile", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
dbDriver, err := db.NewDBDriver(instanceProfile)
|
||||
if err != nil {
|
||||
cancel()
|
||||
slog.Error("failed to create db driver", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
storeInstance := store.New(dbDriver, instanceProfile)
|
||||
if err := storeInstance.Migrate(ctx); err != nil {
|
||||
cancel()
|
||||
slog.Error("failed to migrate", "error", err)
|
||||
return
|
||||
}
|
||||
if err := storeInstance.LoadDeploymentConfiguration(ctx); err != nil {
|
||||
cancel()
|
||||
slog.Error("failed to load deployment configuration", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
s, err := server.NewServer(ctx, instanceProfile, storeInstance)
|
||||
if err != nil {
|
||||
cancel()
|
||||
slog.Error("failed to create server", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
c := make(chan os.Signal, 1)
|
||||
// Trigger graceful shutdown on SIGINT or SIGTERM.
|
||||
// The default signal sent by the `kill` command is SIGTERM,
|
||||
// which is taken as the graceful shutdown signal for many systems, eg., Kubernetes, Gunicorn.
|
||||
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
if err := s.Start(ctx); err != nil {
|
||||
if err != http.ErrServerClosed {
|
||||
slog.Error("failed to start server", "error", err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
printGreetings(instanceProfile)
|
||||
|
||||
go func() {
|
||||
<-c
|
||||
s.Shutdown(ctx)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// Wait for CTRL-C.
|
||||
<-ctx.Done()
|
||||
},
|
||||
}
|
||||
versionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print the current Memos version",
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
fmt.Println(version.GetCurrentVersion())
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cobra.OnInitialize(initSlogDefault)
|
||||
|
||||
viper.SetDefault("demo", false)
|
||||
viper.SetDefault("driver", "sqlite")
|
||||
viper.SetDefault("port", 8081)
|
||||
|
||||
rootCmd.PersistentFlags().Bool("demo", false, "enable demo mode")
|
||||
rootCmd.PersistentFlags().String("addr", "", "address of server")
|
||||
rootCmd.PersistentFlags().Int("port", 8081, "port of server")
|
||||
rootCmd.PersistentFlags().String("unix-sock", "", "path to the unix socket, overrides --addr and --port")
|
||||
rootCmd.PersistentFlags().String("data", "", "data directory")
|
||||
rootCmd.PersistentFlags().String("driver", "sqlite", "database driver")
|
||||
rootCmd.PersistentFlags().String("dsn", "", "database source name(aka. DSN)")
|
||||
rootCmd.PersistentFlags().String("instance-url", "", "the url of your memos instance")
|
||||
rootCmd.PersistentFlags().Bool("allow-private-webhooks", false, "allow webhook URLs to resolve to private/reserved IP addresses")
|
||||
rootCmd.PersistentFlags().String("log-level", "info", "log verbosity level (debug, info, warn, error)")
|
||||
|
||||
if err := viper.BindPFlag("demo", rootCmd.PersistentFlags().Lookup("demo")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("addr", rootCmd.PersistentFlags().Lookup("addr")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("port", rootCmd.PersistentFlags().Lookup("port")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("unix-sock", rootCmd.PersistentFlags().Lookup("unix-sock")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("data", rootCmd.PersistentFlags().Lookup("data")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("driver", rootCmd.PersistentFlags().Lookup("driver")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("dsn", rootCmd.PersistentFlags().Lookup("dsn")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("instance-url", rootCmd.PersistentFlags().Lookup("instance-url")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("allow-private-webhooks", rootCmd.PersistentFlags().Lookup("allow-private-webhooks")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := viper.BindPFlag("log-level", rootCmd.PersistentFlags().Lookup("log-level")); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
viper.SetEnvPrefix("memos")
|
||||
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
|
||||
viper.AutomaticEnv()
|
||||
|
||||
rootCmd.AddCommand(versionCmd)
|
||||
}
|
||||
|
||||
func printGreetings(profile *profile.Profile) {
|
||||
fmt.Printf("Memos %s started successfully!\n", profile.Version)
|
||||
|
||||
if profile.Demo {
|
||||
fmt.Fprint(os.Stderr, "Development mode is enabled\n")
|
||||
if profile.DSN != "" {
|
||||
fmt.Fprintf(os.Stderr, "Database: %s\n", profile.DSN)
|
||||
}
|
||||
}
|
||||
|
||||
// Server information
|
||||
fmt.Printf("Data directory: %s\n", profile.Data)
|
||||
fmt.Printf("Database driver: %s\n", profile.Driver)
|
||||
|
||||
// Connection information
|
||||
if len(profile.UNIXSock) == 0 {
|
||||
if len(profile.Addr) == 0 {
|
||||
fmt.Printf("Server running on port %d\n", profile.Port)
|
||||
fmt.Printf("Access your memos at: http://localhost:%d\n", profile.Port)
|
||||
} else {
|
||||
fmt.Printf("Server running on %s:%d\n", profile.Addr, profile.Port)
|
||||
fmt.Printf("Access your memos at: http://%s:%d\n", profile.Addr, profile.Port)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("Server running on unix socket: %s\n", profile.UNIXSock)
|
||||
}
|
||||
|
||||
// Access mode is derived from instance_url: set = public, unset = private.
|
||||
accessMode := "private"
|
||||
if profile.AllowAnonymous() {
|
||||
accessMode = "public"
|
||||
}
|
||||
fmt.Printf("Access mode: %s\n", accessMode)
|
||||
|
||||
fmt.Println()
|
||||
fmt.Printf("Documentation: %s\n", "https://usememos.com")
|
||||
fmt.Printf("Source code: %s\n", "https://github.com/usememos/memos")
|
||||
fmt.Println("\nHappy note-taking!")
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
// Code is the error code.
|
||||
type Code int
|
||||
|
||||
// Application error codes.
|
||||
const (
|
||||
// 0 ~ 99 general error.
|
||||
Ok Code = 0
|
||||
Internal Code = 1
|
||||
NotAuthorized Code = 2
|
||||
Invalid Code = 3
|
||||
NotFound Code = 4
|
||||
Conflict Code = 5
|
||||
NotImplemented Code = 6
|
||||
)
|
||||
|
||||
// Error represents an application-specific error. Application errors can be
|
||||
// unwrapped by the caller to extract out the code & message.
|
||||
//
|
||||
// Any non-application error (such as a disk error) should be reported as an
|
||||
// Internal error and the human user should only see "Internal error" as the
|
||||
// message. These low-level internal error details should only be logged and
|
||||
// reported to the operator of the application (not the end user).
|
||||
type Error struct {
|
||||
// Machine-readable error code.
|
||||
Code Code
|
||||
|
||||
// Embedded error.
|
||||
Err error
|
||||
}
|
||||
|
||||
// Error implements the error interface. Not used by the application otherwise.
|
||||
func (e *Error) Error() string {
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
// ErrorCode unwraps an application error and returns its code.
|
||||
// Non-application errors always return EINTERNAL.
|
||||
func ErrorCode(err error) Code {
|
||||
var e *Error
|
||||
if err == nil {
|
||||
return Ok
|
||||
} else if errors.As(err, &e) {
|
||||
return e.Code
|
||||
}
|
||||
return Internal
|
||||
}
|
||||
|
||||
// ErrorMessage unwraps an application error and returns its message.
|
||||
// Non-application errors always return "Internal error".
|
||||
func ErrorMessage(err error) string {
|
||||
var e *Error
|
||||
if err == nil {
|
||||
return ""
|
||||
} else if errors.As(err, &e) {
|
||||
return e.Err.Error()
|
||||
}
|
||||
return "Internal error."
|
||||
}
|
||||
|
||||
// Errorf is a helper function to return an Error with a given code and error.
|
||||
func Errorf(code Code, err error) *Error {
|
||||
return &Error{
|
||||
Code: code,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"net/mail"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// HasPrefixes returns true if the string s has any of the given prefixes.
|
||||
func HasPrefixes(src string, prefixes ...string) bool {
|
||||
for _, prefix := range prefixes {
|
||||
if strings.HasPrefix(src, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ValidateEmail validates the email.
|
||||
func ValidateEmail(email string) bool {
|
||||
if _, err := mail.ParseAddress(email); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func GenUUID() string {
|
||||
return uuid.New().String()
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package util //nolint:revive // util is an appropriate package name for utility functions
|
||||
package common
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -14,7 +14,7 @@ func TestValidateEmail(t *testing.T) {
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
email: "@usememos.com",
|
||||
email: "@qq.com",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
@@ -1,6 +1,7 @@
|
||||
version: "3.0"
|
||||
services:
|
||||
memos:
|
||||
image: neosmemo/memos:stable
|
||||
image: neosmemo/memos:latest
|
||||
container_name: memos
|
||||
volumes:
|
||||
- ~/.memos/:/var/opt/memos
|
||||
@@ -1,497 +0,0 @@
|
||||
# Configuration Provisioning
|
||||
|
||||
Status: Implemented
|
||||
|
||||
## Summary
|
||||
|
||||
Memos should follow Mastodon's deployment-configuration model: configuration supplied by the deployment is loaded directly into each server process and
|
||||
remains authoritative for that process lifetime. It is not imported into the database and is not tracked as database-owned application state.
|
||||
|
||||
The first supported file-backed configuration resources are:
|
||||
|
||||
- OAuth2 identity providers.
|
||||
- Instance settings for general policy, storage, memo behavior, notifications, and AI providers.
|
||||
|
||||
Memos scans `/etc/secrets` after database migration and demo seeding, validates every matching file, builds one immutable configuration snapshot, and
|
||||
publishes that snapshot before HTTP or background services start. Applying a changed file requires a process restart.
|
||||
|
||||
Every resource file contains exactly one existing `memos.store` protobuf message encoded as protobuf JSON. No resource envelope, state file, ownership
|
||||
table, or second persistent copy of a secret is introduced. The process necessarily holds decoded secrets in its private runtime snapshot.
|
||||
|
||||
## Design model
|
||||
|
||||
Mastodon reads external authentication and other deployment configuration from environment variables or a dotenv file during process initialization. It
|
||||
does not copy that configuration into an administrator-editable database resource or maintain Terraform-style ownership state.
|
||||
|
||||
Memos should use the same lifecycle while adapting the input format to its existing generated store messages:
|
||||
|
||||
- Mounted JSON files replace a large collection of environment variables.
|
||||
- File-backed resources exist in the effective runtime configuration.
|
||||
- Stored resources continue to exist in the database but are shadowed when a file declares the same stable key.
|
||||
- UI and API mutations cannot change an actively file-backed resource.
|
||||
- Removing a file and restarting removes the runtime override; it does not delete or modify the stored resource.
|
||||
|
||||
This is deployment configuration, not resource reconciliation. Terms such as adoption, import, unmanage, drift, prune, and Terraform state do not apply.
|
||||
|
||||
## Goals
|
||||
|
||||
- Accept secrets through mounted files without committing them to seed SQL or command-line arguments.
|
||||
- Keep each file equal to one generated store protobuf message.
|
||||
- Load and validate the complete file set before exposing any of it.
|
||||
- Make deployment configuration authoritative for the lifetime of the process.
|
||||
- Preserve database-backed UI configuration for keys not supplied by files.
|
||||
- Prevent API writes from appearing to change an effective file-backed resource.
|
||||
- Keep secret values out of logs, API responses, caches that expose values, and additional persistence.
|
||||
- Preserve the administrator password sign-in path when password sign-in is disabled for regular users.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Persist file contents or file ownership metadata in the database.
|
||||
- Add a `provisioning_resource` table or provisioning columns to existing tables.
|
||||
- Reconcile database state to match a desired resource graph.
|
||||
- Delete database resources when files disappear.
|
||||
- Support multiple configuration sources with precedence rules in the first version.
|
||||
- Write UI changes back into mounted files.
|
||||
- Watch files or reload configuration without restarting in the first version.
|
||||
- Support partial field ownership within an instance-setting group.
|
||||
|
||||
## Terminology
|
||||
|
||||
**Stored configuration**
|
||||
: Configuration stored in the existing `idp` and `system_setting` database tables.
|
||||
|
||||
**Deployment configuration**
|
||||
: Configuration decoded from matching files during process startup.
|
||||
|
||||
**Effective configuration**
|
||||
: The configuration used by APIs, authentication, and background services. Deployment configuration shadows stored configuration with the same stable key.
|
||||
|
||||
**Stable key**
|
||||
: The identity-provider UID or instance-setting key used to merge deployment and stored configuration.
|
||||
|
||||
## File discovery
|
||||
|
||||
Memos scans direct children of `/etc/secrets`. The directory may contain unrelated platform secrets; only supported filename patterns are read. Memos does
|
||||
not recurse into subdirectories and does not create, modify, or delete anything in the directory.
|
||||
|
||||
| Filename pattern | Protobuf message | Stable key |
|
||||
| --- | --- | --- |
|
||||
| `memos-idp-<label>.json` | `memos.store.IdentityProvider` | `uid` |
|
||||
| `memos-instance-setting-<label>.json` | `memos.store.InstanceSetting` | `key` |
|
||||
|
||||
`<label>` uses lowercase kebab case and must match `[a-z0-9]+(?:-[a-z0-9]+)*`. Matching is case-sensitive and the extension is lowercase `.json`.
|
||||
For upgrade compatibility, identity-provider filenames accepted by the original bootstrap (`memos-idp-*.json`) continue to load when the label is not
|
||||
lowercase kebab case, but startup logs a deprecation warning. New files should always use the canonical convention.
|
||||
|
||||
Recommended labels mirror the resource key for operator readability:
|
||||
|
||||
| Resource | Canonical filename |
|
||||
| --- | --- |
|
||||
| Identity provider with UID `primary-sso` | `memos-idp-primary-sso.json` |
|
||||
| `GENERAL` | `memos-instance-setting-general.json` |
|
||||
| `STORAGE` | `memos-instance-setting-storage.json` |
|
||||
| `MEMO_RELATED` | `memos-instance-setting-memo-related.json` |
|
||||
| `NOTIFICATION` | `memos-instance-setting-notification.json` |
|
||||
| `AI` | `memos-instance-setting-ai.json` |
|
||||
|
||||
The filename label remains descriptive rather than authoritative. The `uid` or `key` inside the message is the resource identity, so renaming a file does
|
||||
not change account links or effective resource identity. Files are read in lexical order only to produce deterministic diagnostics; ordering has no
|
||||
configuration semantics.
|
||||
|
||||
Each matching file:
|
||||
|
||||
- Must be a valid protobuf JSON representation of the expected message.
|
||||
- Must not contain unknown fields.
|
||||
- Must not exceed 1 MiB.
|
||||
- Must contain exactly one resource.
|
||||
- May contain plaintext secrets because the containing directory is treated as sensitive.
|
||||
- May be a regular file or a platform-managed symlink that resolves to a regular file, as used by Kubernetes Secret volumes.
|
||||
|
||||
A missing directory or a readable directory without matching files is a normal no-op. An unreadable directory or matching file is a startup error. Startup
|
||||
logs include matched counts by resource type so a misspelled filename is visible without logging file contents. A direct child beginning with `memos-` but
|
||||
not matching a supported pattern produces a warning; unrelated filenames are silently ignored.
|
||||
|
||||
### SSO-only deployments
|
||||
|
||||
An SSO-only deployment mounts both an identity-provider file and `memos-instance-setting-general.json` with `disallowPasswordAuth` enabled. These resources
|
||||
are validated and published together during startup. The public demo seed does not contain authentication policy, so both files must be mounted to enable
|
||||
SSO-only behavior. Keep `disallowUserRegistration` disabled when first-time SSO users should be created automatically.
|
||||
|
||||
## Identity-provider files
|
||||
|
||||
An identity-provider file contains exactly one `memos.store.IdentityProvider`. The database-generated `id` must be omitted. `uid` is required and is the
|
||||
stable key.
|
||||
|
||||
Example `/etc/secrets/memos-idp-primary-sso.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"uid": "primary-sso",
|
||||
"name": "Company SSO",
|
||||
"type": "OAUTH2",
|
||||
"identifierFilter": "",
|
||||
"config": {
|
||||
"oauth2Config": {
|
||||
"clientId": "client-id",
|
||||
"clientSecret": "client-secret",
|
||||
"authUrl": "https://idp.example.com/oauth/authorize",
|
||||
"tokenUrl": "https://idp.example.com/oauth/token",
|
||||
"userInfoUrl": "https://idp.example.com/oauth/userinfo",
|
||||
"scopes": ["openid", "profile", "email"],
|
||||
"fieldMapping": {
|
||||
"identifier": "sub",
|
||||
"displayName": "name",
|
||||
"email": "email",
|
||||
"avatarUrl": "picture"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Initial validation supports only OAuth2 providers and requires:
|
||||
|
||||
- A valid, nonempty UID and display name.
|
||||
- Client ID and client secret.
|
||||
- Authorization, token, and user-info URLs.
|
||||
- At least one scope, with no empty scope entries.
|
||||
- A field-mapping object with a nonempty identifier field.
|
||||
|
||||
Duplicate UIDs across files are rejected.
|
||||
|
||||
User identity links already use the provider UID as their stable provider value. A file-backed provider therefore does not need a database-generated IdP ID
|
||||
to preserve account links or complete SSO sign-in.
|
||||
|
||||
## Instance-setting files
|
||||
|
||||
An instance-setting file contains exactly one `memos.store.InstanceSetting`. `key` is required and is the stable key. The populated `oneof` must match the
|
||||
key.
|
||||
|
||||
Example `/etc/secrets/memos-instance-setting-general.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "GENERAL",
|
||||
"generalSetting": {
|
||||
"disallowUserRegistration": false,
|
||||
"disallowPasswordAuth": true,
|
||||
"additionalScript": "",
|
||||
"additionalStyle": "",
|
||||
"weekStartDayOffset": 1,
|
||||
"disallowChangeUsername": false,
|
||||
"disallowChangeNickname": false,
|
||||
"customProfile": {
|
||||
"title": "Company Memos",
|
||||
"description": "Internal notes",
|
||||
"logoUrl": "https://example.com/logo.png"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Example `/etc/secrets/memos-instance-setting-notification.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "NOTIFICATION",
|
||||
"notificationSetting": {
|
||||
"email": {
|
||||
"enabled": true,
|
||||
"smtpHost": "smtp.example.com",
|
||||
"smtpPort": 587,
|
||||
"smtpUsername": "memos",
|
||||
"smtpPassword": "smtp-secret",
|
||||
"fromEmail": "memos@example.com",
|
||||
"fromName": "Memos",
|
||||
"replyTo": "support@example.com",
|
||||
"useTls": true,
|
||||
"useSsl": false
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Supported keys:
|
||||
|
||||
| Key | Deployment use |
|
||||
| --- | --- |
|
||||
| `GENERAL` | Registration, authentication, branding, scripts, styles, and user-profile policy |
|
||||
| `STORAGE` | Attachment storage type, limits, paths, and S3 credentials |
|
||||
| `MEMO_RELATED` | Memo limits, editing behavior, and reactions |
|
||||
| `NOTIFICATION` | SMTP transport and credentials |
|
||||
| `AI` | AI providers, API keys, and transcription defaults |
|
||||
|
||||
Rejected keys:
|
||||
|
||||
- `BASIC` contains the instance secret key and database schema version. Replacing the secret key invalidates sessions, while replacing the schema version
|
||||
interferes with database migration state.
|
||||
- `TAGS` is retained for backward compatibility; active tag metadata is stored per user.
|
||||
|
||||
Only one file may declare a given setting key.
|
||||
|
||||
### Complete-group replacement
|
||||
|
||||
An instance-setting group is the smallest deployment-configured unit. A file replaces the complete effective group. A scalar omitted from protobuf JSON is
|
||||
stored in the decoded message as its protobuf default; omission does not preserve a field from the database value.
|
||||
|
||||
Some existing setting getters apply application defaults after decoding zero values. For example, STORAGE defaults to local storage, a 30 MiB upload limit,
|
||||
and `assets/{timestamp}_{uuid}_{filename}` when the corresponding decoded fields are unspecified. The effective behavior is therefore the decoded file plus
|
||||
the same read-time defaults used for database-backed configuration.
|
||||
|
||||
Empty secret fields in a file mean empty values; they never mean "preserve the database secret." Credential-preservation behavior used by UI updates does
|
||||
not apply to deployment configuration.
|
||||
|
||||
### AI normalization
|
||||
|
||||
AI deployment configuration uses deterministic, self-contained normalization rather than the UI update path:
|
||||
|
||||
- Every provider requires an explicit stable `id`; the loader never generates one.
|
||||
- Every provider requires a title, a supported provider type, and an API key.
|
||||
- An empty OpenAI endpoint becomes `https://api.openai.com/v1`.
|
||||
- An empty Gemini endpoint becomes `https://generativelanguage.googleapis.com/v1beta`.
|
||||
- Duplicate provider IDs are rejected.
|
||||
- A transcription provider ID must reference a provider in the same effective AI setting.
|
||||
- Model, language, and prompt use the same length limits as API-managed settings.
|
||||
- No provider, API key, or transcription value is copied from the shadowed database setting.
|
||||
|
||||
## Configuration format compatibility
|
||||
|
||||
Using store protobuf JSON makes the selected messages a supported deployment-configuration interface even though the messages remain internal to the
|
||||
application. For every provisionable message, Memos must preserve:
|
||||
|
||||
- Existing protobuf JSON field names.
|
||||
- Existing enum names and meanings.
|
||||
- Stable resource-key and `oneof` mappings.
|
||||
- Previously valid omissions for fields that have defaults.
|
||||
|
||||
New optional fields and enum values may be added. A provisionable field may be deprecated, but its existing JSON spelling must continue to decode for the
|
||||
supported upgrade window. Field names must not be reused with a different meaning. New validation should not invalidate an existing safe configuration
|
||||
without an upgrade note and a documented replacement.
|
||||
|
||||
Unknown fields remain startup errors because this catches misspellings and configuration written for a newer, incompatible Memos version. Compatibility
|
||||
tests should keep representative JSON fixtures from earlier releases and decode them with the current loader.
|
||||
|
||||
## Runtime configuration snapshot
|
||||
|
||||
The loader builds an immutable snapshot containing maps keyed by provider UID and setting key. It does not mutate the database while loading. The `Store`
|
||||
owns this snapshot so all existing consumers resolve configuration through one boundary.
|
||||
|
||||
Startup follows this sequence:
|
||||
|
||||
```text
|
||||
Initialize or migrate database
|
||||
-> apply demo seed when enabled
|
||||
-> read all matching deployment-configuration files
|
||||
-> decode and validate every resource
|
||||
-> validate affected cross-resource invariants
|
||||
-> publish one immutable runtime snapshot
|
||||
-> construct HTTP and background services
|
||||
-> accept requests
|
||||
```
|
||||
|
||||
If any matching file is invalid, no snapshot is published and startup fails. Atomicity comes from publishing the snapshot only after complete validation;
|
||||
no cross-database transaction abstraction is required because deployment configuration performs no database writes.
|
||||
|
||||
The snapshot is loaded once. Files changed after startup have no effect until the process restarts.
|
||||
|
||||
### Immutability and copy semantics
|
||||
|
||||
Generated protobuf messages are mutable pointers, so immutability must be enforced rather than assumed:
|
||||
|
||||
- Canonical snapshot messages remain private to the `Store`.
|
||||
- Effective getters return deep clones, using `proto.Clone`, rather than canonical pointers.
|
||||
- Read-time defaults and redaction are applied only to clones.
|
||||
- Canonical snapshot messages are never inserted into the existing instance-setting TTL cache.
|
||||
- Callers cannot obtain a mutable map or message owned by the snapshot.
|
||||
|
||||
This prevents one request, background runner, defaulting helper, or redaction path from changing configuration observed by another goroutine.
|
||||
|
||||
### Effective and stored access
|
||||
|
||||
The store facade has an explicit separation between effective reads and stored-resource access:
|
||||
|
||||
- Normal list/get operations used by authentication, APIs, and background services return effective configuration.
|
||||
- Internal raw list/get operations read only the database and are used by migration, snapshot planning, and permitted mutation paths.
|
||||
- Mutation services check the snapshot source before loading a raw database row.
|
||||
- A file-backed IdP has no database ID and must never be passed to a driver update or delete operation.
|
||||
- The loader reads stored configuration through raw access before publishing the snapshot, avoiding recursive effective resolution.
|
||||
|
||||
## Effective configuration resolution
|
||||
|
||||
### Identity providers
|
||||
|
||||
List and get operations return the union of stored and file-backed providers by UID:
|
||||
|
||||
- A file-backed provider shadows a stored provider with the same UID.
|
||||
- Stored providers with other UIDs remain available.
|
||||
- Authentication resolves the same effective provider collection.
|
||||
- Removing the file and restarting reveals any stored provider that had been shadowed; it does not restore values from the file.
|
||||
- Stored providers retain their database insertion order, and a file-backed provider that shadows one occupies the same position. Providers that exist only
|
||||
in deployment configuration are appended in UID order, keeping existing API and sign-in ordering stable while remaining deterministic.
|
||||
|
||||
Operators migrating an existing stored provider to a file should keep the same UID so existing user-identity links continue to work. They should remove or
|
||||
update the shadowed stored provider before later removing the file if they do not want the old database configuration to reappear.
|
||||
|
||||
### Instance settings
|
||||
|
||||
Every effective instance-setting getter checks the runtime snapshot before its database cache:
|
||||
|
||||
- A file-backed group completely shadows the `system_setting` row with the same key.
|
||||
- Other setting groups continue to use stored values and existing application defaults.
|
||||
- The runtime snapshot must never be overwritten by a cached database value.
|
||||
- Removing a file and restarting returns the group to its stored database value.
|
||||
|
||||
The demo seed writes `MEMO_RELATED` but does not write `GENERAL`. Loading deployment configuration after seeding supplies the complete effective General
|
||||
settings without embedding deployment authentication policy in demo data.
|
||||
|
||||
## Validation and authentication safety
|
||||
|
||||
All file-local validation runs before snapshot publication. Relationships between file-backed resources are validated against the resulting effective
|
||||
configuration when the desired files affect that relationship.
|
||||
|
||||
At minimum, validation rejects:
|
||||
|
||||
- A file-backed `GENERAL` setting that disables password authentication for regular users when the resulting effective configuration has no identity
|
||||
provider.
|
||||
- An instance-setting key whose populated `oneof` does not match the key.
|
||||
- S3 storage without the required endpoint, bucket, region, or credentials.
|
||||
- Enabled email delivery without the required SMTP host, port, or sender.
|
||||
- Duplicate AI provider IDs.
|
||||
- Transcription referencing an AI provider ID absent from the effective AI setting.
|
||||
- Duplicate stable keys across files.
|
||||
|
||||
An unrelated file must not turn an existing database condition into a new startup failure. For example, a STORAGE-only file does not fail startup merely
|
||||
because the database already disables regular-user password sign-in while containing no IdP; Memos logs that existing condition as a warning. A file that
|
||||
configures GENERAL or an IdP evaluates the authentication invariant against the resulting effective state.
|
||||
|
||||
The administrator password path remains available regardless of `disallowPasswordAuth`. Runtime mutations reject transitions from a safe authentication
|
||||
state to one where password sign-in is disabled for regular users without an effective IdP. An unrelated edit may preserve an already-existing legacy
|
||||
violation so an upgrade does not make the complete `GENERAL` group uneditable; the administrator can resolve that state by enabling password sign-in or
|
||||
configuring an IdP. Deleting the last effective IdP from a previously safe state remains rejected.
|
||||
|
||||
The validation and database mutation must be one serializable store operation. In particular, updating `GENERAL` and deleting an IdP cannot use separate
|
||||
check-then-write calls, because concurrent requests could each validate an old safe state and together produce an unsafe state. The narrow runtime mutation
|
||||
operation:
|
||||
|
||||
1. Starts a serializable database transaction.
|
||||
2. Reads the stored `GENERAL` setting and stored IdPs inside that transaction.
|
||||
3. Combines them with the immutable file snapshot and the proposed mutation.
|
||||
4. Validates the resulting effective authentication state.
|
||||
5. Applies the stored-resource mutation and commits.
|
||||
6. Retries serialization conflicts a bounded number of times.
|
||||
|
||||
This transaction is required for runtime authentication-policy safety, not for loading deployment files. It adds no table or schema migration. Each
|
||||
database driver must provide equivalent transaction semantics for this narrow operation.
|
||||
|
||||
## API behavior
|
||||
|
||||
The API operates on effective resources for reads and stored resources for permitted writes.
|
||||
|
||||
Mutation behavior:
|
||||
|
||||
- Creating a stored IdP with a UID reserved by a file-backed provider returns `codes.FailedPrecondition`.
|
||||
- Updating or deleting a file-backed IdP returns `codes.FailedPrecondition`.
|
||||
- Updating a file-backed instance-setting group returns `codes.FailedPrecondition`.
|
||||
- The instance-setting guard runs before validation or future field-mask application, so every masked update to a file-backed group is rejected.
|
||||
- Mutations of unshadowed stored configuration continue normally, subject to authentication safety invariants.
|
||||
- API responses continue to redact client secrets, SMTP passwords, S3 secrets, and AI API keys.
|
||||
|
||||
No API operation writes to the mounted files. Test operations that do not change stored configuration, such as testing the effective SMTP configuration,
|
||||
remain available.
|
||||
|
||||
## Frontend behavior
|
||||
|
||||
The frontend does not receive configuration-source metadata. It presents the normal mutation controls and reports the API's `FailedPrecondition` error when
|
||||
an administrator attempts to create, update, or delete a deployment-managed resource. The API remains the sole authority for mutation enforcement.
|
||||
|
||||
## Security
|
||||
|
||||
- Treat `/etc/secrets` and every matching file as sensitive plaintext.
|
||||
- Recommend owner-only or application-group-readable filesystem permissions.
|
||||
- Never log file contents, decoded messages, before/after values, or secret fields.
|
||||
- Redact secrets from validation errors and startup summaries.
|
||||
- Do not persist deployment secrets in `idp`, `system_setting`, a state file, or ownership metadata.
|
||||
- Fail startup on an invalid or unreadable matching file rather than publishing partial configuration.
|
||||
- Keep the immutable snapshot process-local and expose only redacted API representations.
|
||||
|
||||
## Multiple server replicas
|
||||
|
||||
Every replica independently loads deployment configuration at startup, as Mastodon processes independently load environment configuration. All replicas in
|
||||
one deployment must mount identical files.
|
||||
|
||||
A rolling deployment can temporarily run old and new configuration generations at the same time. Memos does not attempt distributed reconciliation or
|
||||
cache invalidation for this process-local configuration. Deployments changing authentication or storage configuration should use a rollout strategy that
|
||||
does not route traffic to replicas with different file generations, and readiness must be reported only after the new snapshot validates successfully.
|
||||
|
||||
Because file-backed settings bypass the database setting cache, a replica cannot replace a deployment value with a stale cached database value.
|
||||
|
||||
## Database and migration impact
|
||||
|
||||
This design requires no database schema changes and no migrations:
|
||||
|
||||
- File-backed IdPs are not inserted into `idp`.
|
||||
- File-backed settings are not inserted into `system_setting`.
|
||||
- Existing user-identity links remain database-backed and continue to reference provider UIDs.
|
||||
- Existing stored configuration remains untouched beneath runtime overrides.
|
||||
|
||||
### Transition from the database-writing bootstrap
|
||||
|
||||
Versions with the original `memos-idp-*.json` bootstrap copied file-backed IdPs, including client secrets, into the `idp` table during migration. The new
|
||||
loader cannot reliably distinguish those rows from providers created through the UI, so it must not delete or scrub them automatically.
|
||||
|
||||
When a file shadows a stored provider with the same UID, startup logs a secret-free warning that a stored copy remains. Operators who previously used the
|
||||
database-writing bootstrap should clean up explicitly:
|
||||
|
||||
1. Back up the database and retain administrator password access.
|
||||
2. Temporarily remove the IdP file and restart Memos so the stored provider is no longer shadowed.
|
||||
3. Delete or update the stored provider through the administrator UI or API, or perform equivalent offline database maintenance.
|
||||
4. Restore the file and restart Memos.
|
||||
|
||||
Until that cleanup is complete, the old stored provider and secret remain in the database and can reappear if the file is removed. The no-persistence
|
||||
guarantee applies to the new loader; it does not claim to erase secrets written by earlier versions.
|
||||
|
||||
## Implementation
|
||||
|
||||
The implementation:
|
||||
|
||||
1. Replaces the database-writing IdP bootstrap with a typed deployment-configuration loader.
|
||||
2. Decodes and validates `memos-instance-setting-*.json` resources.
|
||||
3. Loads configuration after migration and demo seeding but before service construction.
|
||||
4. Publishes an immutable provider/settings snapshot owned by the store facade, with clone-on-read semantics.
|
||||
5. Resolves file-backed values before database values and caches for IdP authentication and instance settings.
|
||||
6. Uses explicit raw database reads for snapshot planning and permitted mutation paths.
|
||||
7. Validates affected startup state and uses a narrow serializable transaction for runtime authentication mutations.
|
||||
8. Enforces deployment ownership through API mutation guards and returns `codes.FailedPrecondition` for rejected writes.
|
||||
|
||||
## Test strategy
|
||||
|
||||
The implementation requires tests for:
|
||||
|
||||
- Every canonical filename pattern, legacy identity-provider filename compatibility, and supported message type.
|
||||
- Unknown fields, oversized files, unreadable files, invalid protobuf JSON, and invalid enum values.
|
||||
- Duplicate provider UIDs and setting keys.
|
||||
- Rejection of `BASIC`, `TAGS`, and key/`oneof` mismatches.
|
||||
- Complete validation before snapshot publication.
|
||||
- Store-proto JSON compatibility fixtures from earlier releases.
|
||||
- Effective merging and shadowing by provider UID and setting key.
|
||||
- Stable stored ordering and deterministic placement of deployment-only identity providers.
|
||||
- Removal of a file taking effect after constructing a new process snapshot without deleting stored configuration.
|
||||
- Existing user-identity links working with a file-backed provider of the same UID.
|
||||
- File-backed settings bypassing database cache entries.
|
||||
- Clone-on-read behavior and race tests proving snapshot messages cannot be mutated by callers.
|
||||
- Authentication safety for startup and runtime IdP deletion.
|
||||
- Concurrent `GENERAL` updates and IdP deletion preserving the runtime authentication invariant across database drivers.
|
||||
- An unrelated deployment file not failing because of untouched pre-existing authentication state.
|
||||
- Storage read-time defaults, SMTP validation, and deterministic AI normalization without database secret preservation.
|
||||
- Missing-directory behavior and matched-file count logging.
|
||||
- Mutation guards, including requests carrying field masks.
|
||||
- A stored provider shadowed by a file producing a secret-free legacy-bootstrap warning.
|
||||
- Secret redaction in errors, logs, and API responses.
|
||||
|
||||
## Research references
|
||||
|
||||
- [Mastodon environment configuration](https://docs.joinmastodon.org/admin/config/)
|
||||
- [Mastodon OmniAuth initialization](https://github.com/mastodon/mastodon/blob/main/config/initializers/3_omniauth.rb)
|
||||
- [GitLab OpenID Connect configuration](https://docs.gitlab.com/administration/auth/oidc/)
|
||||
- [Keycloak startup import](https://www.keycloak.org/server/importExport)
|
||||
- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/)
|
||||
@@ -0,0 +1,40 @@
|
||||
# Development
|
||||
|
||||
Memos is built with a curated tech stack. It is optimized for developer experience and is very easy to start working on the code:
|
||||
|
||||
1. It has no external dependency.
|
||||
2. It requires zero config.
|
||||
3. 1 command to start backend and 1 command to start frontend, both with live reload support.
|
||||
|
||||
## Tech Stack
|
||||
|
||||
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Go](https://golang.org/doc/install)
|
||||
- [Air](https://github.com/cosmtrek/air#installation) for backend live reload
|
||||
- [Node.js](https://nodejs.org/)
|
||||
- [yarn](https://yarnpkg.com/getting-started/install)
|
||||
|
||||
## Steps
|
||||
|
||||
1. pull source code
|
||||
|
||||
```bash
|
||||
git clone https://github.com/usememos/memos
|
||||
```
|
||||
|
||||
2. start backend using air(with live reload)
|
||||
|
||||
```bash
|
||||
air -c scripts/.air.toml
|
||||
```
|
||||
|
||||
3. start frontend dev server
|
||||
|
||||
```bash
|
||||
cd web && yarn && yarn dev
|
||||
```
|
||||
|
||||
Memos should now be running at [http://localhost:3000](http://localhost:3000) and change either frontend or backend code would trigger live reload.
|
||||
@@ -1,154 +1,38 @@
|
||||
module github.com/usememos/memos
|
||||
|
||||
go 1.26.2
|
||||
go 1.17
|
||||
|
||||
require github.com/mattn/go-sqlite3 v1.14.9
|
||||
|
||||
require github.com/google/uuid v1.3.0
|
||||
|
||||
require (
|
||||
connectrpc.com/connect v1.19.2
|
||||
github.com/at-wat/ebml-go v0.18.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.6
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.16
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.15
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/google/cel-go v0.28.1
|
||||
github.com/google/jsonschema-go v0.4.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/feeds v1.2.0
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/labstack/echo/v5 v5.1.0
|
||||
github.com/lib/pq v1.12.3
|
||||
github.com/lithammer/shortuuid/v4 v4.2.0
|
||||
github.com/moby/moby/api v1.54.2
|
||||
github.com/modelcontextprotocol/go-sdk v1.6.1
|
||||
github.com/openai/openai-go/v3 v3.32.0
|
||||
github.com/pion/opus v0.0.0-20260430223319-81a9c5dc5013
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/yuin/goldmark v1.8.2
|
||||
golang.org/x/crypto v0.50.0
|
||||
golang.org/x/mod v0.35.0
|
||||
golang.org/x/net v0.53.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sync v0.20.0
|
||||
google.golang.org/genai v1.54.0
|
||||
google.golang.org/genproto v0.0.0-20260427160629-7cedc36a6bc4
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4
|
||||
google.golang.org/grpc v1.80.0
|
||||
modernc.org/sqlite v1.50.0
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasttemplate v1.2.1 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
|
||||
golang.org/x/net v0.0.0-20220728030405-41545e8bf201 // indirect
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/client v0.4.1 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.3.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.4 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.2.0 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
|
||||
golang.org/x/image v0.39.0 // indirect
|
||||
google.golang.org/api v0.276.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
modernc.org/libc v1.72.1 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
github.com/gorilla/context v1.1.1 // indirect
|
||||
github.com/labstack/echo/v4 v4.9.0
|
||||
github.com/labstack/gommon v0.3.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 // indirect
|
||||
github.com/aws/smithy-go v1.25.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
golang.org/x/text v0.36.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
github.com/VictoriaMetrics/fastcache v1.10.0
|
||||
github.com/gorilla/securecookie v1.1.1
|
||||
github.com/gorilla/sessions v1.2.1
|
||||
github.com/labstack/echo-contrib v0.13.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
)
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package ai
|
||||
|
||||
// ProviderType identifies an AI provider implementation.
|
||||
type ProviderType string
|
||||
|
||||
const (
|
||||
// ProviderOpenAI is OpenAI's hosted API.
|
||||
ProviderOpenAI ProviderType = "OPENAI"
|
||||
// ProviderGemini is Google's Gemini API.
|
||||
ProviderGemini ProviderType = "GEMINI"
|
||||
)
|
||||
|
||||
// ProviderConfig configures a callable AI provider connection.
|
||||
type ProviderConfig struct {
|
||||
ID string
|
||||
Title string
|
||||
Type ProviderType
|
||||
Endpoint string
|
||||
APIKey string
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
// Package audio provides audio container/codec helpers for AI providers.
|
||||
//
|
||||
// The motivating use case is Gemini transcription: Gemini's audio inputs
|
||||
// require WAV/MP3/AIFF/AAC/OGG/FLAC, but browser MediaRecorder defaults to
|
||||
// WebM/Opus. This package converts WebM/Opus into 16-bit PCM WAV using
|
||||
// pure-Go decoders — no ffmpeg or other system dependency.
|
||||
package audio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/at-wat/ebml-go"
|
||||
"github.com/at-wat/ebml-go/webm"
|
||||
"github.com/pion/opus"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
opusOutputSampleRate = 48000
|
||||
// maxOpusPacketSamples is Opus's spec maximum: 120 ms at 48 kHz.
|
||||
maxOpusPacketSamples = 5760
|
||||
// opusCodecID is the WebM TrackEntry CodecID for an Opus audio track.
|
||||
opusCodecID = "A_OPUS"
|
||||
// opusHeadMinLength is the minimum size of the OpusHead identification
|
||||
// header stored in TrackEntry.CodecPrivate.
|
||||
opusHeadMinLength = 19
|
||||
)
|
||||
|
||||
// WebMOpusToWAV decodes a WebM/Opus file into 16-bit PCM WAV bytes.
|
||||
//
|
||||
// The output is mono or stereo at 48 kHz (Opus's native decode rate),
|
||||
// regardless of the original encoder's hint. Pre-skip samples declared in
|
||||
// the OpusHead are discarded to avoid the encoder's startup padding.
|
||||
//
|
||||
// The function reads the entire WebM document into memory; callers should
|
||||
// enforce their own size limits before invoking it.
|
||||
func WebMOpusToWAV(input []byte) ([]byte, error) {
|
||||
var doc struct {
|
||||
Header webm.EBMLHeader `ebml:"EBML"`
|
||||
Segment webm.Segment `ebml:"Segment"`
|
||||
}
|
||||
if err := ebml.Unmarshal(bytes.NewReader(input), &doc); err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, errors.Wrap(err, "parse webm")
|
||||
}
|
||||
|
||||
track := findOpusTrack(doc.Segment.Tracks.TrackEntry)
|
||||
if track == nil {
|
||||
return nil, errors.New("webm has no Opus audio track")
|
||||
}
|
||||
if len(track.CodecPrivate) < opusHeadMinLength {
|
||||
return nil, errors.Errorf("invalid OpusHead: expected at least %d bytes, got %d", opusHeadMinLength, len(track.CodecPrivate))
|
||||
}
|
||||
|
||||
channels := int(track.Audio.Channels)
|
||||
if channels < 1 || channels > 2 {
|
||||
return nil, errors.Errorf("unsupported Opus channel count: %d", channels)
|
||||
}
|
||||
preSkip := int(binary.LittleEndian.Uint16(track.CodecPrivate[10:12]))
|
||||
|
||||
decoder := opus.NewDecoder()
|
||||
if err := decoder.Init(opusOutputSampleRate, channels); err != nil {
|
||||
return nil, errors.Wrap(err, "init opus decoder")
|
||||
}
|
||||
|
||||
pcm := make([]int16, 0, 1<<16)
|
||||
frame := make([]int16, maxOpusPacketSamples*channels)
|
||||
|
||||
decodeBlock := func(block ebml.Block) error {
|
||||
if block.TrackNumber != track.TrackNumber {
|
||||
return nil
|
||||
}
|
||||
for _, packet := range block.Data {
|
||||
if len(packet) == 0 {
|
||||
continue
|
||||
}
|
||||
n, err := decoder.DecodeToInt16(packet, frame)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "decode opus packet")
|
||||
}
|
||||
pcm = append(pcm, frame[:n*channels]...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, cluster := range doc.Segment.Cluster {
|
||||
for _, sb := range cluster.SimpleBlock {
|
||||
if err := decodeBlock(sb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, bg := range cluster.BlockGroup {
|
||||
if err := decodeBlock(bg.Block); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
skip := min(preSkip*channels, len(pcm))
|
||||
pcm = pcm[skip:]
|
||||
|
||||
return encodeWAV(pcm, opusOutputSampleRate, channels), nil
|
||||
}
|
||||
|
||||
// IsWebMContentType reports whether the MIME type is WebM audio.
|
||||
// Both "audio/webm" and "audio/webm; codecs=opus" return true.
|
||||
func IsWebMContentType(contentType string) bool {
|
||||
contentType = strings.TrimSpace(contentType)
|
||||
if contentType == "" {
|
||||
return false
|
||||
}
|
||||
if i := strings.IndexByte(contentType, ';'); i >= 0 {
|
||||
contentType = contentType[:i]
|
||||
}
|
||||
return strings.EqualFold(strings.TrimSpace(contentType), "audio/webm")
|
||||
}
|
||||
|
||||
func findOpusTrack(entries []webm.TrackEntry) *webm.TrackEntry {
|
||||
for i := range entries {
|
||||
entry := &entries[i]
|
||||
if entry.CodecID == opusCodecID && entry.Audio != nil {
|
||||
return entry
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// encodeWAV writes a standard RIFF/WAVE container around 16-bit PCM samples.
|
||||
// Reference layout: http://soundfile.sapp.org/doc/WaveFormat/
|
||||
func encodeWAV(samples []int16, sampleRate, channels int) []byte {
|
||||
const bitsPerSample = 16
|
||||
const bytesPerSample = bitsPerSample / 8
|
||||
blockAlign := channels * bytesPerSample
|
||||
byteRate := sampleRate * blockAlign
|
||||
dataSize := len(samples) * bytesPerSample
|
||||
|
||||
buf := bytes.NewBuffer(make([]byte, 0, 44+dataSize))
|
||||
buf.WriteString("RIFF")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(36+dataSize))
|
||||
buf.WriteString("WAVE")
|
||||
|
||||
buf.WriteString("fmt ")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(16))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(1)) // PCM
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(channels))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(byteRate))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(blockAlign))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(bitsPerSample))
|
||||
|
||||
buf.WriteString("data")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(dataSize))
|
||||
_ = binary.Write(buf, binary.LittleEndian, samples)
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package audio
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsWebMContentType(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"audio/webm", true},
|
||||
{"audio/webm;codecs=opus", true},
|
||||
{"audio/webm; codecs=opus", true},
|
||||
{"AUDIO/WEBM", true},
|
||||
{" audio/webm ", true},
|
||||
{"audio/wav", false},
|
||||
{"audio/mp4", false},
|
||||
{"video/webm", false},
|
||||
{"", false},
|
||||
{"webm", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.in, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, IsWebMContentType(tc.in))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebMOpusToWAV_RejectsInvalidInput(t *testing.T) {
|
||||
t.Run("empty", func(t *testing.T) {
|
||||
_, err := WebMOpusToWAV(nil)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("not webm", func(t *testing.T) {
|
||||
_, err := WebMOpusToWAV([]byte("hello world this is not webm"))
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("truncated webm header bytes", func(t *testing.T) {
|
||||
// Valid EBML magic but no Segment.
|
||||
_, err := WebMOpusToWAV([]byte{0x1A, 0x45, 0xDF, 0xA3})
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Package audiollm defines the multimodal-audio capability for AI providers.
|
||||
// Implementations call chat-completions or generate-content style APIs that
|
||||
// accept audio as input. For deterministic transcription, prefer internal/ai/stt
|
||||
// where a dedicated STT endpoint exists.
|
||||
package audiollm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Model invokes a multimodal LLM with audio input.
|
||||
type Model interface {
|
||||
GenerateFromAudio(ctx context.Context, req Request) (*Response, error)
|
||||
}
|
||||
|
||||
// Request is the input to a multimodal-audio call.
|
||||
type Request struct {
|
||||
Audio io.Reader
|
||||
Size int64
|
||||
ContentType string
|
||||
Model string
|
||||
Instructions string // literal instruction the model is expected to follow
|
||||
Temperature *float32 // optional; nil leaves the provider default in place
|
||||
}
|
||||
|
||||
// Response is the output of a multimodal-audio call.
|
||||
type Response struct {
|
||||
Text string
|
||||
FinishReason FinishReason
|
||||
}
|
||||
|
||||
// FinishReason describes why the model stopped generating.
|
||||
type FinishReason string
|
||||
|
||||
const (
|
||||
FinishStop FinishReason = "stop" // model finished normally
|
||||
FinishLength FinishReason = "length" // truncated by max-tokens
|
||||
FinishSafety FinishReason = "safety" // safety filter blocked output
|
||||
FinishOther FinishReason = "other" // anything else, including unknown
|
||||
)
|
||||
@@ -1,202 +0,0 @@
|
||||
// Package gemini implements audiollm.Model against the Gemini generateContent
|
||||
// endpoint. Used by Memos transcription when the user picks a Gemini provider:
|
||||
// the handler issues a transcription instruction via audiollm.Request.Instructions.
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/genai"
|
||||
|
||||
"github.com/usememos/memos/internal/ai"
|
||||
"github.com/usememos/memos/internal/ai/audio"
|
||||
"github.com/usememos/memos/internal/ai/audiollm"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEndpoint = "https://generativelanguage.googleapis.com/v1beta"
|
||||
defaultAPIVersion = "v1beta"
|
||||
maxInlineSize = 14 * 1024 * 1024
|
||||
providerName = "Gemini"
|
||||
)
|
||||
|
||||
var supportedContentTypes = map[string]string{
|
||||
"audio/wav": "audio/wav",
|
||||
"audio/x-wav": "audio/wav",
|
||||
"audio/mp3": "audio/mp3",
|
||||
"audio/mpeg": "audio/mp3",
|
||||
"audio/aiff": "audio/aiff",
|
||||
"audio/aac": "audio/aac",
|
||||
"audio/ogg": "audio/ogg",
|
||||
"audio/flac": "audio/flac",
|
||||
"audio/x-flac": "audio/flac",
|
||||
}
|
||||
|
||||
// Model implements audiollm.Model for Gemini generateContent.
|
||||
type Model struct {
|
||||
client *genai.Client
|
||||
}
|
||||
|
||||
// New constructs a Model from a provider config.
|
||||
func New(cfg ai.ProviderConfig, options audiollm.Options) (*Model, error) {
|
||||
endpoint, err := normalizeEndpoint(cfg.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.APIKey == "" {
|
||||
return nil, errors.Errorf("%s API key is required", providerName)
|
||||
}
|
||||
baseURL, apiVersion, err := splitEndpoint(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpOptions := genai.HTTPOptions{BaseURL: baseURL, APIVersion: apiVersion}
|
||||
if options.HTTPClient != nil && options.HTTPClient.Timeout > 0 {
|
||||
timeout := options.HTTPClient.Timeout
|
||||
httpOptions.Timeout = &timeout
|
||||
}
|
||||
client, err := genai.NewClient(context.Background(), &genai.ClientConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Backend: genai.BackendGeminiAPI,
|
||||
HTTPClient: options.HTTPClient,
|
||||
HTTPOptions: httpOptions,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create Gemini client")
|
||||
}
|
||||
return &Model{client: client}, nil
|
||||
}
|
||||
|
||||
// GenerateFromAudio calls Gemini generateContent with the audio attached.
|
||||
func (m *Model) GenerateFromAudio(ctx context.Context, req audiollm.Request) (*audiollm.Response, error) {
|
||||
if strings.TrimSpace(req.Model) == "" {
|
||||
return nil, errors.New("model is required")
|
||||
}
|
||||
if req.Audio == nil {
|
||||
return nil, errors.New("audio is required")
|
||||
}
|
||||
if strings.TrimSpace(req.Instructions) == "" {
|
||||
return nil, errors.New("instructions are required")
|
||||
}
|
||||
|
||||
audioBytes, err := io.ReadAll(req.Audio)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read audio")
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, errors.New("audio is required")
|
||||
}
|
||||
|
||||
contentType := req.ContentType
|
||||
if audio.IsWebMContentType(contentType) {
|
||||
wav, err := audio.WebMOpusToWAV(audioBytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to transcode webm audio for Gemini")
|
||||
}
|
||||
audioBytes = wav
|
||||
contentType = "audio/wav"
|
||||
}
|
||||
|
||||
if len(audioBytes) > maxInlineSize {
|
||||
return nil, errors.Errorf("audio is too large for Gemini inline request; maximum size is %d bytes", maxInlineSize)
|
||||
}
|
||||
|
||||
contentType, err = normalizeContentType(contentType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := &genai.GenerateContentConfig{}
|
||||
if req.Temperature != nil {
|
||||
t := *req.Temperature
|
||||
cfg.Temperature = &t
|
||||
}
|
||||
|
||||
resp, err := m.client.Models.GenerateContent(ctx, normalizeModelName(req.Model), []*genai.Content{
|
||||
genai.NewContentFromParts([]*genai.Part{
|
||||
genai.NewPartFromBytes(audioBytes, contentType),
|
||||
genai.NewPartFromText(req.Instructions),
|
||||
}, genai.RoleUser),
|
||||
}, cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to send Gemini request")
|
||||
}
|
||||
|
||||
return &audiollm.Response{
|
||||
Text: strings.TrimSpace(resp.Text()),
|
||||
FinishReason: mapFinishReason(resp),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func mapFinishReason(resp *genai.GenerateContentResponse) audiollm.FinishReason {
|
||||
if resp == nil || len(resp.Candidates) == 0 {
|
||||
return audiollm.FinishOther
|
||||
}
|
||||
switch resp.Candidates[0].FinishReason {
|
||||
case genai.FinishReasonStop:
|
||||
return audiollm.FinishStop
|
||||
case genai.FinishReasonMaxTokens:
|
||||
return audiollm.FinishLength
|
||||
case genai.FinishReasonSafety,
|
||||
genai.FinishReasonRecitation,
|
||||
genai.FinishReasonProhibitedContent,
|
||||
genai.FinishReasonSPII,
|
||||
genai.FinishReasonBlocklist,
|
||||
genai.FinishReasonImageSafety,
|
||||
genai.FinishReasonImageProhibitedContent,
|
||||
genai.FinishReasonImageRecitation:
|
||||
return audiollm.FinishSafety
|
||||
default:
|
||||
return audiollm.FinishOther
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeEndpoint(endpoint string) (string, error) {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
endpoint = defaultEndpoint
|
||||
}
|
||||
if _, err := url.ParseRequestURI(endpoint); err != nil {
|
||||
return "", errors.Wrapf(err, "invalid %s endpoint", providerName)
|
||||
}
|
||||
return strings.TrimRight(endpoint, "/"), nil
|
||||
}
|
||||
|
||||
func splitEndpoint(endpoint string) (string, string, error) {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "invalid Gemini endpoint")
|
||||
}
|
||||
path := strings.TrimRight(parsed.Path, "/")
|
||||
apiVersion := defaultAPIVersion
|
||||
for _, supported := range []string{"v1alpha", "v1beta", "v1"} {
|
||||
if path == "/"+supported || strings.HasSuffix(path, "/"+supported) {
|
||||
apiVersion = supported
|
||||
parsed.Path = strings.TrimSuffix(path, "/"+supported)
|
||||
break
|
||||
}
|
||||
}
|
||||
return strings.TrimRight(parsed.String(), "/"), apiVersion, nil
|
||||
}
|
||||
|
||||
func normalizeContentType(contentType string) (string, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(strings.TrimSpace(contentType))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "invalid audio content type")
|
||||
}
|
||||
mediaType = strings.ToLower(mediaType)
|
||||
normalized, ok := supportedContentTypes[mediaType]
|
||||
if !ok {
|
||||
return "", errors.Errorf("audio content type %q is not supported by Gemini", mediaType)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func normalizeModelName(model string) string {
|
||||
return strings.TrimPrefix(strings.TrimSpace(model), "models/")
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package gemini_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/usememos/memos/internal/ai"
|
||||
"github.com/usememos/memos/internal/ai/audiollm"
|
||||
audiollmgemini "github.com/usememos/memos/internal/ai/audiollm/gemini"
|
||||
)
|
||||
|
||||
func TestGenerateFromAudio(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodPost, r.Method)
|
||||
require.Equal(t, "/v1beta/models/gemini-2.5-flash:generateContent", r.URL.Path)
|
||||
require.Equal(t, "test-key", r.Header.Get("x-goog-api-key"))
|
||||
require.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||
|
||||
var request struct {
|
||||
Contents []struct {
|
||||
Parts []struct {
|
||||
Text string `json:"text"`
|
||||
InlineData *struct {
|
||||
MIMEType string `json:"mimeType"`
|
||||
Data string `json:"data"`
|
||||
} `json:"inlineData"`
|
||||
} `json:"parts"`
|
||||
} `json:"contents"`
|
||||
GenerationConfig map[string]json.Number `json:"generationConfig"`
|
||||
}
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
|
||||
require.Len(t, request.Contents, 1)
|
||||
require.Len(t, request.Contents[0].Parts, 2)
|
||||
require.NotNil(t, request.Contents[0].Parts[0].InlineData)
|
||||
require.Equal(t, "audio/mp3", request.Contents[0].Parts[0].InlineData.MIMEType)
|
||||
audio, err := base64.StdEncoding.DecodeString(request.Contents[0].Parts[0].InlineData.Data)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "audio bytes", string(audio))
|
||||
require.Equal(t, "transcribe please", request.Contents[0].Parts[1].Text)
|
||||
require.Equal(t, json.Number("0"), request.GenerationConfig["temperature"])
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
|
||||
"candidates": []map[string]any{
|
||||
{
|
||||
"finishReason": "STOP",
|
||||
"content": map[string]any{
|
||||
"parts": []map[string]string{{"text": "hello from gemini"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
model, err := audiollmgemini.New(ai.ProviderConfig{
|
||||
Type: ai.ProviderGemini,
|
||||
Endpoint: server.URL + "/v1beta",
|
||||
APIKey: "test-key",
|
||||
}, audiollm.ApplyOptions(nil))
|
||||
require.NoError(t, err)
|
||||
|
||||
temp := float32(0)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
resp, err := model.GenerateFromAudio(ctx, audiollm.Request{
|
||||
Model: "models/gemini-2.5-flash",
|
||||
ContentType: "audio/mpeg",
|
||||
Audio: strings.NewReader("audio bytes"),
|
||||
Instructions: "transcribe please",
|
||||
Temperature: &temp,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "hello from gemini", resp.Text)
|
||||
require.Equal(t, audiollm.FinishStop, resp.FinishReason)
|
||||
}
|
||||
|
||||
func TestGenerateFromAudioRejectsUnsupportedContentType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
model, err := audiollmgemini.New(ai.ProviderConfig{
|
||||
Type: ai.ProviderGemini,
|
||||
Endpoint: "https://example.com/v1beta",
|
||||
APIKey: "test-key",
|
||||
}, audiollm.ApplyOptions(nil))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = model.GenerateFromAudio(context.Background(), audiollm.Request{
|
||||
Model: "gemini-2.5-flash",
|
||||
ContentType: "video/mp4",
|
||||
Audio: strings.NewReader("video bytes"),
|
||||
Instructions: "transcribe please",
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "not supported by Gemini")
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package audiollm
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultHTTPTimeout = 2 * time.Minute
|
||||
|
||||
// Options is the resolved option set passed to provider implementations.
|
||||
type Options struct {
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// ModelOption customizes a Model.
|
||||
type ModelOption func(*Options)
|
||||
|
||||
// WithHTTPClient overrides the HTTP client used by the model.
|
||||
func WithHTTPClient(client *http.Client) ModelOption {
|
||||
return func(o *Options) {
|
||||
if client != nil {
|
||||
o.HTTPClient = client
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyOptions resolves a ModelOption slice into Options with defaults.
|
||||
func ApplyOptions(opts []ModelOption) Options {
|
||||
resolved := Options{HTTPClient: &http.Client{Timeout: defaultHTTPTimeout}}
|
||||
for _, apply := range opts {
|
||||
apply(&resolved)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package ai
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
var (
|
||||
// ErrProviderNotFound indicates that a requested provider ID does not exist.
|
||||
ErrProviderNotFound = errors.New("AI provider not found")
|
||||
// ErrCapabilityUnsupported indicates that the provider does not support the requested capability.
|
||||
ErrCapabilityUnsupported = errors.New("AI provider capability unsupported")
|
||||
// ErrSTTNotSupported indicates that the provider does not have a dedicated
|
||||
// speech-to-text endpoint. Use the audiollm package for multimodal audio
|
||||
// understanding when this is returned.
|
||||
ErrSTTNotSupported = errors.New("provider does not support speech-to-text capability")
|
||||
// ErrAudioLLMNotSupported indicates that the provider does not have a
|
||||
// multimodal-audio LLM available in this codebase.
|
||||
ErrAudioLLMNotSupported = errors.New("provider does not support multimodal audio capability")
|
||||
)
|
||||
@@ -1,22 +0,0 @@
|
||||
package ai
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
const (
|
||||
// DefaultOpenAITranscriptionModel is the built-in OpenAI transcription model.
|
||||
DefaultOpenAITranscriptionModel = "whisper-1"
|
||||
// DefaultGeminiTranscriptionModel is the built-in Gemini transcription model.
|
||||
DefaultGeminiTranscriptionModel = "gemini-2.5-flash"
|
||||
)
|
||||
|
||||
// DefaultTranscriptionModel returns the built-in transcription model for a provider.
|
||||
func DefaultTranscriptionModel(providerType ProviderType) (string, error) {
|
||||
switch providerType {
|
||||
case ProviderOpenAI:
|
||||
return DefaultOpenAITranscriptionModel, nil
|
||||
case ProviderGemini:
|
||||
return DefaultGeminiTranscriptionModel, nil
|
||||
default:
|
||||
return "", errors.Wrapf(ErrCapabilityUnsupported, "provider type %q", providerType)
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
package ai
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
// FindProvider returns the provider with the given ID.
|
||||
func FindProvider(providers []ProviderConfig, providerID string) (*ProviderConfig, error) {
|
||||
if providerID == "" {
|
||||
return nil, errors.Wrap(ErrProviderNotFound, "provider ID is required")
|
||||
}
|
||||
for _, provider := range providers {
|
||||
if provider.ID == providerID {
|
||||
return &provider, nil
|
||||
}
|
||||
}
|
||||
return nil, errors.Wrapf(ErrProviderNotFound, "provider ID %q", providerID)
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
// Package openai implements stt.Transcriber against the OpenAI
|
||||
// /audio/transcriptions endpoint (and any compatible third-party endpoint
|
||||
// such as Groq Whisper, faster-whisper self-hosted, or Azure Whisper).
|
||||
package openai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"mime"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
openaisdk "github.com/openai/openai-go/v3"
|
||||
openaioption "github.com/openai/openai-go/v3/option"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/usememos/memos/internal/ai"
|
||||
"github.com/usememos/memos/internal/ai/stt"
|
||||
)
|
||||
|
||||
const defaultEndpoint = "https://api.openai.com/v1"
|
||||
|
||||
// Transcriber implements stt.Transcriber for OpenAI-compatible STT endpoints.
|
||||
type Transcriber struct {
|
||||
client openaisdk.Client
|
||||
}
|
||||
|
||||
// New constructs a Transcriber from a provider config.
|
||||
func New(cfg ai.ProviderConfig, options stt.Options) (*Transcriber, error) {
|
||||
endpoint, err := normalizeEndpoint(cfg.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.APIKey == "" {
|
||||
return nil, errors.New("OpenAI API key is required")
|
||||
}
|
||||
return &Transcriber{
|
||||
client: openaisdk.NewClient(
|
||||
openaioption.WithAPIKey(cfg.APIKey),
|
||||
openaioption.WithBaseURL(endpoint),
|
||||
openaioption.WithHTTPClient(options.HTTPClient),
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Transcribe sends the audio to /audio/transcriptions.
|
||||
func (t *Transcriber) Transcribe(ctx context.Context, req stt.Request) (*stt.Response, error) {
|
||||
if strings.TrimSpace(req.Model) == "" {
|
||||
return nil, errors.New("model is required")
|
||||
}
|
||||
if req.Audio == nil {
|
||||
return nil, errors.New("audio is required")
|
||||
}
|
||||
|
||||
filename, contentType, err := normalizeAudioMetadata(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
params := openaisdk.AudioTranscriptionNewParams{
|
||||
File: openaisdk.File(req.Audio, filename, contentType),
|
||||
Model: openaisdk.AudioModel(req.Model),
|
||||
ResponseFormat: openaisdk.AudioResponseFormatJSON,
|
||||
}
|
||||
if req.Prompt != "" {
|
||||
params.Prompt = openaisdk.String(req.Prompt)
|
||||
}
|
||||
if req.Language != "" {
|
||||
params.Language = openaisdk.String(req.Language)
|
||||
}
|
||||
|
||||
resp, err := t.client.Audio.Transcriptions.New(ctx, params)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to send OpenAI transcription request")
|
||||
}
|
||||
return &stt.Response{
|
||||
Text: resp.Text,
|
||||
Language: resp.Language,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeEndpoint(endpoint string) (string, error) {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
endpoint = defaultEndpoint
|
||||
}
|
||||
if _, err := url.ParseRequestURI(endpoint); err != nil {
|
||||
return "", errors.Wrap(err, "invalid OpenAI endpoint")
|
||||
}
|
||||
return strings.TrimRight(endpoint, "/"), nil
|
||||
}
|
||||
|
||||
func normalizeAudioMetadata(req stt.Request) (string, string, error) {
|
||||
filename := strings.TrimSpace(req.Filename)
|
||||
if filename == "" {
|
||||
filename = "audio"
|
||||
}
|
||||
contentType := strings.TrimSpace(req.ContentType)
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
} else {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "invalid audio content type")
|
||||
}
|
||||
contentType = mediaType
|
||||
}
|
||||
return sanitizeFilename(filename), contentType, nil
|
||||
}
|
||||
|
||||
func sanitizeFilename(filename string) string {
|
||||
filename = strings.NewReplacer("\r", "_", "\n", "_").Replace(filename)
|
||||
if strings.TrimSpace(filename) == "" {
|
||||
return "audio"
|
||||
}
|
||||
return filename
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package openai_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/usememos/memos/internal/ai"
|
||||
"github.com/usememos/memos/internal/ai/stt"
|
||||
sttopenai "github.com/usememos/memos/internal/ai/stt/openai"
|
||||
)
|
||||
|
||||
func TestTranscribe(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodPost, r.Method)
|
||||
require.Equal(t, "/audio/transcriptions", r.URL.Path)
|
||||
require.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))
|
||||
require.NoError(t, r.ParseMultipartForm(10<<20))
|
||||
require.Equal(t, "gpt-4o-transcribe", r.FormValue("model"))
|
||||
require.Equal(t, "json", r.FormValue("response_format"))
|
||||
require.Equal(t, "domain words", r.FormValue("prompt"))
|
||||
require.Equal(t, "en", r.FormValue("language"))
|
||||
|
||||
file, header, err := r.FormFile("file")
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
require.Equal(t, "voice.wav", header.Filename)
|
||||
require.Equal(t, "audio/wav", header.Header.Get("Content-Type"))
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
|
||||
"text": "hello world",
|
||||
"language": "en",
|
||||
"duration": 1.5,
|
||||
}))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
transcriber, err := sttopenai.New(ai.ProviderConfig{
|
||||
Type: ai.ProviderOpenAI,
|
||||
Endpoint: server.URL,
|
||||
APIKey: "test-key",
|
||||
}, stt.ApplyOptions(nil))
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
response, err := transcriber.Transcribe(ctx, stt.Request{
|
||||
Model: "gpt-4o-transcribe",
|
||||
Filename: "voice.wav",
|
||||
ContentType: "audio/wav",
|
||||
Audio: strings.NewReader("RIFF"),
|
||||
Prompt: "domain words",
|
||||
Language: "en",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "hello world", response.Text)
|
||||
require.Equal(t, "en", response.Language)
|
||||
// Note: Duration intentionally omitted from stt.Response — not exposed in the new contract.
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package stt
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultHTTPTimeout = 2 * time.Minute
|
||||
|
||||
// Options is the resolved option set passed to provider implementations.
|
||||
type Options struct {
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// TranscriberOption customizes a Transcriber.
|
||||
type TranscriberOption func(*Options)
|
||||
|
||||
// WithHTTPClient overrides the HTTP client used by the transcriber.
|
||||
func WithHTTPClient(client *http.Client) TranscriberOption {
|
||||
return func(o *Options) {
|
||||
if client != nil {
|
||||
o.HTTPClient = client
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyOptions resolves a TranscriberOption slice into Options with defaults.
|
||||
func ApplyOptions(opts []TranscriberOption) Options {
|
||||
resolved := Options{HTTPClient: &http.Client{Timeout: defaultHTTPTimeout}}
|
||||
for _, apply := range opts {
|
||||
apply(&resolved)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Package stt defines the speech-to-text capability for AI providers.
|
||||
// Implementations call dedicated STT endpoints (e.g. OpenAI /audio/transcriptions)
|
||||
// and return deterministic transcription output. For multimodal LLMs that
|
||||
// happen to accept audio input, see internal/ai/audiollm.
|
||||
package stt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Transcriber transcribes audio to text using a provider's dedicated STT endpoint.
|
||||
type Transcriber interface {
|
||||
Transcribe(ctx context.Context, req Request) (*Response, error)
|
||||
}
|
||||
|
||||
// Request is the input to a transcription call.
|
||||
type Request struct {
|
||||
Audio io.Reader
|
||||
Size int64
|
||||
Filename string
|
||||
ContentType string // IANA media type, e.g. "audio/wav"
|
||||
Model string // provider-specific model id (e.g. "whisper-1", "gpt-4o-transcribe")
|
||||
Prompt string // soft spelling/vocabulary hint (Whisper "prompt" parameter)
|
||||
Language string // ISO 639-1, optional
|
||||
}
|
||||
|
||||
// Response is the output of a transcription call.
|
||||
type Response struct {
|
||||
Text string
|
||||
Language string // empty if provider did not return it
|
||||
Segments []Segment // empty unless provider returned timestamps
|
||||
}
|
||||
|
||||
// Segment is a timestamped portion of the transcript.
|
||||
type Segment struct {
|
||||
Text string
|
||||
Start float64
|
||||
End float64
|
||||
Speaker string // empty unless using a diarization-capable model
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package base
|
||||
|
||||
import "regexp"
|
||||
|
||||
var (
|
||||
UIDMatcher = regexp.MustCompile(`^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$`)
|
||||
)
|
||||
@@ -1,38 +0,0 @@
|
||||
package base
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUIDMatcher(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected bool
|
||||
}{
|
||||
{"", false},
|
||||
{"-abc123", false},
|
||||
{"012345678901234567890123456789", true},
|
||||
{"1abc-123", true},
|
||||
{"A123B456C789", true},
|
||||
{"a", true},
|
||||
{"ab", true},
|
||||
{"a*b&c", false},
|
||||
{"a--b", true},
|
||||
{"a-1b-2c", true},
|
||||
{"a1234567890123456789012345678901", true},
|
||||
{"abc123", true},
|
||||
{"abc123-", false},
|
||||
{"123e4567-e89b-12d3-a456-426614174000", true}, // UUID v4 from IDP
|
||||
{"a12345678901234567890123456789012345", true}, // 36 characters.
|
||||
{"a123456789012345678901234567890123456", false}, // 37 characters.
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.input, func(*testing.T) {
|
||||
result := UIDMatcher.MatchString(test.input)
|
||||
if result != test.expected {
|
||||
t.Errorf("For input '%s', expected %v but got %v", test.input, test.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,507 +0,0 @@
|
||||
# Email Plugin
|
||||
|
||||
SMTP email sending functionality for self-hosted Memos instances.
|
||||
|
||||
## Overview
|
||||
|
||||
This plugin provides a simple, reliable email sending interface following industry-standard SMTP protocols. It's designed for self-hosted environments where instance administrators configure their own email service, similar to platforms like GitHub, GitLab, and Discourse.
|
||||
|
||||
## Features
|
||||
|
||||
- Standard SMTP protocol support
|
||||
- TLS/STARTTLS and SSL/TLS encryption
|
||||
- HTML and plain text emails
|
||||
- Multiple recipients (To, Cc, Bcc)
|
||||
- Synchronous and asynchronous sending
|
||||
- Detailed error reporting with context
|
||||
- Works with all major email providers
|
||||
- Reply-To header support
|
||||
- RFC 5322 compliant message formatting
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Configure SMTP Settings
|
||||
|
||||
```go
|
||||
import "github.com/usememos/memos/internal/email"
|
||||
|
||||
config := &email.Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "your-email@gmail.com",
|
||||
SMTPPassword: "your-app-password",
|
||||
FromEmail: "noreply@yourdomain.com",
|
||||
FromName: "Memos",
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Create and Send Email
|
||||
|
||||
```go
|
||||
message := &email.Message{
|
||||
To: []string{"user@example.com"},
|
||||
Subject: "Welcome to Memos!",
|
||||
Body: "Thanks for signing up.",
|
||||
IsHTML: false,
|
||||
}
|
||||
|
||||
// Synchronous send (waits for result)
|
||||
err := email.Send(config, message)
|
||||
if err != nil {
|
||||
log.Printf("Failed to send email: %v", err)
|
||||
}
|
||||
|
||||
// Asynchronous send (returns immediately)
|
||||
email.SendAsync(config, message)
|
||||
```
|
||||
|
||||
## Provider Configuration
|
||||
|
||||
### Gmail
|
||||
|
||||
Requires an [App Password](https://support.google.com/accounts/answer/185833) (2FA must be enabled):
|
||||
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "your-email@gmail.com",
|
||||
SMTPPassword: "your-16-char-app-password",
|
||||
FromEmail: "your-email@gmail.com",
|
||||
FromName: "Memos",
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
**Alternative (SSL):**
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 465,
|
||||
SMTPUsername: "your-email@gmail.com",
|
||||
SMTPPassword: "your-16-char-app-password",
|
||||
FromEmail: "your-email@gmail.com",
|
||||
FromName: "Memos",
|
||||
UseSSL: true,
|
||||
}
|
||||
```
|
||||
|
||||
### SendGrid
|
||||
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPHost: "smtp.sendgrid.net",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "apikey",
|
||||
SMTPPassword: "your-sendgrid-api-key",
|
||||
FromEmail: "noreply@yourdomain.com",
|
||||
FromName: "Memos",
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
### AWS SES
|
||||
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPHost: "email-smtp.us-east-1.amazonaws.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "your-smtp-username",
|
||||
SMTPPassword: "your-smtp-password",
|
||||
FromEmail: "verified@yourdomain.com",
|
||||
FromName: "Memos",
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** Replace `us-east-1` with your AWS region. Email must be verified in SES.
|
||||
|
||||
### Mailgun
|
||||
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPHost: "smtp.mailgun.org",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "postmaster@yourdomain.com",
|
||||
SMTPPassword: "your-mailgun-smtp-password",
|
||||
FromEmail: "noreply@yourdomain.com",
|
||||
FromName: "Memos",
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
### Self-Hosted SMTP (Postfix, Exim, etc.)
|
||||
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPHost: "mail.yourdomain.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "username",
|
||||
SMTPPassword: "password",
|
||||
FromEmail: "noreply@yourdomain.com",
|
||||
FromName: "Memos",
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
## HTML Emails
|
||||
|
||||
```go
|
||||
message := &email.Message{
|
||||
To: []string{"user@example.com"},
|
||||
Subject: "Welcome to Memos!",
|
||||
Body: `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
</head>
|
||||
<body style="font-family: Arial, sans-serif;">
|
||||
<h1 style="color: #333;">Welcome to Memos!</h1>
|
||||
<p>We're excited to have you on board.</p>
|
||||
<a href="https://yourdomain.com" style="background-color: #4CAF50; color: white; padding: 10px 20px; text-decoration: none; border-radius: 5px;">Get Started</a>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
IsHTML: true,
|
||||
}
|
||||
|
||||
email.Send(config, message)
|
||||
```
|
||||
|
||||
## Multiple Recipients
|
||||
|
||||
```go
|
||||
message := &email.Message{
|
||||
To: []string{"user1@example.com", "user2@example.com"},
|
||||
Cc: []string{"manager@example.com"},
|
||||
Bcc: []string{"admin@example.com"},
|
||||
Subject: "Team Update",
|
||||
Body: "Important team announcement...",
|
||||
ReplyTo: "support@yourdomain.com",
|
||||
}
|
||||
|
||||
email.Send(config, message)
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Run Tests
|
||||
|
||||
```bash
|
||||
# All tests
|
||||
go test ./internal/email/... -v
|
||||
|
||||
# With coverage
|
||||
go test ./internal/email/... -v -cover
|
||||
|
||||
# With race detector
|
||||
go test ./internal/email/... -race
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
Create a simple test program:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"github.com/usememos/memos/internal/email"
|
||||
)
|
||||
|
||||
func main() {
|
||||
config := &email.Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "your-email@gmail.com",
|
||||
SMTPPassword: "your-app-password",
|
||||
FromEmail: "your-email@gmail.com",
|
||||
FromName: "Test",
|
||||
UseTLS: true,
|
||||
}
|
||||
|
||||
message := &email.Message{
|
||||
To: []string{"recipient@example.com"},
|
||||
Subject: "Test Email",
|
||||
Body: "This is a test email from Memos email plugin.",
|
||||
}
|
||||
|
||||
if err := email.Send(config, message); err != nil {
|
||||
log.Fatalf("Failed to send email: %v", err)
|
||||
}
|
||||
|
||||
log.Println("Email sent successfully!")
|
||||
}
|
||||
```
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
### 1. Use TLS/SSL Encryption
|
||||
|
||||
Always enable encryption in production:
|
||||
|
||||
```go
|
||||
// STARTTLS (port 587) - Recommended
|
||||
config.UseTLS = true
|
||||
|
||||
// SSL/TLS (port 465)
|
||||
config.UseSSL = true
|
||||
```
|
||||
|
||||
### 2. Secure Credential Storage
|
||||
|
||||
Never hardcode credentials. Use environment variables:
|
||||
|
||||
```go
|
||||
import "os"
|
||||
|
||||
config := &email.Config{
|
||||
SMTPHost: os.Getenv("SMTP_HOST"),
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: os.Getenv("SMTP_USERNAME"),
|
||||
SMTPPassword: os.Getenv("SMTP_PASSWORD"),
|
||||
FromEmail: os.Getenv("SMTP_FROM_EMAIL"),
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Use App-Specific Passwords
|
||||
|
||||
For Gmail and similar services, use app passwords instead of your main account password.
|
||||
|
||||
### 4. Validate and Sanitize Input
|
||||
|
||||
Always validate email addresses and sanitize content:
|
||||
|
||||
```go
|
||||
// Validate before sending
|
||||
if err := message.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Implement Rate Limiting
|
||||
|
||||
Prevent abuse by limiting email sending:
|
||||
|
||||
```go
|
||||
// Example using golang.org/x/time/rate
|
||||
limiter := rate.NewLimiter(rate.Every(time.Second), 10) // 10 emails per second
|
||||
|
||||
if !limiter.Allow() {
|
||||
return errors.New("rate limit exceeded")
|
||||
}
|
||||
```
|
||||
|
||||
### 6. Monitor and Log
|
||||
|
||||
Log email sending activity for security monitoring:
|
||||
|
||||
```go
|
||||
if err := email.Send(config, message); err != nil {
|
||||
slog.Error("Email send failed",
|
||||
slog.String("recipient", message.To[0]),
|
||||
slog.Any("error", err))
|
||||
}
|
||||
```
|
||||
|
||||
## Common Ports
|
||||
|
||||
| Port | Protocol | Security | Use Case |
|
||||
|------|----------|----------|----------|
|
||||
| **587** | SMTP + STARTTLS | Encrypted | **Recommended** for most providers |
|
||||
| **465** | SMTP over SSL/TLS | Encrypted | Alternative secure option |
|
||||
| **25** | SMTP | Unencrypted | Legacy, often blocked by ISPs |
|
||||
| **2525** | SMTP + STARTTLS | Encrypted | Alternative when 587 is blocked |
|
||||
|
||||
**Port 587 (STARTTLS)** is the recommended standard for modern SMTP:
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPPort: 587,
|
||||
UseTLS: true,
|
||||
}
|
||||
```
|
||||
|
||||
**Port 465 (SSL/TLS)** is the alternative:
|
||||
```go
|
||||
config := &email.Config{
|
||||
SMTPPort: 465,
|
||||
UseSSL: true,
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The package provides detailed, contextual errors:
|
||||
|
||||
```go
|
||||
err := email.Send(config, message)
|
||||
if err != nil {
|
||||
// Error messages include context:
|
||||
switch {
|
||||
case strings.Contains(err.Error(), "invalid email configuration"):
|
||||
// Configuration error (missing host, invalid port, etc.)
|
||||
log.Printf("Configuration error: %v", err)
|
||||
|
||||
case strings.Contains(err.Error(), "invalid email message"):
|
||||
// Message validation error (missing recipients, subject, body)
|
||||
log.Printf("Message error: %v", err)
|
||||
|
||||
case strings.Contains(err.Error(), "authentication failed"):
|
||||
// SMTP authentication failed (wrong credentials)
|
||||
log.Printf("Auth error: %v", err)
|
||||
|
||||
case strings.Contains(err.Error(), "failed to connect"):
|
||||
// Network/connection error
|
||||
log.Printf("Connection error: %v", err)
|
||||
|
||||
case strings.Contains(err.Error(), "recipient rejected"):
|
||||
// SMTP server rejected recipient
|
||||
log.Printf("Recipient error: %v", err)
|
||||
|
||||
default:
|
||||
log.Printf("Unknown error: %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Common Error Messages
|
||||
|
||||
```
|
||||
❌ "invalid email configuration: SMTP host is required"
|
||||
→ Fix: Set config.SMTPHost
|
||||
|
||||
❌ "invalid email configuration: SMTP port must be between 1 and 65535"
|
||||
→ Fix: Set valid config.SMTPPort (usually 587 or 465)
|
||||
|
||||
❌ "invalid email configuration: from email is required"
|
||||
→ Fix: Set config.FromEmail
|
||||
|
||||
❌ "invalid email message: at least one recipient is required"
|
||||
→ Fix: Set message.To with at least one email address
|
||||
|
||||
❌ "invalid email message: subject is required"
|
||||
→ Fix: Set message.Subject
|
||||
|
||||
❌ "invalid email message: body is required"
|
||||
→ Fix: Set message.Body
|
||||
|
||||
❌ "SMTP authentication failed"
|
||||
→ Fix: Check credentials (username/password)
|
||||
|
||||
❌ "failed to connect to SMTP server"
|
||||
→ Fix: Verify host/port, check firewall, ensure TLS/SSL settings match server
|
||||
```
|
||||
|
||||
### Async Error Handling
|
||||
|
||||
For async sending, errors are logged automatically:
|
||||
|
||||
```go
|
||||
email.SendAsync(config, message)
|
||||
// Errors logged as:
|
||||
// [WARN] Failed to send email asynchronously recipients=user@example.com error=...
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
### Required
|
||||
|
||||
- **Go 1.25+**
|
||||
- Standard library: `net/smtp`, `crypto/tls`
|
||||
- `github.com/pkg/errors` - Error wrapping with context
|
||||
|
||||
### No External SMTP Libraries
|
||||
|
||||
This plugin uses Go's standard `net/smtp` library for maximum compatibility and minimal dependencies.
|
||||
|
||||
## API Reference
|
||||
|
||||
### Types
|
||||
|
||||
#### `Config`
|
||||
```go
|
||||
type Config struct {
|
||||
SMTPHost string // SMTP server hostname
|
||||
SMTPPort int // SMTP server port
|
||||
SMTPUsername string // SMTP auth username
|
||||
SMTPPassword string // SMTP auth password
|
||||
FromEmail string // From email address
|
||||
FromName string // From display name (optional)
|
||||
UseTLS bool // Enable STARTTLS (port 587)
|
||||
UseSSL bool // Enable SSL/TLS (port 465)
|
||||
}
|
||||
```
|
||||
|
||||
#### `Message`
|
||||
```go
|
||||
type Message struct {
|
||||
To []string // Recipients
|
||||
Cc []string // CC recipients (optional)
|
||||
Bcc []string // BCC recipients (optional)
|
||||
Subject string // Email subject
|
||||
Body string // Email body (plain text or HTML)
|
||||
IsHTML bool // true for HTML, false for plain text
|
||||
ReplyTo string // Reply-To address (optional)
|
||||
}
|
||||
```
|
||||
|
||||
### Functions
|
||||
|
||||
#### `Send(config *Config, message *Message) error`
|
||||
Sends an email synchronously. Blocks until email is sent or error occurs.
|
||||
|
||||
#### `SendAsync(config *Config, message *Message)`
|
||||
Sends an email asynchronously in a goroutine. Returns immediately. Errors are logged.
|
||||
|
||||
#### `NewClient(config *Config) *Client`
|
||||
Creates a new SMTP client for advanced usage.
|
||||
|
||||
#### `Client.Send(message *Message) error`
|
||||
Sends email using the client's configuration.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
internal/email/
|
||||
├── config.go # SMTP configuration types
|
||||
├── message.go # Email message types and formatting
|
||||
├── client.go # SMTP client implementation
|
||||
├── email.go # High-level Send/SendAsync API
|
||||
├── doc.go # Package documentation
|
||||
└── *_test.go # Unit tests
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
Part of the Memos project. See main repository for license details.
|
||||
|
||||
## Contributing
|
||||
|
||||
This package follows the Memos contribution guidelines. Please ensure:
|
||||
|
||||
1. All code is tested (TDD approach)
|
||||
2. Tests pass: `go test ./internal/email/... -v`
|
||||
3. Code is formatted: `go fmt ./internal/email/...`
|
||||
4. No linting errors: `golangci-lint run ./internal/email/...`
|
||||
|
||||
## Support
|
||||
|
||||
For issues and questions:
|
||||
|
||||
- Memos GitHub Issues: https://github.com/usememos/memos/issues
|
||||
- Memos Documentation: https://usememos.com/docs
|
||||
|
||||
## Roadmap
|
||||
|
||||
Future enhancements may include:
|
||||
|
||||
- Email template system
|
||||
- Attachment support
|
||||
- Inline image embedding
|
||||
- Email queuing system
|
||||
- Delivery status tracking
|
||||
- Bounce handling
|
||||
@@ -1,173 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/smtp"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const smtpOperationTimeout = 15 * time.Second
|
||||
|
||||
// Client represents an SMTP email client.
|
||||
type Client struct {
|
||||
config *Config
|
||||
}
|
||||
|
||||
// NewClient creates a new email client with the given configuration.
|
||||
func NewClient(config *Config) *Client {
|
||||
return &Client{
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
// validateConfig validates the client configuration.
|
||||
func (c *Client) validateConfig() error {
|
||||
if c.config == nil {
|
||||
return errors.New("email configuration is required")
|
||||
}
|
||||
return c.config.Validate()
|
||||
}
|
||||
|
||||
// createAuth creates an SMTP auth mechanism if credentials are provided.
|
||||
func (c *Client) createAuth() smtp.Auth {
|
||||
if c.config.SMTPUsername == "" && c.config.SMTPPassword == "" {
|
||||
return nil
|
||||
}
|
||||
return smtp.PlainAuth("", c.config.SMTPUsername, c.config.SMTPPassword, c.config.SMTPHost)
|
||||
}
|
||||
|
||||
// createTLSConfig creates a TLS configuration for secure connections.
|
||||
func (c *Client) createTLSConfig() *tls.Config {
|
||||
return &tls.Config{
|
||||
ServerName: c.config.SMTPHost,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
}
|
||||
|
||||
// Send sends an email message via SMTP.
|
||||
func (c *Client) Send(message *Message) error {
|
||||
// Validate configuration
|
||||
if err := c.validateConfig(); err != nil {
|
||||
return errors.Wrap(err, "invalid email configuration")
|
||||
}
|
||||
|
||||
// Validate message
|
||||
if message == nil {
|
||||
return errors.New("message is required")
|
||||
}
|
||||
if err := message.Validate(); err != nil {
|
||||
return errors.Wrap(err, "invalid email message")
|
||||
}
|
||||
|
||||
// Format the message
|
||||
body := message.Format(c.config.FromEmail, c.config.FromName)
|
||||
|
||||
// Get all recipients
|
||||
recipients := message.GetAllRecipients()
|
||||
|
||||
// Create auth
|
||||
auth := c.createAuth()
|
||||
|
||||
// Send based on encryption type
|
||||
if c.config.UseSSL {
|
||||
return c.sendWithSSL(auth, recipients, body)
|
||||
}
|
||||
return c.sendWithTLS(auth, recipients, body)
|
||||
}
|
||||
|
||||
// sendWithTLS sends email using STARTTLS (port 587).
|
||||
func (c *Client) sendWithTLS(auth smtp.Auth, recipients []string, body string) error {
|
||||
serverAddr := c.config.GetServerAddress()
|
||||
|
||||
dialer := &net.Dialer{Timeout: smtpOperationTimeout}
|
||||
conn, err := dialer.Dial("tcp", serverAddr)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to connect to SMTP server: %s", serverAddr)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.SetDeadline(time.Now().Add(smtpOperationTimeout)); err != nil {
|
||||
return errors.Wrap(err, "failed to set SMTP connection deadline")
|
||||
}
|
||||
|
||||
client, err := smtp.NewClient(conn, c.config.SMTPHost)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create SMTP client")
|
||||
}
|
||||
defer client.Quit()
|
||||
|
||||
if c.config.UseTLS {
|
||||
if ok, _ := client.Extension("STARTTLS"); !ok {
|
||||
return errors.New("SMTP server does not support STARTTLS")
|
||||
}
|
||||
if err := client.StartTLS(c.createTLSConfig()); err != nil {
|
||||
return errors.Wrap(err, "failed to start SMTP STARTTLS")
|
||||
}
|
||||
}
|
||||
|
||||
return c.sendWithClient(client, auth, recipients, body)
|
||||
}
|
||||
|
||||
// sendWithSSL sends email using SSL/TLS (port 465).
|
||||
func (c *Client) sendWithSSL(auth smtp.Auth, recipients []string, body string) error {
|
||||
serverAddr := c.config.GetServerAddress()
|
||||
|
||||
// Create TLS connection
|
||||
tlsConfig := c.createTLSConfig()
|
||||
dialer := &net.Dialer{Timeout: smtpOperationTimeout}
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", serverAddr, tlsConfig)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to connect to SMTP server with SSL: %s", serverAddr)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.SetDeadline(time.Now().Add(smtpOperationTimeout)); err != nil {
|
||||
return errors.Wrap(err, "failed to set SMTP connection deadline")
|
||||
}
|
||||
|
||||
// Create SMTP client
|
||||
client, err := smtp.NewClient(conn, c.config.SMTPHost)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create SMTP client")
|
||||
}
|
||||
defer client.Quit()
|
||||
|
||||
return c.sendWithClient(client, auth, recipients, body)
|
||||
}
|
||||
|
||||
func (c *Client) sendWithClient(client *smtp.Client, auth smtp.Auth, recipients []string, body string) error {
|
||||
if auth != nil {
|
||||
if err := client.Auth(auth); err != nil {
|
||||
return errors.Wrap(err, "SMTP authentication failed")
|
||||
}
|
||||
}
|
||||
|
||||
// Set sender
|
||||
if err := client.Mail(c.config.FromEmail); err != nil {
|
||||
return errors.Wrap(err, "failed to set sender")
|
||||
}
|
||||
|
||||
// Set recipients
|
||||
for _, recipient := range recipients {
|
||||
if err := client.Rcpt(recipient); err != nil {
|
||||
return errors.Wrapf(err, "failed to set recipient: %s", recipient)
|
||||
}
|
||||
}
|
||||
|
||||
// Send message body
|
||||
writer, err := client.Data()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to send DATA command")
|
||||
}
|
||||
|
||||
if _, err := writer.Write([]byte(body)); err != nil {
|
||||
return errors.Wrap(err, "failed to write message body")
|
||||
}
|
||||
|
||||
if err := writer.Close(); err != nil {
|
||||
return errors.Wrap(err, "failed to close message writer")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNewClient(t *testing.T) {
|
||||
config := &Config{
|
||||
SMTPHost: "smtp.example.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "user@example.com",
|
||||
SMTPPassword: "password",
|
||||
FromEmail: "noreply@example.com",
|
||||
FromName: "Test App",
|
||||
UseTLS: true,
|
||||
}
|
||||
|
||||
client := NewClient(config)
|
||||
|
||||
assert.NotNil(t, client)
|
||||
assert.Equal(t, config, client.config)
|
||||
}
|
||||
|
||||
func TestClientValidateConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config *Config
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid config",
|
||||
config: &Config{
|
||||
SMTPHost: "smtp.example.com",
|
||||
SMTPPort: 587,
|
||||
FromEmail: "test@example.com",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "nil config",
|
||||
config: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid config",
|
||||
config: &Config{
|
||||
SMTPHost: "",
|
||||
SMTPPort: 587,
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := NewClient(tt.config)
|
||||
err := client.validateConfig()
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientSendValidation(t *testing.T) {
|
||||
config := &Config{
|
||||
SMTPHost: "smtp.example.com",
|
||||
SMTPPort: 587,
|
||||
FromEmail: "test@example.com",
|
||||
}
|
||||
client := NewClient(config)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
message *Message
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid message",
|
||||
message: &Message{
|
||||
To: []string{"recipient@example.com"},
|
||||
Subject: "Test",
|
||||
Body: "Test body",
|
||||
},
|
||||
wantErr: false, // Will fail on actual send, but passes validation
|
||||
},
|
||||
{
|
||||
name: "nil message",
|
||||
message: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid message",
|
||||
message: &Message{
|
||||
To: []string{},
|
||||
Subject: "Test",
|
||||
Body: "Test",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := client.Send(tt.message)
|
||||
// We expect validation errors for invalid messages
|
||||
// For valid messages, we'll get connection errors (which is expected in tests)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
// Should fail validation before attempting connection
|
||||
assert.NotContains(t, err.Error(), "dial")
|
||||
}
|
||||
// Note: We don't assert NoError for valid messages because
|
||||
// we don't have a real SMTP server in tests
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Config represents the SMTP configuration for email sending.
|
||||
// These settings should be provided by the self-hosted instance administrator.
|
||||
type Config struct {
|
||||
// SMTPHost is the SMTP server hostname (e.g., "smtp.gmail.com")
|
||||
SMTPHost string
|
||||
// SMTPPort is the SMTP server port (common: 587 for TLS, 465 for SSL, 25 for unencrypted)
|
||||
SMTPPort int
|
||||
// SMTPUsername is the SMTP authentication username (usually the email address)
|
||||
SMTPUsername string
|
||||
// SMTPPassword is the SMTP authentication password or app-specific password
|
||||
SMTPPassword string
|
||||
// FromEmail is the email address that will appear in the "From" field
|
||||
FromEmail string
|
||||
// FromName is the display name that will appear in the "From" field
|
||||
FromName string
|
||||
// UseTLS enables STARTTLS encryption (recommended for port 587)
|
||||
UseTLS bool
|
||||
// UseSSL enables SSL/TLS encryption (for port 465)
|
||||
UseSSL bool
|
||||
}
|
||||
|
||||
// Validate checks if the configuration is valid.
|
||||
func (c *Config) Validate() error {
|
||||
if c.SMTPHost == "" {
|
||||
return errors.New("SMTP host is required")
|
||||
}
|
||||
if c.SMTPPort <= 0 || c.SMTPPort > 65535 {
|
||||
return errors.New("SMTP port must be between 1 and 65535")
|
||||
}
|
||||
if c.FromEmail == "" {
|
||||
return errors.New("from email is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetServerAddress returns the SMTP server address in the format "host:port".
|
||||
func (c *Config) GetServerAddress() string {
|
||||
return fmt.Sprintf("%s:%d", c.SMTPHost, c.SMTPPort)
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestConfigValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config *Config
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid config",
|
||||
config: &Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "user@example.com",
|
||||
SMTPPassword: "password",
|
||||
FromEmail: "noreply@example.com",
|
||||
FromName: "Memos",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "missing host",
|
||||
config: &Config{
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "user@example.com",
|
||||
SMTPPassword: "password",
|
||||
FromEmail: "noreply@example.com",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid port",
|
||||
config: &Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 0,
|
||||
SMTPUsername: "user@example.com",
|
||||
SMTPPassword: "password",
|
||||
FromEmail: "noreply@example.com",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing from email",
|
||||
config: &Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUsername: "user@example.com",
|
||||
SMTPPassword: "password",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.config.Validate()
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigGetServerAddress(t *testing.T) {
|
||||
config := &Config{
|
||||
SMTPHost: "smtp.gmail.com",
|
||||
SMTPPort: 587,
|
||||
}
|
||||
|
||||
expected := "smtp.gmail.com:587"
|
||||
assert.Equal(t, expected, config.GetServerAddress())
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
// Package email provides SMTP email sending functionality for self-hosted Memos instances.
|
||||
//
|
||||
// This package is designed for self-hosted environments where instance administrators
|
||||
// configure their own SMTP servers. It follows industry-standard patterns used by
|
||||
// platforms like GitHub, GitLab, and Discourse.
|
||||
//
|
||||
// # Configuration
|
||||
//
|
||||
// The package requires SMTP server configuration provided by the instance administrator:
|
||||
//
|
||||
// config := &email.Config{
|
||||
// SMTPHost: "smtp.gmail.com",
|
||||
// SMTPPort: 587,
|
||||
// SMTPUsername: "your-email@gmail.com",
|
||||
// SMTPPassword: "your-app-password",
|
||||
// FromEmail: "noreply@yourdomain.com",
|
||||
// FromName: "Memos Notifications",
|
||||
// UseTLS: true,
|
||||
// }
|
||||
//
|
||||
// # Common SMTP Settings
|
||||
//
|
||||
// Gmail (requires App Password):
|
||||
// - Host: smtp.gmail.com
|
||||
// - Port: 587 (TLS) or 465 (SSL)
|
||||
// - Username: your-email@gmail.com
|
||||
// - UseTLS: true (for port 587) or UseSSL: true (for port 465)
|
||||
//
|
||||
// SendGrid:
|
||||
// - Host: smtp.sendgrid.net
|
||||
// - Port: 587
|
||||
// - Username: apikey
|
||||
// - Password: your-sendgrid-api-key
|
||||
// - UseTLS: true
|
||||
//
|
||||
// AWS SES:
|
||||
// - Host: email-smtp.[region].amazonaws.com
|
||||
// - Port: 587
|
||||
// - Username: your-smtp-username
|
||||
// - Password: your-smtp-password
|
||||
// - UseTLS: true
|
||||
//
|
||||
// Mailgun:
|
||||
// - Host: smtp.mailgun.org
|
||||
// - Port: 587
|
||||
// - Username: your-mailgun-smtp-username
|
||||
// - Password: your-mailgun-smtp-password
|
||||
// - UseTLS: true
|
||||
//
|
||||
// # Sending Email
|
||||
//
|
||||
// Synchronous (waits for completion):
|
||||
//
|
||||
// message := &email.Message{
|
||||
// To: []string{"user@example.com"},
|
||||
// Subject: "Welcome to Memos",
|
||||
// Body: "Thank you for joining!",
|
||||
// IsHTML: false,
|
||||
// }
|
||||
//
|
||||
// err := email.Send(config, message)
|
||||
// if err != nil {
|
||||
// // Handle error
|
||||
// }
|
||||
//
|
||||
// Asynchronous (returns immediately):
|
||||
//
|
||||
// email.SendAsync(config, message)
|
||||
// // Errors are logged but not returned
|
||||
//
|
||||
// # HTML Email
|
||||
//
|
||||
// message := &email.Message{
|
||||
// To: []string{"user@example.com"},
|
||||
// Subject: "Welcome!",
|
||||
// Body: "<html><body><h1>Welcome to Memos!</h1></body></html>",
|
||||
// IsHTML: true,
|
||||
// }
|
||||
//
|
||||
// # Security Considerations
|
||||
//
|
||||
// - Always use TLS (port 587) or SSL (port 465) for production
|
||||
// - Store SMTP credentials securely (environment variables or secrets management)
|
||||
// - Use app-specific passwords for services like Gmail
|
||||
// - Validate and sanitize email content to prevent injection attacks
|
||||
// - Rate limit email sending to prevent abuse
|
||||
//
|
||||
// # Error Handling
|
||||
//
|
||||
// The package returns descriptive errors for common issues:
|
||||
// - Configuration validation errors (missing host, invalid port, etc.)
|
||||
// - Message validation errors (missing recipients, subject, or body)
|
||||
// - Connection errors (cannot reach SMTP server)
|
||||
// - Authentication errors (invalid credentials)
|
||||
// - SMTP protocol errors (recipient rejected, etc.)
|
||||
//
|
||||
// All errors are wrapped with context using github.com/pkg/errors for better debugging.
|
||||
package email
|
||||
@@ -1,61 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type asyncEmailRequest struct {
|
||||
config *Config
|
||||
message *Message
|
||||
}
|
||||
|
||||
var asyncEmailQueue = make(chan asyncEmailRequest, 128)
|
||||
|
||||
func init() {
|
||||
for range 2 {
|
||||
go func() {
|
||||
for request := range asyncEmailQueue {
|
||||
if err := Send(request.config, request.message); err != nil {
|
||||
recipients := ""
|
||||
if request.message != nil && len(request.message.To) > 0 {
|
||||
recipients = request.message.To[0]
|
||||
if len(request.message.To) > 1 {
|
||||
recipients += " and others"
|
||||
}
|
||||
}
|
||||
|
||||
slog.Warn("Failed to send email asynchronously",
|
||||
slog.String("recipients", recipients),
|
||||
slog.Any("error", err))
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// Send sends an email synchronously.
|
||||
// Returns an error if the email fails to send.
|
||||
func Send(config *Config, message *Message) error {
|
||||
if config == nil {
|
||||
return errors.New("email configuration is required")
|
||||
}
|
||||
if message == nil {
|
||||
return errors.New("email message is required")
|
||||
}
|
||||
|
||||
client := NewClient(config)
|
||||
return client.Send(message)
|
||||
}
|
||||
|
||||
// SendAsync sends an email asynchronously.
|
||||
// It enqueues the message for bounded asynchronous sending and does not wait for the response.
|
||||
// Any errors are logged but not returned.
|
||||
func SendAsync(config *Config, message *Message) {
|
||||
select {
|
||||
case asyncEmailQueue <- asyncEmailRequest{config: config, message: message}:
|
||||
default:
|
||||
slog.Warn("Dropped email because the async queue is full")
|
||||
}
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func TestSend(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer listener.Close()
|
||||
|
||||
connectionClosed := make(chan struct{})
|
||||
go func() {
|
||||
defer close(connectionClosed)
|
||||
conn, err := listener.Accept()
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
config := &Config{
|
||||
SMTPHost: "127.0.0.1",
|
||||
SMTPPort: listener.Addr().(*net.TCPAddr).Port,
|
||||
FromEmail: "test@example.com",
|
||||
}
|
||||
|
||||
message := &Message{
|
||||
To: []string{"recipient@example.com"},
|
||||
Subject: "Test",
|
||||
Body: "Test body",
|
||||
}
|
||||
|
||||
// The local server closes before sending an SMTP greeting, proving Send made
|
||||
// it through validation and attempted to create an SMTP client.
|
||||
err = Send(config, message)
|
||||
select {
|
||||
case <-connectionClosed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("SMTP listener did not accept a connection")
|
||||
}
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to create SMTP client")
|
||||
}
|
||||
|
||||
func TestSendValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config *Config
|
||||
message *Message
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{
|
||||
name: "nil config",
|
||||
config: nil,
|
||||
message: &Message{To: []string{"test@example.com"}, Subject: "Test", Body: "Test"},
|
||||
wantErr: true,
|
||||
errMsg: "configuration is required",
|
||||
},
|
||||
{
|
||||
name: "nil message",
|
||||
config: &Config{SMTPHost: "smtp.example.com", SMTPPort: 587, FromEmail: "from@example.com"},
|
||||
message: nil,
|
||||
wantErr: true,
|
||||
errMsg: "message is required",
|
||||
},
|
||||
{
|
||||
name: "invalid config",
|
||||
config: &Config{
|
||||
SMTPHost: "",
|
||||
SMTPPort: 587,
|
||||
},
|
||||
message: &Message{To: []string{"test@example.com"}, Subject: "Test", Body: "Test"},
|
||||
wantErr: true,
|
||||
errMsg: "invalid email configuration",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := Send(tt.config, tt.message)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.errMsg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendAsync(t *testing.T) {
|
||||
config := &Config{
|
||||
SMTPHost: "smtp.example.com",
|
||||
SMTPPort: 587,
|
||||
FromEmail: "test@example.com",
|
||||
}
|
||||
|
||||
message := &Message{
|
||||
To: []string{"recipient@example.com"},
|
||||
Subject: "Test Async",
|
||||
Body: "Test async body",
|
||||
}
|
||||
|
||||
// SendAsync should not block
|
||||
start := time.Now()
|
||||
SendAsync(config, message)
|
||||
duration := time.Since(start)
|
||||
|
||||
// Should return almost immediately (< 100ms)
|
||||
assert.Less(t, duration, 100*time.Millisecond)
|
||||
|
||||
// Give goroutine time to start
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
func TestSendAsyncConcurrent(t *testing.T) {
|
||||
config := &Config{
|
||||
SMTPHost: "smtp.example.com",
|
||||
SMTPPort: 587,
|
||||
FromEmail: "test@example.com",
|
||||
}
|
||||
|
||||
g := errgroup.Group{}
|
||||
count := 5
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
g.Go(func() error {
|
||||
message := &Message{
|
||||
To: []string{"recipient@example.com"},
|
||||
Subject: "Concurrent Test",
|
||||
Body: "Test body",
|
||||
}
|
||||
SendAsync(config, message)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
t.Fatalf("SendAsync calls failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Message represents an email message to be sent.
|
||||
type Message struct {
|
||||
To []string // Required: recipient email addresses
|
||||
Cc []string // Optional: carbon copy recipients
|
||||
Bcc []string // Optional: blind carbon copy recipients
|
||||
Subject string // Required: email subject
|
||||
Body string // Required: email body content
|
||||
IsHTML bool // Whether the body is HTML (default: false for plain text)
|
||||
ReplyTo string // Optional: reply-to address
|
||||
}
|
||||
|
||||
// Validate checks that the message has all required fields.
|
||||
func (m *Message) Validate() error {
|
||||
if len(m.To) == 0 {
|
||||
return errors.New("at least one recipient is required")
|
||||
}
|
||||
if m.Subject == "" {
|
||||
return errors.New("subject is required")
|
||||
}
|
||||
if m.Body == "" {
|
||||
return errors.New("body is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Format creates an RFC 5322 formatted email message.
|
||||
func (m *Message) Format(fromEmail, fromName string) string {
|
||||
var sb strings.Builder
|
||||
fromEmail = sanitizeEmailHeaderValue(fromEmail)
|
||||
fromName = sanitizeEmailHeaderValue(fromName)
|
||||
to := sanitizeEmailHeaderValues(m.To)
|
||||
cc := sanitizeEmailHeaderValues(m.Cc)
|
||||
replyTo := sanitizeEmailHeaderValue(m.ReplyTo)
|
||||
subject := sanitizeEmailHeaderValue(m.Subject)
|
||||
|
||||
// From header
|
||||
if fromName != "" {
|
||||
fmt.Fprintf(&sb, "From: %s <%s>\r\n", fromName, fromEmail)
|
||||
} else {
|
||||
fmt.Fprintf(&sb, "From: %s\r\n", fromEmail)
|
||||
}
|
||||
|
||||
// To header
|
||||
fmt.Fprintf(&sb, "To: %s\r\n", strings.Join(to, ", "))
|
||||
|
||||
// Cc header (optional)
|
||||
if len(cc) > 0 {
|
||||
fmt.Fprintf(&sb, "Cc: %s\r\n", strings.Join(cc, ", "))
|
||||
}
|
||||
|
||||
// Reply-To header (optional)
|
||||
if replyTo != "" {
|
||||
fmt.Fprintf(&sb, "Reply-To: %s\r\n", replyTo)
|
||||
}
|
||||
|
||||
// Subject header
|
||||
fmt.Fprintf(&sb, "Subject: %s\r\n", subject)
|
||||
|
||||
// Date header (RFC 5322 format)
|
||||
fmt.Fprintf(&sb, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
|
||||
|
||||
// MIME headers
|
||||
sb.WriteString("MIME-Version: 1.0\r\n")
|
||||
|
||||
// Content-Type header
|
||||
if m.IsHTML {
|
||||
sb.WriteString("Content-Type: text/html; charset=utf-8\r\n")
|
||||
} else {
|
||||
sb.WriteString("Content-Type: text/plain; charset=utf-8\r\n")
|
||||
}
|
||||
|
||||
// Empty line separating headers from body
|
||||
sb.WriteString("\r\n")
|
||||
|
||||
// Body
|
||||
sb.WriteString(m.Body)
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func sanitizeEmailHeaderValue(value string) string {
|
||||
value = strings.NewReplacer("\r", " ", "\n", " ").Replace(value)
|
||||
return strings.Join(strings.Fields(value), " ")
|
||||
}
|
||||
|
||||
func sanitizeEmailHeaderValues(values []string) []string {
|
||||
sanitized := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
sanitized = append(sanitized, sanitizeEmailHeaderValue(value))
|
||||
}
|
||||
return sanitized
|
||||
}
|
||||
|
||||
// GetAllRecipients returns all recipients (To, Cc, Bcc) as a single slice.
|
||||
func (m *Message) GetAllRecipients() []string {
|
||||
var recipients []string
|
||||
recipients = append(recipients, m.To...)
|
||||
recipients = append(recipients, m.Cc...)
|
||||
recipients = append(recipients, m.Bcc...)
|
||||
return recipients
|
||||
}
|
||||
@@ -1,204 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMessageValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
msg Message
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid message",
|
||||
msg: Message{
|
||||
To: []string{"user@example.com"},
|
||||
Subject: "Test Subject",
|
||||
Body: "Test Body",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "no recipients",
|
||||
msg: Message{
|
||||
To: []string{},
|
||||
Subject: "Test Subject",
|
||||
Body: "Test Body",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "no subject",
|
||||
msg: Message{
|
||||
To: []string{"user@example.com"},
|
||||
Subject: "",
|
||||
Body: "Test Body",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "no body",
|
||||
msg: Message{
|
||||
To: []string{"user@example.com"},
|
||||
Subject: "Test Subject",
|
||||
Body: "",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "multiple recipients",
|
||||
msg: Message{
|
||||
To: []string{"user1@example.com", "user2@example.com"},
|
||||
Cc: []string{"cc@example.com"},
|
||||
Subject: "Test Subject",
|
||||
Body: "Test Body",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.msg.Validate()
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageFormatPlainText(t *testing.T) {
|
||||
msg := Message{
|
||||
To: []string{"user@example.com"},
|
||||
Subject: "Test Subject",
|
||||
Body: "Test Body",
|
||||
IsHTML: false,
|
||||
}
|
||||
|
||||
formatted := msg.Format("sender@example.com", "Sender Name")
|
||||
|
||||
// Check required headers
|
||||
if !strings.Contains(formatted, "From: Sender Name <sender@example.com>") {
|
||||
t.Error("Missing or incorrect From header")
|
||||
}
|
||||
if !strings.Contains(formatted, "To: user@example.com") {
|
||||
t.Error("Missing or incorrect To header")
|
||||
}
|
||||
if !strings.Contains(formatted, "Subject: Test Subject") {
|
||||
t.Error("Missing or incorrect Subject header")
|
||||
}
|
||||
if !strings.Contains(formatted, "Content-Type: text/plain; charset=utf-8") {
|
||||
t.Error("Missing or incorrect Content-Type header for plain text")
|
||||
}
|
||||
if !strings.Contains(formatted, "Test Body") {
|
||||
t.Error("Missing message body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageFormatHTML(t *testing.T) {
|
||||
msg := Message{
|
||||
To: []string{"user@example.com"},
|
||||
Subject: "Test Subject",
|
||||
Body: "<html><body>Test Body</body></html>",
|
||||
IsHTML: true,
|
||||
}
|
||||
|
||||
formatted := msg.Format("sender@example.com", "Sender Name")
|
||||
|
||||
// Check HTML content-type
|
||||
if !strings.Contains(formatted, "Content-Type: text/html; charset=utf-8") {
|
||||
t.Error("Missing or incorrect Content-Type header for HTML")
|
||||
}
|
||||
if !strings.Contains(formatted, "<html><body>Test Body</body></html>") {
|
||||
t.Error("Missing HTML body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageFormatMultipleRecipients(t *testing.T) {
|
||||
msg := Message{
|
||||
To: []string{"user1@example.com", "user2@example.com"},
|
||||
Cc: []string{"cc1@example.com", "cc2@example.com"},
|
||||
Bcc: []string{"bcc@example.com"},
|
||||
Subject: "Test Subject",
|
||||
Body: "Test Body",
|
||||
ReplyTo: "reply@example.com",
|
||||
}
|
||||
|
||||
formatted := msg.Format("sender@example.com", "Sender Name")
|
||||
|
||||
// Check To header formatting
|
||||
if !strings.Contains(formatted, "To: user1@example.com, user2@example.com") {
|
||||
t.Error("Missing or incorrect To header with multiple recipients")
|
||||
}
|
||||
// Check Cc header formatting
|
||||
if !strings.Contains(formatted, "Cc: cc1@example.com, cc2@example.com") {
|
||||
t.Error("Missing or incorrect Cc header")
|
||||
}
|
||||
// Bcc should NOT appear in the formatted message
|
||||
if strings.Contains(formatted, "Bcc:") {
|
||||
t.Error("Bcc header should not appear in formatted message")
|
||||
}
|
||||
// Check Reply-To header
|
||||
if !strings.Contains(formatted, "Reply-To: reply@example.com") {
|
||||
t.Error("Missing or incorrect Reply-To header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageFormatSanitizesHeaderValues(t *testing.T) {
|
||||
msg := Message{
|
||||
To: []string{"user@example.com\r\nX-Injected-To: bad"},
|
||||
Cc: []string{"cc@example.com\r\nX-Injected-Cc: bad"},
|
||||
Subject: "Test\r\nX-Injected-Subject: bad",
|
||||
Body: "Test Body",
|
||||
ReplyTo: "reply@example.com\r\nX-Injected-Reply-To: bad",
|
||||
}
|
||||
|
||||
formatted := msg.Format("sender@example.com\r\nX-Injected-From: bad", "Sender\r\nX-Injected-Name: bad")
|
||||
headers := strings.SplitN(formatted, "\r\n\r\n", 2)[0]
|
||||
|
||||
if strings.Contains(headers, "\r\nX-Injected") {
|
||||
t.Fatalf("header value injection was not sanitized:\n%s", headers)
|
||||
}
|
||||
if !strings.Contains(headers, "Subject: Test X-Injected-Subject: bad") {
|
||||
t.Error("subject header was not normalized")
|
||||
}
|
||||
if !strings.Contains(headers, "From: Sender X-Injected-Name: bad <sender@example.com X-Injected-From: bad>") {
|
||||
t.Error("from header was not normalized")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAllRecipients(t *testing.T) {
|
||||
msg := Message{
|
||||
To: []string{"user1@example.com", "user2@example.com"},
|
||||
Cc: []string{"cc@example.com"},
|
||||
Bcc: []string{"bcc@example.com"},
|
||||
}
|
||||
|
||||
recipients := msg.GetAllRecipients()
|
||||
|
||||
// Should have all 4 recipients
|
||||
if len(recipients) != 4 {
|
||||
t.Errorf("GetAllRecipients() returned %d recipients, want 4", len(recipients))
|
||||
}
|
||||
|
||||
// Check all recipients are present
|
||||
expectedRecipients := map[string]bool{
|
||||
"user1@example.com": true,
|
||||
"user2@example.com": true,
|
||||
"cc@example.com": true,
|
||||
"bcc@example.com": true,
|
||||
}
|
||||
|
||||
for _, recipient := range recipients {
|
||||
if !expectedRecipients[recipient] {
|
||||
t.Errorf("Unexpected recipient: %s", recipient)
|
||||
}
|
||||
delete(expectedRecipients, recipient)
|
||||
}
|
||||
|
||||
if len(expectedRecipients) > 0 {
|
||||
t.Error("Not all expected recipients were returned")
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
# Maintaining the Memo Filter Engine
|
||||
|
||||
The engine is memo-specific; any future field or behavior changes must stay
|
||||
consistent with the memo schema and store implementations. Use this guide when
|
||||
extending or debugging the package.
|
||||
|
||||
## Adding a New Memo Field
|
||||
|
||||
1. **Update the schema**
|
||||
- Add the field entry in `schema.go`.
|
||||
- Define the backing column (`Column`), JSON path (if applicable), type, and
|
||||
allowed operators.
|
||||
- Include the CEL variable in `EnvOptions`.
|
||||
2. **Adjust parser or renderer (if needed)**
|
||||
- For non-scalar fields (JSON booleans, lists), add handling in
|
||||
`parser.go` or extend the renderer helpers.
|
||||
- Keep validation in the parser (e.g., reject unsupported operators).
|
||||
3. **Write a golden test**
|
||||
- Extend the dialect-specific memo filter tests under
|
||||
`store/db/{sqlite,mysql,postgres}/memo_filter_test.go` with a case that
|
||||
exercises the new field.
|
||||
4. **Run `go test ./...`** to ensure the SQL output matches expectations across
|
||||
all dialects.
|
||||
|
||||
## Supporting Dialect Nuances
|
||||
|
||||
- Centralize differences inside `render.go`. If a new dialect-specific behavior
|
||||
emerges (e.g., JSON operators), add the logic there rather than leaking it
|
||||
into store code.
|
||||
- Use the renderer helpers (`jsonExtractExpr`, `jsonArrayExpr`, etc.) rather than
|
||||
sprinkling ad-hoc SQL strings.
|
||||
- When placeholders change, adjust `addArg` so that argument numbering stays in
|
||||
sync with store queries.
|
||||
|
||||
## Debugging Tips
|
||||
|
||||
- **Parser errors** – Most originate in `buildCondition` or schema validation.
|
||||
Enable logging around `parser.go` when diagnosing unknown identifier/operator
|
||||
messages.
|
||||
- **Renderer output** – Temporary printf/log statements in `renderCondition` help
|
||||
identify which IR node produced unexpected SQL.
|
||||
- **Store integration** – Ensure drivers call `filter.DefaultEngine()` exactly once
|
||||
per process; the singleton caches the parsed CEL environment.
|
||||
|
||||
## Testing Checklist
|
||||
|
||||
- `go test ./store/...` ensures all dialect tests consume the engine correctly.
|
||||
- Add targeted unit tests whenever new IR nodes or renderer paths are introduced.
|
||||
- When changing boolean or JSON handling, verify all three dialect test suites
|
||||
(SQLite, MySQL, Postgres) to avoid regression.
|
||||
@@ -1,105 +0,0 @@
|
||||
# Memo Filter Engine
|
||||
|
||||
This package houses the memo-only filter engine that turns standard CEL syntax
|
||||
into SQL fragments for the subset of expressions supported by the memo schema.
|
||||
The engine follows a three phase pipeline inspired by systems
|
||||
such as Calcite or Prisma:
|
||||
|
||||
1. **Parsing** – CEL expressions are parsed with `cel-go` and validated against
|
||||
the memo-specific environment declared in `schema.go`. Only fields that
|
||||
exist in the schema can surface in the filter, and non-standard legacy
|
||||
coercions are rejected.
|
||||
2. **Normalization** – the raw CEL AST is converted into an intermediate
|
||||
representation (IR) defined in `ir.go`. The IR is a dialect-agnostic tree of
|
||||
conditions (logical operators, comparisons, list membership, etc.). This
|
||||
step enforces schema rules (e.g. operator compatibility, type checks).
|
||||
3. **Rendering** – the renderer in `render.go` walks the IR and produces a SQL
|
||||
fragment plus placeholder arguments tailored to a target dialect
|
||||
(`sqlite`, `mysql`, or `postgres`). Dialect differences such as JSON access,
|
||||
boolean semantics, placeholders, and `LIKE` vs `ILIKE` are encapsulated in
|
||||
renderer helpers.
|
||||
|
||||
The entry point is `filter.DefaultEngine()` from `engine.go`. It lazily constructs
|
||||
an `Engine` configured with the memo schema and exposes:
|
||||
|
||||
```go
|
||||
engine, _ := filter.DefaultEngine()
|
||||
stmt, _ := engine.CompileToStatement(ctx, `has_task_list && visibility == "PUBLIC"`, filter.RenderOptions{
|
||||
Dialect: filter.DialectPostgres,
|
||||
})
|
||||
// stmt.SQL -> "((memo.payload->'property'->>'hasTaskList')::boolean IS TRUE AND memo.visibility = $1)"
|
||||
// stmt.Args -> ["PUBLIC"]
|
||||
```
|
||||
|
||||
## Core Files
|
||||
|
||||
| File | Responsibility |
|
||||
| ------------- | ------------------------------------------------------------------------------- |
|
||||
| `schema.go` | Declares memo fields, their types, backing columns, CEL environment options |
|
||||
| `ir.go` | IR node definitions used across the pipeline |
|
||||
| `parser.go` | Converts CEL `Expr` into IR while applying schema validation |
|
||||
| `render.go` | Translates IR into SQL, handling dialect-specific behavior |
|
||||
| `engine.go` | Glue between the phases; exposes `Compile`, `CompileToStatement`, and `DefaultEngine` |
|
||||
| `helpers.go` | Convenience helpers for store integration (appending conditions) |
|
||||
|
||||
## SQL Generation Notes
|
||||
|
||||
- **Placeholders** — `?` is used for SQLite/MySQL, `$n` for Postgres. The renderer
|
||||
tracks offsets to compose queries with pre-existing arguments.
|
||||
- **JSON Fields** — Memo metadata lives in `memo.payload`. The renderer handles
|
||||
`JSON_EXTRACT`/`json_extract`/`->`/`->>` variations and boolean coercion.
|
||||
- **Time Fields** — `created_ts`, `updated_ts`, and attachment `create_time` are
|
||||
CEL `timestamp` values. Express instants with the `now` variable,
|
||||
`duration("…")` (e.g. `created_ts >= now - duration("24h")`), or
|
||||
`timestamp("2006-01-02T15:04:05Z")` / `timestamp(<epoch-seconds>)`. These fold
|
||||
to epoch seconds at compile time — `now` is frozen once per compile (injectable
|
||||
for tests via the engine clock) — so the backing columns stay unchanged.
|
||||
- **Tag Operations** — `tag in [...]` and `"tag" in tags` become JSON array
|
||||
predicates. SQLite uses `LIKE` patterns, MySQL uses `JSON_CONTAINS`, and
|
||||
Postgres uses `@>`.
|
||||
- **Boolean Flags** — Fields such as `has_task_list` render as `IS TRUE` equality
|
||||
checks, or comparisons against `CAST('true' AS JSON)` depending on the dialect.
|
||||
- **String Matching** — `content.contains(x)`, `content.startsWith(x)`, and
|
||||
`content.endsWith(x)` render as case-insensitive `LIKE`/`ILIKE` with LIKE
|
||||
metacharacters (`%`, `_`, `\`) escaped. Available on scalar string fields whose
|
||||
schema sets `SupportsContains` (memo `content`; attachment `filename`,
|
||||
`mime_type`).
|
||||
- **Regex** — `field.matches("pattern")` renders to `~` (Postgres) or `REGEXP`
|
||||
(MySQL/SQLite). SQLite uses a Go-backed `regexp` function registered in
|
||||
`store/db/sqlite/functions.go`. Patterns are validated at compile time against
|
||||
Go's RE2 via `cel.ValidateRegexLiterals()`. **Caveat:** regex *syntax* differs
|
||||
per engine (Go RE2 on SQLite, POSIX ERE on Postgres, ICU on MySQL 8.0+), so
|
||||
engine-specific patterns may not be portable.
|
||||
- **Tag `all()` / `exists_one()`** — `tags.all(t, <pred>)` matches only non-empty
|
||||
tag sets where every element satisfies the predicate; `tags.exists_one(t,
|
||||
<pred>)` matches when exactly one element does (`COUNT(...) = 1`). Both iterate
|
||||
per-element (`json_each` / `jsonb_array_elements_text` / `JSON_TABLE`).
|
||||
- **Timestamp Accessors** — `created_ts.getFullYear()`, `getMonth()`, `getDate()`,
|
||||
`getDayOfMonth()`, `getDayOfWeek()`, `getDayOfYear()`, `getHours()`,
|
||||
`getMinutes()`, `getSeconds()` render to date-part extraction (`strftime` /
|
||||
`EXTRACT` / `YEAR`/`MONTH`/…). Results are normalized to CEL's base (0-based
|
||||
month, 0-based day-of-week with 0 = Sunday). The same accessors on `now` fold
|
||||
to literal date parts of the frozen evaluation time (UTC), so saved filters
|
||||
like `created_ts.getMonth() == now.getMonth() && created_ts.getDate() ==
|
||||
now.getDate()` ("on this day") re-resolve on every compile. Extraction is UTC
|
||||
on SQLite/Postgres (epoch columns); on MySQL the `TIMESTAMP` column is read in
|
||||
the session time zone. A timezone argument is not supported.
|
||||
- **Set Operations** — `ext.Sets()`: `sets.contains(tags, [...])`,
|
||||
`sets.intersects(tags, [...])`, and `sets.equivalent(tags, [...])` desugar to
|
||||
exact-membership checks (AND / OR of `"v" in tags`); `equivalent` adds a
|
||||
`size(tags)` length check (relies on tags being a set).
|
||||
- **`size()`** — `size(tags)` renders to JSON array length; `size(content)` (and
|
||||
other string fields) render to `LENGTH` / `CHAR_LENGTH` (MySQL) for code-point
|
||||
counts.
|
||||
- **Arithmetic** — `+`, `-`, `*`, `/`, `%` constant-fold on literal/`now`/`duration`
|
||||
operands (division and modulo guard against a zero divisor).
|
||||
|
||||
## Typical Integration
|
||||
|
||||
1. Fetch the engine with `filter.DefaultEngine()`.
|
||||
2. Call `CompileToStatement` using the appropriate dialect enum.
|
||||
3. Append the emitted SQL fragment/args to the existing `WHERE` clause.
|
||||
4. Execute the resulting query through the store driver.
|
||||
|
||||
The `helpers.AppendConditions` helper encapsulates steps 2–3 when a driver needs
|
||||
to process an array of filters.
|
||||
@@ -1,124 +0,0 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/cel-go/cel"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Engine parses CEL filters into a dialect-agnostic condition tree.
|
||||
type Engine struct {
|
||||
schema Schema
|
||||
env *cel.Env
|
||||
// nowFunc resolves the value of the `now` variable. It is frozen once per
|
||||
// Compile so a single filter sees a single instant, and is overridable in
|
||||
// tests for deterministic folding.
|
||||
nowFunc func() time.Time
|
||||
}
|
||||
|
||||
// NewEngine builds a new Engine for the provided schema.
|
||||
func NewEngine(schema Schema) (*Engine, error) {
|
||||
env, err := cel.NewEnv(schema.EnvOptions...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create CEL environment")
|
||||
}
|
||||
return &Engine{
|
||||
schema: schema,
|
||||
env: env,
|
||||
nowFunc: time.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Program stores a compiled filter condition.
|
||||
type Program struct {
|
||||
schema Schema
|
||||
condition Condition
|
||||
}
|
||||
|
||||
// ConditionTree exposes the underlying condition tree.
|
||||
func (p *Program) ConditionTree() Condition {
|
||||
return p.condition
|
||||
}
|
||||
|
||||
// Compile parses the filter string into an executable program.
|
||||
func (e *Engine) Compile(_ context.Context, filter string) (*Program, error) {
|
||||
if strings.TrimSpace(filter) == "" {
|
||||
return nil, errors.New("filter expression is empty")
|
||||
}
|
||||
|
||||
ast, issues := e.env.Compile(filter)
|
||||
if issues != nil && issues.Err() != nil {
|
||||
return nil, errors.Wrap(issues.Err(), "failed to compile filter")
|
||||
}
|
||||
parsed, err := cel.AstToParsedExpr(ast)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to convert AST")
|
||||
}
|
||||
|
||||
cond, err := buildCondition(parsed.GetExpr(), parseContext{schema: e.schema, now: e.nowFunc()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Program{
|
||||
schema: e.schema,
|
||||
condition: cond,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CompileToStatement compiles and renders the filter in a single step.
|
||||
func (e *Engine) CompileToStatement(ctx context.Context, filter string, opts RenderOptions) (Statement, error) {
|
||||
program, err := e.Compile(ctx, filter)
|
||||
if err != nil {
|
||||
return Statement{}, err
|
||||
}
|
||||
return program.Render(opts)
|
||||
}
|
||||
|
||||
// RenderOptions configure SQL rendering.
|
||||
type RenderOptions struct {
|
||||
Dialect DialectName
|
||||
PlaceholderOffset int
|
||||
DisableNullChecks bool
|
||||
}
|
||||
|
||||
// Statement contains the rendered SQL fragment and its args.
|
||||
type Statement struct {
|
||||
SQL string
|
||||
Args []any
|
||||
}
|
||||
|
||||
// Render converts the program into a dialect-specific SQL fragment.
|
||||
func (p *Program) Render(opts RenderOptions) (Statement, error) {
|
||||
renderer := newRenderer(p.schema, opts)
|
||||
return renderer.Render(p.condition)
|
||||
}
|
||||
|
||||
var (
|
||||
defaultOnce sync.Once
|
||||
defaultInst *Engine
|
||||
defaultErr error
|
||||
defaultAttachmentOnce sync.Once
|
||||
defaultAttachmentInst *Engine
|
||||
defaultAttachmentErr error
|
||||
)
|
||||
|
||||
// DefaultEngine returns the process-wide memo filter engine.
|
||||
func DefaultEngine() (*Engine, error) {
|
||||
defaultOnce.Do(func() {
|
||||
defaultInst, defaultErr = NewEngine(NewSchema())
|
||||
})
|
||||
return defaultInst, defaultErr
|
||||
}
|
||||
|
||||
// DefaultAttachmentEngine returns the process-wide attachment filter engine.
|
||||
func DefaultAttachmentEngine() (*Engine, error) {
|
||||
defaultAttachmentOnce.Do(func() {
|
||||
defaultAttachmentInst, defaultAttachmentErr = NewEngine(NewAttachmentSchema())
|
||||
})
|
||||
return defaultAttachmentInst, defaultAttachmentErr
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCompileAcceptsStandardTagEqualityPredicate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = engine.Compile(context.Background(), `tags.exists(t, t == "1231")`)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestCompileRejectsLegacyNumericLogicalOperand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = engine.Compile(context.Background(), `pinned && 1`)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "failed to compile filter")
|
||||
}
|
||||
|
||||
func TestCompileRejectsNonBooleanTopLevelConstant(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = engine.Compile(context.Background(), `1`)
|
||||
require.EqualError(t, err, "filter must evaluate to a boolean value")
|
||||
}
|
||||
|
||||
func TestCompileRejectsMalformedRegex(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = engine.Compile(context.Background(), `content.matches("(")`)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestCompileMatchesRendersRegexOperator(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `content.matches("v[0-9]+")`, RenderOptions{Dialect: DialectPostgres})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, stmt.SQL, "~")
|
||||
require.Equal(t, []any{"v[0-9]+"}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileRejectsStartsWithOnUnsupportedField(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = engine.Compile(context.Background(), `visibility.startsWith("P")`)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "does not support text matching")
|
||||
}
|
||||
|
||||
func TestCompileContainsEscapesLikeWildcards(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `content.contains("50%_off")`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
// The % and _ in the value must be escaped so they are matched literally,
|
||||
// and SQLite needs an explicit ESCAPE clause.
|
||||
require.Contains(t, stmt.SQL, `ESCAPE '\'`)
|
||||
require.Equal(t, []any{`%50\%\_off%`}, stmt.Args)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Cross-dialect rendering tests (no DB required; complements the SQLite-only
|
||||
// behavioral tests in store/test by asserting MySQL/Postgres SQL generation).
|
||||
// =============================================================================
|
||||
|
||||
func TestRenderStartsWithPerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
dialect DialectName
|
||||
fragments []string
|
||||
}{
|
||||
{DialectSQLite, []string{"memos_unicode_lower(", "`memo`.`content`", `ESCAPE '\'`}},
|
||||
{DialectPostgres, []string{"memo.content ILIKE $1"}},
|
||||
{DialectMySQL, []string{"`memo`.`content` LIKE ?"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `content.startsWith("TODO")`, RenderOptions{Dialect: tc.dialect})
|
||||
require.NoError(t, err, tc.dialect)
|
||||
for _, frag := range tc.fragments {
|
||||
require.Contains(t, stmt.SQL, frag, "dialect %s", tc.dialect)
|
||||
}
|
||||
require.Equal(t, []any{"TODO%"}, stmt.Args, "dialect %s", tc.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderEndsWithPerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, dialect := range []DialectName{DialectSQLite, DialectPostgres, DialectMySQL} {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `content.endsWith(".md")`, RenderOptions{Dialect: dialect})
|
||||
require.NoError(t, err, dialect)
|
||||
require.Equal(t, []any{"%.md"}, stmt.Args, "dialect %s", dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderMatchesPerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
dialect DialectName
|
||||
fragment string
|
||||
}{
|
||||
{DialectSQLite, "`memo`.`content` REGEXP ?"},
|
||||
{DialectMySQL, "`memo`.`content` REGEXP ?"},
|
||||
{DialectPostgres, "memo.content ~ $1"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `content.matches("v[0-9]+")`, RenderOptions{Dialect: tc.dialect})
|
||||
require.NoError(t, err, tc.dialect)
|
||||
require.Contains(t, stmt.SQL, tc.fragment, "dialect %s", tc.dialect)
|
||||
require.Equal(t, []any{"v[0-9]+"}, stmt.Args, "dialect %s", tc.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderTagsAllPerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
dialect DialectName
|
||||
fragments []string
|
||||
}{
|
||||
{DialectSQLite, []string{"NOT EXISTS", "json_each(", "!= '[]'", "memos_unicode_lower(value)"}},
|
||||
{DialectPostgres, []string{"NOT EXISTS", "jsonb_array_elements_text(", "jsonb_array_length(", "value ILIKE"}},
|
||||
{DialectMySQL, []string{"NOT EXISTS", "JSON_TABLE(", "JSON_LENGTH(", "value LIKE"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `tags.all(t, t.startsWith("work/"))`, RenderOptions{Dialect: tc.dialect})
|
||||
require.NoError(t, err, tc.dialect)
|
||||
for _, frag := range tc.fragments {
|
||||
require.Contains(t, stmt.SQL, frag, "dialect %s", tc.dialect)
|
||||
}
|
||||
require.Equal(t, []any{"work/%"}, stmt.Args, "dialect %s", tc.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderTextMatchEscaping(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Both % and _ in the value must be escaped so they match literally.
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `content.contains("a%b_c")`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{`%a\%b\_c%`}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestRenderAllRejectsUnsupportedPredicate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
// size() is not a valid per-element predicate inside all().
|
||||
_, err = engine.CompileToStatement(context.Background(), `tags.all(t, size(t) > 2)`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -1,377 +0,0 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Arithmetic folding: division and modulo
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestCompileDivisionFolds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `creator_id == 100 / 10`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(10)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileModuloFolds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `creator_id == 17 % 5`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(2)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileDivisionByZeroErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
_, err = engine.Compile(context.Background(), `creator_id == 10 / 0`)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// size() on scalar string fields -> SQL length
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestCompileSizeOnContentRendersLengthPerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
dialect DialectName
|
||||
fragment string
|
||||
}{
|
||||
{DialectSQLite, "LENGTH("},
|
||||
{DialectPostgres, "LENGTH("},
|
||||
{DialectMySQL, "CHAR_LENGTH("},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `size(content) > 5`, RenderOptions{Dialect: tc.dialect})
|
||||
require.NoError(t, err, tc.dialect)
|
||||
require.Contains(t, stmt.SQL, tc.fragment, "dialect %s", tc.dialect)
|
||||
require.Equal(t, []any{int64(5)}, stmt.Args, "dialect %s", tc.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Timestamp accessor methods (getFullYear, getMonth, ...)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestCompileTimestampAccessorsPerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
filter string
|
||||
dialect DialectName
|
||||
fragments []string
|
||||
arg int64
|
||||
}{
|
||||
// getFullYear == 2024
|
||||
{"sqlite year", `created_ts.getFullYear() == 2024`, DialectSQLite, []string{"strftime('%Y'", "'unixepoch'"}, 2024},
|
||||
{"pg year", `created_ts.getFullYear() == 2024`, DialectPostgres, []string{"EXTRACT(YEAR FROM to_timestamp(", "AT TIME ZONE 'UTC'"}, 2024},
|
||||
{"mysql year", `created_ts.getFullYear() == 2024`, DialectMySQL, []string{"YEAR(`memo`.`created_ts`)"}, 2024},
|
||||
// getMonth is 0-based -> SQL must subtract 1
|
||||
{"sqlite month", `created_ts.getMonth() == 5`, DialectSQLite, []string{"strftime('%m'", "- 1)"}, 5},
|
||||
{"pg month", `created_ts.getMonth() == 5`, DialectPostgres, []string{"EXTRACT(MONTH FROM", "- 1)"}, 5},
|
||||
{"mysql month", `created_ts.getMonth() == 5`, DialectMySQL, []string{"MONTH(`memo`.`created_ts`)", "- 1)"}, 5},
|
||||
// getDayOfWeek 0=Sunday -> MySQL DAYOFWEEK is 1-based and must subtract 1
|
||||
{"mysql dow", `created_ts.getDayOfWeek() == 0`, DialectMySQL, []string{"DAYOFWEEK(`memo`.`created_ts`)", "- 1)"}, 0},
|
||||
{"sqlite dow", `created_ts.getDayOfWeek() == 0`, DialectSQLite, []string{"strftime('%w'"}, 0},
|
||||
// getDate is 1-based -> no offset
|
||||
{"sqlite date", `created_ts.getDate() == 22`, DialectSQLite, []string{"strftime('%d'"}, 22},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), tc.filter, RenderOptions{Dialect: tc.dialect})
|
||||
require.NoError(t, err, tc.name)
|
||||
for _, frag := range tc.fragments {
|
||||
require.Contains(t, stmt.SQL, frag, tc.name)
|
||||
}
|
||||
require.Equal(t, []any{tc.arg}, stmt.Args, tc.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileNowAccessorsFoldToInjectedClock(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// 2026-07-07T10:30:45Z, a Tuesday (year day 188).
|
||||
engine := memoEngineAt(t, time.Date(2026, time.July, 7, 10, 30, 45, 0, time.UTC).Unix())
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
filter string
|
||||
args []any
|
||||
}{
|
||||
{"on this day", `created_ts.getMonth() == now.getMonth() && created_ts.getDate() == now.getDate()`, []any{int64(6), int64(7)}},
|
||||
{"year", `created_ts.getFullYear() < now.getFullYear()`, []any{int64(2026)}},
|
||||
{"day of month", `created_ts.getDayOfMonth() == now.getDayOfMonth()`, []any{int64(6)}},
|
||||
{"day of week", `created_ts.getDayOfWeek() == now.getDayOfWeek()`, []any{int64(2)}},
|
||||
{"day of year", `created_ts.getDayOfYear() == now.getDayOfYear()`, []any{int64(187)}},
|
||||
{"clock parts", `created_ts.getHours() == now.getHours() || created_ts.getMinutes() == now.getMinutes() || created_ts.getSeconds() == now.getSeconds()`, []any{int64(10), int64(30), int64(45)}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), tc.filter, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err, tc.name)
|
||||
require.Equal(t, tc.args, stmt.Args, tc.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileNowAccessorRejectsTimezoneArg(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
_, err = engine.Compile(context.Background(), `created_ts.getMonth() == now.getMonth("America/New_York")`)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestCompileOnThisDayPerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const onThisDay = `created_ts.getMonth() == now.getMonth() && created_ts.getDate() == now.getDate()`
|
||||
engine := memoEngineAt(t, time.Date(2026, time.July, 7, 10, 30, 45, 0, time.UTC).Unix())
|
||||
|
||||
cases := []struct {
|
||||
dialect DialectName
|
||||
fragments []string
|
||||
}{
|
||||
{DialectSQLite, []string{"strftime('%m'", "strftime('%d'", "'unixepoch'"}},
|
||||
{DialectPostgres, []string{"EXTRACT(MONTH FROM", "EXTRACT(DAY FROM"}},
|
||||
{DialectMySQL, []string{"MONTH(`memo`.`created_ts`)", "DAYOFMONTH(`memo`.`created_ts`)"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), onThisDay, RenderOptions{Dialect: tc.dialect})
|
||||
require.NoError(t, err, tc.dialect)
|
||||
for _, frag := range tc.fragments {
|
||||
require.Contains(t, stmt.SQL, frag, "dialect %s", tc.dialect)
|
||||
}
|
||||
require.Equal(t, []any{int64(6), int64(7)}, stmt.Args, "dialect %s", tc.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileNowAccessorsAtBoundaryDates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Every accessor folded at calendar edges: year rollover, leap day, Sunday.
|
||||
allAccessors := `created_ts.getFullYear() == now.getFullYear() ` +
|
||||
`&& created_ts.getMonth() == now.getMonth() ` +
|
||||
`&& created_ts.getDate() == now.getDate() ` +
|
||||
`&& created_ts.getDayOfMonth() == now.getDayOfMonth() ` +
|
||||
`&& created_ts.getDayOfWeek() == now.getDayOfWeek() ` +
|
||||
`&& created_ts.getDayOfYear() == now.getDayOfYear() ` +
|
||||
`&& created_ts.getHours() == now.getHours() ` +
|
||||
`&& created_ts.getMinutes() == now.getMinutes() ` +
|
||||
`&& created_ts.getSeconds() == now.getSeconds()`
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
clock time.Time
|
||||
// year, month, date, dayOfMonth, dayOfWeek, dayOfYear, hours, minutes, seconds
|
||||
args []any
|
||||
}{
|
||||
{
|
||||
"new year's eve",
|
||||
time.Date(2026, time.December, 31, 23, 59, 59, 0, time.UTC),
|
||||
[]any{int64(2026), int64(11), int64(31), int64(30), int64(4), int64(364), int64(23), int64(59), int64(59)},
|
||||
},
|
||||
{
|
||||
"new year's day",
|
||||
time.Date(2027, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
[]any{int64(2027), int64(0), int64(1), int64(0), int64(5), int64(0), int64(0), int64(0), int64(0)},
|
||||
},
|
||||
{
|
||||
"leap day",
|
||||
time.Date(2028, time.February, 29, 12, 0, 0, 0, time.UTC),
|
||||
[]any{int64(2028), int64(1), int64(29), int64(28), int64(2), int64(59), int64(12), int64(0), int64(0)},
|
||||
},
|
||||
{
|
||||
"sunday",
|
||||
time.Date(2026, time.July, 5, 8, 15, 30, 0, time.UTC),
|
||||
[]any{int64(2026), int64(6), int64(5), int64(4), int64(0), int64(185), int64(8), int64(15), int64(30)},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
engine := memoEngineAt(t, tc.clock.Unix())
|
||||
stmt, err := engine.CompileToStatement(context.Background(), allAccessors, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err, tc.name)
|
||||
require.Equal(t, tc.args, stmt.Args, tc.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileNowAccessorFoldsInUTC(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A clock in UTC+8 at 05:00 on July 8 is still July 7, 21:00 in UTC;
|
||||
// folding must not leak the clock's zone.
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
engine.nowFunc = func() time.Time {
|
||||
return time.Date(2026, time.July, 8, 5, 0, 0, 0, time.FixedZone("UTC+8", 8*3600))
|
||||
}
|
||||
|
||||
stmt, err := engine.CompileToStatement(
|
||||
context.Background(),
|
||||
`created_ts.getDate() == now.getDate() && created_ts.getHours() == now.getHours()`,
|
||||
RenderOptions{Dialect: DialectSQLite},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(7), int64(21)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileNowAccessorOnLeftSide(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine := memoEngineAt(t, time.Date(2026, time.July, 7, 10, 30, 45, 0, time.UTC).Unix())
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `now.getMonth() == created_ts.getMonth()`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, stmt.SQL, "strftime('%m'")
|
||||
require.Equal(t, []any{int64(6)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileNowAccessorMirrorsOrderingOperator(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Swapping the literal to the right must flip < to > to keep the meaning.
|
||||
engine := memoEngineAt(t, time.Date(2026, time.July, 7, 10, 30, 45, 0, time.UTC).Unix())
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `now.getHours() < created_ts.getHours()`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, stmt.SQL, "> ?")
|
||||
require.Equal(t, []any{int64(10)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileNowAccessorAgainstLiteralFoldsToConstant(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Both sides fold to literals; the comparison folds to a constant condition.
|
||||
engine := memoEngineAt(t, time.Date(2026, time.July, 7, 10, 30, 45, 0, time.UTC).Unix())
|
||||
|
||||
// True → trivial filter (empty SQL, matches everything).
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `now.getFullYear() >= 2026`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, stmt.SQL)
|
||||
require.Empty(t, stmt.Args)
|
||||
|
||||
// False → unsatisfiable filter.
|
||||
stmt, err = engine.CompileToStatement(context.Background(), `now.getFullYear() < 2026`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "1 = 0", stmt.SQL)
|
||||
require.Empty(t, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileAttachmentNowAccessors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewAttachmentSchema())
|
||||
require.NoError(t, err)
|
||||
engine.nowFunc = fixedClock(time.Date(2026, time.July, 7, 10, 30, 45, 0, time.UTC).Unix())
|
||||
|
||||
stmt, err := engine.CompileToStatement(
|
||||
context.Background(),
|
||||
`create_time.getMonth() == now.getMonth() && create_time.getDate() == now.getDate()`,
|
||||
RenderOptions{Dialect: DialectSQLite},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, stmt.SQL, "`attachment`.`created_ts`")
|
||||
require.Equal(t, []any{int64(6), int64(7)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileTimestampAccessorRejectsTimezoneArg(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
_, err = engine.Compile(context.Background(), `created_ts.getHours("America/New_York") == 9`)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestCompileTimestampAccessorRejectsNonTimestampField(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
// content is a string, not a timestamp.
|
||||
_, err = engine.Compile(context.Background(), `content.getFullYear() == 2024`)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ext.Sets(): sets.contains / sets.intersects / sets.equivalent over tags
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestCompileSetsContainsRendersAndOfMemberships(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `sets.contains(tags, ["a", "b"])`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, stmt.SQL, " AND ")
|
||||
require.Len(t, stmt.Args, 2)
|
||||
}
|
||||
|
||||
func TestCompileSetsIntersectsRendersOrOfMemberships(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `sets.intersects(tags, ["a", "b"])`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, stmt.SQL, " OR ")
|
||||
require.Len(t, stmt.Args, 2)
|
||||
}
|
||||
|
||||
func TestCompileSetsEquivalentAddsLengthCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `sets.equivalent(tags, ["a", "b"])`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, stmt.SQL, "JSON_ARRAY_LENGTH")
|
||||
require.Contains(t, stmt.Args, int64(2))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// exists_one() comprehension on tags
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestCompileExistsOnePerDialect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
dialect DialectName
|
||||
fragments []string
|
||||
}{
|
||||
{DialectSQLite, []string{"COUNT(", "json_each(", ") = 1"}},
|
||||
{DialectPostgres, []string{"COUNT(", "jsonb_array_elements_text(", ") = 1"}},
|
||||
{DialectMySQL, []string{"COUNT(", "JSON_TABLE(", ") = 1"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `tags.exists_one(t, t == "urgent")`, RenderOptions{Dialect: tc.dialect})
|
||||
require.NoError(t, err, tc.dialect)
|
||||
for _, frag := range tc.fragments {
|
||||
require.Contains(t, stmt.SQL, frag, "dialect %s", tc.dialect)
|
||||
}
|
||||
require.Equal(t, []any{"urgent"}, stmt.Args, "dialect %s", tc.dialect)
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// AppendConditions compiles the provided filters and appends the resulting SQL fragments and args.
|
||||
func AppendConditions(ctx context.Context, engine *Engine, filters []string, dialect DialectName, where *[]string, args *[]any) error {
|
||||
for _, filterStr := range filters {
|
||||
stmt, err := engine.CompileToStatement(ctx, filterStr, RenderOptions{
|
||||
Dialect: dialect,
|
||||
PlaceholderOffset: len(*args),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stmt.SQL == "" {
|
||||
continue
|
||||
}
|
||||
*where = append(*where, fmt.Sprintf("(%s)", stmt.SQL))
|
||||
*args = append(*args, stmt.Args...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
package filter
|
||||
|
||||
// Condition represents a boolean expression derived from the CEL filter.
|
||||
type Condition interface {
|
||||
isCondition()
|
||||
}
|
||||
|
||||
// LogicalOperator enumerates the supported logical operators.
|
||||
type LogicalOperator string
|
||||
|
||||
const (
|
||||
LogicalAnd LogicalOperator = "AND"
|
||||
LogicalOr LogicalOperator = "OR"
|
||||
)
|
||||
|
||||
// LogicalCondition composes two conditions with a logical operator.
|
||||
type LogicalCondition struct {
|
||||
Operator LogicalOperator
|
||||
Left Condition
|
||||
Right Condition
|
||||
}
|
||||
|
||||
func (*LogicalCondition) isCondition() {}
|
||||
|
||||
// NotCondition negates a child condition.
|
||||
type NotCondition struct {
|
||||
Expr Condition
|
||||
}
|
||||
|
||||
func (*NotCondition) isCondition() {}
|
||||
|
||||
// FieldPredicateCondition asserts that a field evaluates to true.
|
||||
type FieldPredicateCondition struct {
|
||||
Field string
|
||||
}
|
||||
|
||||
func (*FieldPredicateCondition) isCondition() {}
|
||||
|
||||
// ComparisonOperator lists supported comparison operators.
|
||||
type ComparisonOperator string
|
||||
|
||||
const (
|
||||
CompareEq ComparisonOperator = "="
|
||||
CompareNeq ComparisonOperator = "!="
|
||||
CompareLt ComparisonOperator = "<"
|
||||
CompareLte ComparisonOperator = "<="
|
||||
CompareGt ComparisonOperator = ">"
|
||||
CompareGte ComparisonOperator = ">="
|
||||
)
|
||||
|
||||
// ComparisonCondition represents a binary comparison.
|
||||
type ComparisonCondition struct {
|
||||
Left ValueExpr
|
||||
Operator ComparisonOperator
|
||||
Right ValueExpr
|
||||
}
|
||||
|
||||
func (*ComparisonCondition) isCondition() {}
|
||||
|
||||
// InCondition represents an IN predicate with literal list values.
|
||||
type InCondition struct {
|
||||
Left ValueExpr
|
||||
Values []ValueExpr
|
||||
}
|
||||
|
||||
func (*InCondition) isCondition() {}
|
||||
|
||||
// ElementInCondition represents the CEL syntax `"value" in field`.
|
||||
type ElementInCondition struct {
|
||||
Element ValueExpr
|
||||
Field string
|
||||
}
|
||||
|
||||
func (*ElementInCondition) isCondition() {}
|
||||
|
||||
// TextMatchMode enumerates LIKE-based string match modes.
|
||||
type TextMatchMode string
|
||||
|
||||
const (
|
||||
TextMatchContains TextMatchMode = "contains"
|
||||
TextMatchPrefix TextMatchMode = "prefix"
|
||||
TextMatchSuffix TextMatchMode = "suffix"
|
||||
)
|
||||
|
||||
// TextMatchCondition models a case-insensitive LIKE match on a scalar string field
|
||||
// (content.contains/startsWith/endsWith).
|
||||
type TextMatchCondition struct {
|
||||
Field string
|
||||
Mode TextMatchMode
|
||||
Value string
|
||||
}
|
||||
|
||||
func (*TextMatchCondition) isCondition() {}
|
||||
|
||||
// RegexCondition models field.matches("pattern") on a string field.
|
||||
type RegexCondition struct {
|
||||
Field string
|
||||
Pattern string
|
||||
}
|
||||
|
||||
func (*RegexCondition) isCondition() {}
|
||||
|
||||
// ConstantCondition captures a literal boolean outcome.
|
||||
type ConstantCondition struct {
|
||||
Value bool
|
||||
}
|
||||
|
||||
func (*ConstantCondition) isCondition() {}
|
||||
|
||||
// ValueExpr models arithmetic or scalar expressions whose result feeds a comparison.
|
||||
type ValueExpr interface {
|
||||
isValueExpr()
|
||||
}
|
||||
|
||||
// FieldRef references a named schema field.
|
||||
type FieldRef struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
func (*FieldRef) isValueExpr() {}
|
||||
|
||||
// LiteralValue holds a literal scalar.
|
||||
type LiteralValue struct {
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
func (*LiteralValue) isValueExpr() {}
|
||||
|
||||
// FunctionValue captures simple function calls like size(tags).
|
||||
type FunctionValue struct {
|
||||
Name string
|
||||
Args []ValueExpr
|
||||
}
|
||||
|
||||
func (*FunctionValue) isValueExpr() {}
|
||||
|
||||
// FieldAccessorValue captures a CEL timestamp accessor on a field, such as
|
||||
// created_ts.getMonth(). It renders to a dialect-specific date-part extraction.
|
||||
type FieldAccessorValue struct {
|
||||
Field string
|
||||
Accessor string // e.g. "getFullYear", "getMonth"
|
||||
}
|
||||
|
||||
func (*FieldAccessorValue) isValueExpr() {}
|
||||
|
||||
// ListComprehensionCondition represents CEL macros like exists(), all(), filter().
|
||||
type ListComprehensionCondition struct {
|
||||
Kind ComprehensionKind
|
||||
Field string // The list field to iterate over (e.g., "tags")
|
||||
IterVar string // The iteration variable name (e.g., "t")
|
||||
Predicate PredicateExpr // The predicate to evaluate for each element
|
||||
}
|
||||
|
||||
func (*ListComprehensionCondition) isCondition() {}
|
||||
|
||||
// ComprehensionKind enumerates the types of list comprehensions.
|
||||
type ComprehensionKind string
|
||||
|
||||
const (
|
||||
ComprehensionExists ComprehensionKind = "exists"
|
||||
ComprehensionAll ComprehensionKind = "all"
|
||||
ComprehensionExistsOne ComprehensionKind = "exists_one"
|
||||
)
|
||||
|
||||
// PredicateExpr represents predicates used in comprehensions.
|
||||
type PredicateExpr interface {
|
||||
isPredicateExpr()
|
||||
}
|
||||
|
||||
// StartsWithPredicate represents t.startsWith("prefix").
|
||||
type StartsWithPredicate struct {
|
||||
Prefix string
|
||||
}
|
||||
|
||||
func (*StartsWithPredicate) isPredicateExpr() {}
|
||||
|
||||
// EndsWithPredicate represents t.endsWith("suffix").
|
||||
type EndsWithPredicate struct {
|
||||
Suffix string
|
||||
}
|
||||
|
||||
func (*EndsWithPredicate) isPredicateExpr() {}
|
||||
|
||||
// ContainsPredicate represents t.contains("substring").
|
||||
type ContainsPredicate struct {
|
||||
Substring string
|
||||
}
|
||||
|
||||
func (*ContainsPredicate) isPredicateExpr() {}
|
||||
|
||||
// EqualsPredicate represents t == "value".
|
||||
type EqualsPredicate struct {
|
||||
Value string
|
||||
}
|
||||
|
||||
func (*EqualsPredicate) isPredicateExpr() {}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,980 +0,0 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type renderer struct {
|
||||
schema Schema
|
||||
dialect DialectName
|
||||
placeholderOffset int
|
||||
placeholderCounter int
|
||||
args []any
|
||||
}
|
||||
|
||||
type renderResult struct {
|
||||
sql string
|
||||
trivial bool
|
||||
unsatisfiable bool
|
||||
}
|
||||
|
||||
func newRenderer(schema Schema, opts RenderOptions) *renderer {
|
||||
return &renderer{
|
||||
schema: schema,
|
||||
dialect: opts.Dialect,
|
||||
placeholderOffset: opts.PlaceholderOffset,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) Render(cond Condition) (Statement, error) {
|
||||
result, err := r.renderCondition(cond)
|
||||
if err != nil {
|
||||
return Statement{}, err
|
||||
}
|
||||
args := r.args
|
||||
if args == nil {
|
||||
args = []any{}
|
||||
}
|
||||
|
||||
switch {
|
||||
case result.unsatisfiable:
|
||||
return Statement{
|
||||
SQL: "1 = 0",
|
||||
Args: args,
|
||||
}, nil
|
||||
case result.trivial:
|
||||
return Statement{
|
||||
SQL: "",
|
||||
Args: args,
|
||||
}, nil
|
||||
default:
|
||||
return Statement{
|
||||
SQL: result.sql,
|
||||
Args: args,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) renderCondition(cond Condition) (renderResult, error) {
|
||||
switch c := cond.(type) {
|
||||
case *LogicalCondition:
|
||||
return r.renderLogicalCondition(c)
|
||||
case *NotCondition:
|
||||
return r.renderNotCondition(c)
|
||||
case *FieldPredicateCondition:
|
||||
return r.renderFieldPredicate(c)
|
||||
case *ComparisonCondition:
|
||||
return r.renderComparison(c)
|
||||
case *InCondition:
|
||||
return r.renderInCondition(c)
|
||||
case *ElementInCondition:
|
||||
return r.renderElementInCondition(c)
|
||||
case *TextMatchCondition:
|
||||
return r.renderTextMatch(c)
|
||||
case *RegexCondition:
|
||||
return r.renderRegex(c)
|
||||
case *ListComprehensionCondition:
|
||||
return r.renderListComprehension(c)
|
||||
case *ConstantCondition:
|
||||
if c.Value {
|
||||
return renderResult{trivial: true}, nil
|
||||
}
|
||||
return renderResult{sql: "1 = 0", unsatisfiable: true}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported condition type %T", c)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) renderLogicalCondition(cond *LogicalCondition) (renderResult, error) {
|
||||
left, err := r.renderCondition(cond.Left)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
right, err := r.renderCondition(cond.Right)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
|
||||
switch cond.Operator {
|
||||
case LogicalAnd:
|
||||
return combineAnd(left, right), nil
|
||||
case LogicalOr:
|
||||
return combineOr(left, right), nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported logical operator %s", cond.Operator)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) renderNotCondition(cond *NotCondition) (renderResult, error) {
|
||||
child, err := r.renderCondition(cond.Expr)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
|
||||
if child.trivial {
|
||||
return renderResult{sql: "1 = 0", unsatisfiable: true}, nil
|
||||
}
|
||||
if child.unsatisfiable {
|
||||
return renderResult{trivial: true}, nil
|
||||
}
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("NOT (%s)", child.sql),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *renderer) renderFieldPredicate(cond *FieldPredicateCondition) (renderResult, error) {
|
||||
field, ok := r.schema.Field(cond.Field)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", cond.Field)
|
||||
}
|
||||
|
||||
switch field.Kind {
|
||||
case FieldKindBoolColumn:
|
||||
column := qualifyColumn(r.dialect, field.Column)
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("%s IS TRUE", column),
|
||||
}, nil
|
||||
case FieldKindJSONBool:
|
||||
sql, err := r.jsonBoolPredicate(field)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
return renderResult{sql: sql}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("field %q cannot be used as a predicate", cond.Field)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) renderComparison(cond *ComparisonCondition) (renderResult, error) {
|
||||
switch left := cond.Left.(type) {
|
||||
case *FieldRef:
|
||||
field, ok := r.schema.Field(left.Name)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", left.Name)
|
||||
}
|
||||
switch field.Kind {
|
||||
case FieldKindBoolColumn:
|
||||
return r.renderBoolColumnComparison(field, cond.Operator, cond.Right)
|
||||
case FieldKindJSONBool:
|
||||
return r.renderJSONBoolComparison(field, cond.Operator, cond.Right)
|
||||
case FieldKindScalar:
|
||||
return r.renderScalarComparison(field, cond.Operator, cond.Right)
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("field %q does not support comparison", field.Name)
|
||||
}
|
||||
case *FunctionValue:
|
||||
return r.renderFunctionComparison(left, cond.Operator, cond.Right)
|
||||
case *FieldAccessorValue:
|
||||
return r.renderAccessorComparison(left, cond.Operator, cond.Right)
|
||||
default:
|
||||
return renderResult{}, errors.New("comparison must start with a field reference or supported function")
|
||||
}
|
||||
}
|
||||
|
||||
// accessorSpec maps a CEL timestamp accessor to per-dialect SQL date-part tokens
|
||||
// and the offset to subtract so the result matches CEL's base (e.g. CEL months
|
||||
// are 0-based but every dialect reports 1-based, so off=1). off is indexed
|
||||
// [sqlite, postgres, mysql].
|
||||
type accessorSpec struct {
|
||||
sqlite string // strftime format specifier
|
||||
pg string // EXTRACT field
|
||||
mysql string // function name
|
||||
off [3]int
|
||||
}
|
||||
|
||||
var accessorSpecs = map[string]accessorSpec{
|
||||
"getFullYear": {"%Y", "YEAR", "YEAR", [3]int{0, 0, 0}},
|
||||
"getMonth": {"%m", "MONTH", "MONTH", [3]int{1, 1, 1}},
|
||||
"getDate": {"%d", "DAY", "DAYOFMONTH", [3]int{0, 0, 0}},
|
||||
"getDayOfMonth": {"%d", "DAY", "DAYOFMONTH", [3]int{1, 1, 1}},
|
||||
"getDayOfWeek": {"%w", "DOW", "DAYOFWEEK", [3]int{0, 0, 1}},
|
||||
"getDayOfYear": {"%j", "DOY", "DAYOFYEAR", [3]int{1, 1, 1}},
|
||||
"getHours": {"%H", "HOUR", "HOUR", [3]int{0, 0, 0}},
|
||||
"getMinutes": {"%M", "MINUTE", "MINUTE", [3]int{0, 0, 0}},
|
||||
"getSeconds": {"%S", "SECOND", "SECOND", [3]int{0, 0, 0}},
|
||||
}
|
||||
|
||||
func (r *renderer) renderAccessorComparison(acc *FieldAccessorValue, op ComparisonOperator, right ValueExpr) (renderResult, error) {
|
||||
field, ok := r.schema.Field(acc.Field)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", acc.Field)
|
||||
}
|
||||
value, err := expectNumericLiteral(right)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
expr, err := r.timestampAccessorExpr(field, acc.Accessor)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
placeholder := r.addArg(value)
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("%s %s %s", expr, sqlOperator(op), placeholder),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// timestampAccessorExpr builds a dialect-specific integer expression for a CEL
|
||||
// timestamp accessor. Extraction is UTC on SQLite/Postgres (epoch columns); on
|
||||
// MySQL the TIMESTAMP column is read in the server session time zone.
|
||||
func (r *renderer) timestampAccessorExpr(field Field, accessor string) (string, error) {
|
||||
spec, ok := accessorSpecs[accessor]
|
||||
if !ok {
|
||||
return "", errors.Errorf("unsupported timestamp accessor %q", accessor)
|
||||
}
|
||||
col := qualifyColumn(r.dialect, field.Column)
|
||||
var base string
|
||||
var off int
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
base = fmt.Sprintf("CAST(strftime('%s', %s, 'unixepoch') AS INTEGER)", spec.sqlite, col)
|
||||
off = spec.off[0]
|
||||
case DialectPostgres:
|
||||
base = fmt.Sprintf("CAST(EXTRACT(%s FROM to_timestamp(%s) AT TIME ZONE 'UTC') AS INTEGER)", spec.pg, col)
|
||||
off = spec.off[1]
|
||||
case DialectMySQL:
|
||||
base = fmt.Sprintf("%s(%s)", spec.mysql, col)
|
||||
off = spec.off[2]
|
||||
default:
|
||||
return "", errors.Errorf("unsupported dialect %q", r.dialect)
|
||||
}
|
||||
if off != 0 {
|
||||
base = fmt.Sprintf("(%s - %d)", base, off)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
func (r *renderer) renderFunctionComparison(fn *FunctionValue, op ComparisonOperator, right ValueExpr) (renderResult, error) {
|
||||
if fn.Name != "size" {
|
||||
return renderResult{}, errors.Errorf("unsupported function %s in comparison", fn.Name)
|
||||
}
|
||||
if len(fn.Args) != 1 {
|
||||
return renderResult{}, errors.New("size() expects one argument")
|
||||
}
|
||||
fieldArg, ok := fn.Args[0].(*FieldRef)
|
||||
if !ok {
|
||||
return renderResult{}, errors.New("size() argument must be a field")
|
||||
}
|
||||
|
||||
field, ok := r.schema.Field(fieldArg.Name)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", fieldArg.Name)
|
||||
}
|
||||
if field.Kind == FieldKindVirtualAlias {
|
||||
field, ok = r.schema.ResolveAlias(fieldArg.Name)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("invalid alias %q", fieldArg.Name)
|
||||
}
|
||||
}
|
||||
|
||||
value, err := expectNumericLiteral(right)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
|
||||
var expr string
|
||||
switch {
|
||||
case field.Kind == FieldKindJSONList:
|
||||
expr = jsonArrayLengthExpr(r.dialect, field)
|
||||
case field.Kind == FieldKindScalar && field.Type == FieldTypeString:
|
||||
expr = stringLengthExpr(r.dialect, field.columnExpr(r.dialect))
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("size() does not support field %q", field.Name)
|
||||
}
|
||||
|
||||
placeholder := r.addArg(value)
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("%s %s %s", expr, sqlOperator(op), placeholder),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// stringLengthExpr returns the character-count expression for a string column.
|
||||
// MySQL's LENGTH counts bytes, so CHAR_LENGTH is used to count characters and
|
||||
// match CEL's size() code-point semantics; SQLite/Postgres LENGTH already counts
|
||||
// characters.
|
||||
func stringLengthExpr(d DialectName, colExpr string) string {
|
||||
if d == DialectMySQL {
|
||||
return fmt.Sprintf("CHAR_LENGTH(%s)", colExpr)
|
||||
}
|
||||
return fmt.Sprintf("LENGTH(%s)", colExpr)
|
||||
}
|
||||
|
||||
func (r *renderer) renderScalarComparison(field Field, op ComparisonOperator, right ValueExpr) (renderResult, error) {
|
||||
lit, err := expectLiteral(right)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
|
||||
columnExpr := field.columnExpr(r.dialect)
|
||||
if lit == nil {
|
||||
switch op {
|
||||
case CompareEq:
|
||||
return renderResult{sql: fmt.Sprintf("%s IS NULL", columnExpr)}, nil
|
||||
case CompareNeq:
|
||||
return renderResult{sql: fmt.Sprintf("%s IS NOT NULL", columnExpr)}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("operator %s not supported for null comparison", op)
|
||||
}
|
||||
}
|
||||
|
||||
placeholder := ""
|
||||
switch field.Type {
|
||||
case FieldTypeString:
|
||||
value, ok := lit.(string)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("field %q expects string value", field.Name)
|
||||
}
|
||||
placeholder = r.addArg(value)
|
||||
case FieldTypeInt, FieldTypeTimestamp:
|
||||
num, err := toInt64(lit)
|
||||
if err != nil {
|
||||
return renderResult{}, errors.Wrapf(err, "field %q expects integer value", field.Name)
|
||||
}
|
||||
placeholder = r.addArg(num)
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported data type %q for field %s", field.Type, field.Name)
|
||||
}
|
||||
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("%s %s %s", columnExpr, sqlOperator(op), placeholder),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *renderer) renderBoolColumnComparison(field Field, op ComparisonOperator, right ValueExpr) (renderResult, error) {
|
||||
value, err := expectBool(right)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
placeholder := r.addBoolArg(value)
|
||||
column := qualifyColumn(r.dialect, field.Column)
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("%s %s %s", column, sqlOperator(op), placeholder),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *renderer) renderJSONBoolComparison(field Field, op ComparisonOperator, right ValueExpr) (renderResult, error) {
|
||||
value, err := expectBool(right)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
|
||||
jsonExpr := jsonExtractExpr(r.dialect, field)
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
switch op {
|
||||
case CompareEq:
|
||||
if field.Name == "has_task_list" {
|
||||
target := "0"
|
||||
if value {
|
||||
target = "1"
|
||||
}
|
||||
return renderResult{sql: fmt.Sprintf("%s = %s", jsonExpr, target)}, nil
|
||||
}
|
||||
if value {
|
||||
return renderResult{sql: fmt.Sprintf("%s IS TRUE", jsonExpr)}, nil
|
||||
}
|
||||
return renderResult{sql: fmt.Sprintf("NOT(%s IS TRUE)", jsonExpr)}, nil
|
||||
case CompareNeq:
|
||||
if field.Name == "has_task_list" {
|
||||
target := "0"
|
||||
if value {
|
||||
target = "1"
|
||||
}
|
||||
return renderResult{sql: fmt.Sprintf("%s != %s", jsonExpr, target)}, nil
|
||||
}
|
||||
if value {
|
||||
return renderResult{sql: fmt.Sprintf("NOT(%s IS TRUE)", jsonExpr)}, nil
|
||||
}
|
||||
return renderResult{sql: fmt.Sprintf("%s IS TRUE", jsonExpr)}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("operator %s not supported for boolean JSON field", op)
|
||||
}
|
||||
case DialectMySQL:
|
||||
boolStr := "false"
|
||||
if value {
|
||||
boolStr = "true"
|
||||
}
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("%s %s CAST('%s' AS JSON)", jsonExpr, sqlOperator(op), boolStr),
|
||||
}, nil
|
||||
case DialectPostgres:
|
||||
placeholder := r.addArg(value)
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("(%s)::boolean %s %s", jsonExpr, sqlOperator(op), placeholder),
|
||||
}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) renderInCondition(cond *InCondition) (renderResult, error) {
|
||||
fieldRef, ok := cond.Left.(*FieldRef)
|
||||
if !ok {
|
||||
return renderResult{}, errors.New("IN operator requires a field on the left-hand side")
|
||||
}
|
||||
|
||||
if fieldRef.Name == "tag" {
|
||||
return r.renderTagInList(cond.Values)
|
||||
}
|
||||
|
||||
field, ok := r.schema.Field(fieldRef.Name)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", fieldRef.Name)
|
||||
}
|
||||
|
||||
if field.Kind != FieldKindScalar {
|
||||
return renderResult{}, errors.Errorf("field %q does not support IN()", fieldRef.Name)
|
||||
}
|
||||
|
||||
return r.renderScalarInCondition(field, cond.Values)
|
||||
}
|
||||
|
||||
func (r *renderer) renderTagInList(values []ValueExpr) (renderResult, error) {
|
||||
field, ok := r.schema.ResolveAlias("tag")
|
||||
if !ok {
|
||||
return renderResult{}, errors.New("tag attribute is not configured")
|
||||
}
|
||||
|
||||
conditions := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
lit, err := expectLiteral(v)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
str, ok := lit.(string)
|
||||
if !ok {
|
||||
return renderResult{}, errors.New("tags must be compared with string literals")
|
||||
}
|
||||
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
// Support hierarchical tags: match exact tag OR tags with this prefix (e.g., "book" matches "book" and "book/something")
|
||||
exactMatch := fmt.Sprintf("%s LIKE %s", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`%%"%s"%%`, str)))
|
||||
prefixMatch := fmt.Sprintf("%s LIKE %s", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`%%"%s/%%`, str)))
|
||||
expr := fmt.Sprintf("(%s OR %s)", exactMatch, prefixMatch)
|
||||
conditions = append(conditions, expr)
|
||||
case DialectMySQL:
|
||||
// Support hierarchical tags: match exact tag OR tags with this prefix
|
||||
exactMatch := fmt.Sprintf("JSON_CONTAINS(%s, %s)", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`"%s"`, str)))
|
||||
prefixMatch := fmt.Sprintf("%s LIKE %s", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`%%"%s/%%`, str)))
|
||||
expr := fmt.Sprintf("(%s OR %s)", exactMatch, prefixMatch)
|
||||
conditions = append(conditions, expr)
|
||||
case DialectPostgres:
|
||||
// Support hierarchical tags: match exact tag OR tags with this prefix
|
||||
exactMatch := fmt.Sprintf("%s @> jsonb_build_array(%s::json)", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`"%s"`, str)))
|
||||
prefixMatch := fmt.Sprintf("(%s)::text LIKE %s", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`%%"%s/%%`, str)))
|
||||
expr := fmt.Sprintf("(%s OR %s)", exactMatch, prefixMatch)
|
||||
conditions = append(conditions, expr)
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
if len(conditions) == 1 {
|
||||
return renderResult{sql: conditions[0]}, nil
|
||||
}
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("(%s)", strings.Join(conditions, " OR ")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *renderer) renderElementInCondition(cond *ElementInCondition) (renderResult, error) {
|
||||
field, ok := r.schema.Field(cond.Field)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", cond.Field)
|
||||
}
|
||||
if field.Kind != FieldKindJSONList {
|
||||
return renderResult{}, errors.Errorf("field %q is not a tag list", cond.Field)
|
||||
}
|
||||
|
||||
lit, err := expectLiteral(cond.Element)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
str, ok := lit.(string)
|
||||
if !ok {
|
||||
return renderResult{}, errors.New("tags membership requires string literal")
|
||||
}
|
||||
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
sql := fmt.Sprintf("%s LIKE %s", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`%%"%s"%%`, str)))
|
||||
return renderResult{sql: sql}, nil
|
||||
case DialectMySQL:
|
||||
sql := fmt.Sprintf("JSON_CONTAINS(%s, %s)", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`"%s"`, str)))
|
||||
return renderResult{sql: sql}, nil
|
||||
case DialectPostgres:
|
||||
sql := fmt.Sprintf("%s @> jsonb_build_array(%s::json)", jsonArrayExpr(r.dialect, field), r.addArg(fmt.Sprintf(`"%s"`, str)))
|
||||
return renderResult{sql: sql}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) renderScalarInCondition(field Field, values []ValueExpr) (renderResult, error) {
|
||||
placeholders := make([]string, 0, len(values))
|
||||
|
||||
for _, v := range values {
|
||||
lit, err := expectLiteral(v)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
switch field.Type {
|
||||
case FieldTypeString:
|
||||
str, ok := lit.(string)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("field %q expects string values", field.Name)
|
||||
}
|
||||
placeholders = append(placeholders, r.addArg(str))
|
||||
case FieldTypeInt:
|
||||
num, err := toInt64(lit)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
placeholders = append(placeholders, r.addArg(num))
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("field %q does not support IN() comparisons", field.Name)
|
||||
}
|
||||
}
|
||||
|
||||
column := field.columnExpr(r.dialect)
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("%s IN (%s)", column, strings.Join(placeholders, ",")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *renderer) renderTextMatch(cond *TextMatchCondition) (renderResult, error) {
|
||||
field, ok := r.schema.Field(cond.Field)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", cond.Field)
|
||||
}
|
||||
column := field.columnExpr(r.dialect)
|
||||
pattern := likePattern(cond.Mode, cond.Value)
|
||||
return renderResult{sql: r.foldedLike(column, pattern)}, nil
|
||||
}
|
||||
|
||||
func (r *renderer) renderRegex(cond *RegexCondition) (renderResult, error) {
|
||||
field, ok := r.schema.Field(cond.Field)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", cond.Field)
|
||||
}
|
||||
column := field.columnExpr(r.dialect)
|
||||
switch r.dialect {
|
||||
case DialectPostgres:
|
||||
// POSIX regex match operator.
|
||||
return renderResult{sql: fmt.Sprintf("%s ~ %s", column, r.addArg(cond.Pattern))}, nil
|
||||
case DialectMySQL, DialectSQLite:
|
||||
// MySQL has a native REGEXP operator; SQLite uses the registered regexp() function.
|
||||
return renderResult{sql: fmt.Sprintf("%s REGEXP %s", column, r.addArg(cond.Pattern))}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
// foldedLike renders a case-insensitive LIKE comparison of colExpr against a
|
||||
// (already metacharacter-escaped) pattern, using each dialect's case-folding.
|
||||
func (r *renderer) foldedLike(colExpr, pattern string) string {
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
// memos_unicode_lower gives Unicode-aware folding; ESCAPE '\' is required
|
||||
// because SQLite has no default LIKE escape character.
|
||||
return fmt.Sprintf(`memos_unicode_lower(%s) LIKE memos_unicode_lower(%s) ESCAPE '\'`, colExpr, r.addArg(pattern))
|
||||
case DialectPostgres:
|
||||
// ILIKE is case-insensitive; backslash is the default escape character.
|
||||
return fmt.Sprintf("%s ILIKE %s", colExpr, r.addArg(pattern))
|
||||
default: // MySQL: default collation is case-insensitive; backslash is the default escape.
|
||||
return fmt.Sprintf("%s LIKE %s", colExpr, r.addArg(pattern))
|
||||
}
|
||||
}
|
||||
|
||||
// likePattern escapes LIKE metacharacters in value and wraps it for the mode.
|
||||
func likePattern(mode TextMatchMode, value string) string {
|
||||
escaped := escapeLikeLiteral(value)
|
||||
switch mode {
|
||||
case TextMatchPrefix:
|
||||
return escaped + "%"
|
||||
case TextMatchSuffix:
|
||||
return "%" + escaped
|
||||
default:
|
||||
return "%" + escaped + "%"
|
||||
}
|
||||
}
|
||||
|
||||
// escapeLikeLiteral escapes the LIKE metacharacters \, %, and _ so user input
|
||||
// is matched literally. Backslash is the escape character on all three dialects.
|
||||
func escapeLikeLiteral(s string) string {
|
||||
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
|
||||
}
|
||||
|
||||
func (r *renderer) renderListComprehension(cond *ListComprehensionCondition) (renderResult, error) {
|
||||
field, ok := r.schema.Field(cond.Field)
|
||||
if !ok {
|
||||
return renderResult{}, errors.Errorf("unknown field %q", cond.Field)
|
||||
}
|
||||
|
||||
if field.Kind != FieldKindJSONList {
|
||||
return renderResult{}, errors.Errorf("field %q is not a JSON list", cond.Field)
|
||||
}
|
||||
|
||||
if cond.Kind == ComprehensionAll {
|
||||
return r.renderTagAll(field, cond.Predicate)
|
||||
}
|
||||
|
||||
if cond.Kind == ComprehensionExistsOne {
|
||||
return r.renderTagExistsOne(field, cond.Predicate)
|
||||
}
|
||||
|
||||
// Render based on predicate type
|
||||
switch pred := cond.Predicate.(type) {
|
||||
case *EqualsPredicate:
|
||||
return r.renderTagEquals(field, pred.Value, cond.Kind)
|
||||
case *StartsWithPredicate:
|
||||
return r.renderTagStartsWith(field, pred.Prefix, cond.Kind)
|
||||
case *EndsWithPredicate:
|
||||
return r.renderTagEndsWith(field, pred.Suffix, cond.Kind)
|
||||
case *ContainsPredicate:
|
||||
return r.renderTagContains(field, pred.Substring, cond.Kind)
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported predicate type %T in comprehension", pred)
|
||||
}
|
||||
}
|
||||
|
||||
// renderTagAll renders tags.all(t, <pred>): the array is non-empty AND no element
|
||||
// fails the predicate. Element predicates use plain CEL semantics (case-insensitive
|
||||
// for startsWith/endsWith/contains, case-sensitive for ==), evaluated per element.
|
||||
func (r *renderer) renderTagAll(field Field, pred PredicateExpr) (renderResult, error) {
|
||||
arrayExpr := jsonArrayExpr(r.dialect, field)
|
||||
elemCond, err := r.elementPredicateSQL(pred)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
nonEmpty := fmt.Sprintf("%s IS NOT NULL AND %s != '[]'", arrayExpr, arrayExpr)
|
||||
sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM json_each(%s) WHERE NOT (%s))", arrayExpr, elemCond)
|
||||
return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil
|
||||
case DialectMySQL:
|
||||
nonEmpty := fmt.Sprintf("%s IS NOT NULL AND JSON_LENGTH(%s) > 0", arrayExpr, arrayExpr)
|
||||
sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM JSON_TABLE(%s, '$[*]' COLUMNS (value VARCHAR(512) PATH '$')) AS elem WHERE NOT (%s))", arrayExpr, elemCond)
|
||||
return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil
|
||||
case DialectPostgres:
|
||||
nonEmpty := fmt.Sprintf("%s IS NOT NULL AND jsonb_array_length(%s) > 0", arrayExpr, arrayExpr)
|
||||
sub := fmt.Sprintf("NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(%s) AS elem(value) WHERE NOT (%s))", arrayExpr, elemCond)
|
||||
return renderResult{sql: fmt.Sprintf("(%s AND %s)", nonEmpty, sub)}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
// renderTagExistsOne renders tags.exists_one(t, <pred>): exactly one element
|
||||
// satisfies the predicate, via a COUNT(...) = 1 subquery. A null or empty array
|
||||
// yields COUNT 0, which is correctly not equal to 1.
|
||||
func (r *renderer) renderTagExistsOne(field Field, pred PredicateExpr) (renderResult, error) {
|
||||
arrayExpr := jsonArrayExpr(r.dialect, field)
|
||||
elemCond, err := r.elementPredicateSQL(pred)
|
||||
if err != nil {
|
||||
return renderResult{}, err
|
||||
}
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
return renderResult{sql: fmt.Sprintf("(SELECT COUNT(*) FROM json_each(%s) WHERE %s) = 1", arrayExpr, elemCond)}, nil
|
||||
case DialectMySQL:
|
||||
return renderResult{sql: fmt.Sprintf("(SELECT COUNT(*) FROM JSON_TABLE(%s, '$[*]' COLUMNS (value VARCHAR(512) PATH '$')) AS elem WHERE %s) = 1", arrayExpr, elemCond)}, nil
|
||||
case DialectPostgres:
|
||||
return renderResult{sql: fmt.Sprintf("(SELECT COUNT(*) FROM jsonb_array_elements_text(%s) AS elem(value) WHERE %s) = 1", arrayExpr, elemCond)}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
// elementPredicateSQL builds the per-element SQL condition for an all() predicate.
|
||||
// The iterated element is exposed as the unqualified column `value` on all dialects
|
||||
// (json_each.value / JSON_TABLE column / elem(value)).
|
||||
func (r *renderer) elementPredicateSQL(pred PredicateExpr) (string, error) {
|
||||
switch p := pred.(type) {
|
||||
case *EqualsPredicate:
|
||||
return fmt.Sprintf("value = %s", r.addArg(p.Value)), nil
|
||||
case *StartsWithPredicate:
|
||||
return r.foldedLike("value", likePattern(TextMatchPrefix, p.Prefix)), nil
|
||||
case *EndsWithPredicate:
|
||||
return r.foldedLike("value", likePattern(TextMatchSuffix, p.Suffix)), nil
|
||||
case *ContainsPredicate:
|
||||
return r.foldedLike("value", likePattern(TextMatchContains, p.Substring)), nil
|
||||
default:
|
||||
return "", errors.Errorf("unsupported predicate %T in all()", pred)
|
||||
}
|
||||
}
|
||||
|
||||
// renderTagEquals generates SQL for tags.exists(t, t == "value").
|
||||
func (r *renderer) renderTagEquals(field Field, value string, _ ComprehensionKind) (renderResult, error) {
|
||||
arrayExpr := jsonArrayExpr(r.dialect, field)
|
||||
|
||||
switch r.dialect {
|
||||
case DialectSQLite, DialectMySQL:
|
||||
exactMatch := r.buildJSONArrayLike(arrayExpr, fmt.Sprintf(`%%"%s"%%`, value))
|
||||
return renderResult{sql: r.wrapWithNullCheck(arrayExpr, exactMatch)}, nil
|
||||
case DialectPostgres:
|
||||
exactMatch := fmt.Sprintf("%s @> jsonb_build_array(%s::json)", arrayExpr, r.addArg(fmt.Sprintf(`"%s"`, value)))
|
||||
return renderResult{sql: r.wrapWithNullCheck(arrayExpr, exactMatch)}, nil
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
// renderTagStartsWith generates SQL for tags.exists(t, t.startsWith("prefix")).
|
||||
func (r *renderer) renderTagStartsWith(field Field, prefix string, _ ComprehensionKind) (renderResult, error) {
|
||||
arrayExpr := jsonArrayExpr(r.dialect, field)
|
||||
|
||||
switch r.dialect {
|
||||
case DialectSQLite, DialectMySQL:
|
||||
// Match exact tag or tags with this prefix (hierarchical support)
|
||||
exactMatch := r.buildJSONArrayLike(arrayExpr, fmt.Sprintf(`%%"%s"%%`, prefix))
|
||||
prefixMatch := r.buildJSONArrayLike(arrayExpr, fmt.Sprintf(`%%"%s%%`, prefix))
|
||||
condition := fmt.Sprintf("(%s OR %s)", exactMatch, prefixMatch)
|
||||
return renderResult{sql: r.wrapWithNullCheck(arrayExpr, condition)}, nil
|
||||
|
||||
case DialectPostgres:
|
||||
// Use PostgreSQL's powerful JSON operators
|
||||
exactMatch := fmt.Sprintf("%s @> jsonb_build_array(%s::json)", arrayExpr, r.addArg(fmt.Sprintf(`"%s"`, prefix)))
|
||||
prefixMatch := fmt.Sprintf("(%s)::text LIKE %s", arrayExpr, r.addArg(fmt.Sprintf(`%%"%s%%`, prefix)))
|
||||
condition := fmt.Sprintf("(%s OR %s)", exactMatch, prefixMatch)
|
||||
return renderResult{sql: r.wrapWithNullCheck(arrayExpr, condition)}, nil
|
||||
|
||||
default:
|
||||
return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
// renderTagEndsWith generates SQL for tags.exists(t, t.endsWith("suffix")).
|
||||
func (r *renderer) renderTagEndsWith(field Field, suffix string, _ ComprehensionKind) (renderResult, error) {
|
||||
arrayExpr := jsonArrayExpr(r.dialect, field)
|
||||
pattern := fmt.Sprintf(`%%%s"%%`, suffix)
|
||||
|
||||
likeExpr := r.buildJSONArrayLike(arrayExpr, pattern)
|
||||
return renderResult{sql: r.wrapWithNullCheck(arrayExpr, likeExpr)}, nil
|
||||
}
|
||||
|
||||
// renderTagContains generates SQL for tags.exists(t, t.contains("substring")).
|
||||
func (r *renderer) renderTagContains(field Field, substring string, _ ComprehensionKind) (renderResult, error) {
|
||||
arrayExpr := jsonArrayExpr(r.dialect, field)
|
||||
pattern := fmt.Sprintf(`%%%s%%`, substring)
|
||||
|
||||
likeExpr := r.buildJSONArrayLike(arrayExpr, pattern)
|
||||
return renderResult{sql: r.wrapWithNullCheck(arrayExpr, likeExpr)}, nil
|
||||
}
|
||||
|
||||
// buildJSONArrayLike builds a LIKE expression for matching within a JSON array.
|
||||
// Returns the LIKE clause without NULL/empty checks.
|
||||
func (r *renderer) buildJSONArrayLike(arrayExpr, pattern string) string {
|
||||
switch r.dialect {
|
||||
case DialectSQLite, DialectMySQL:
|
||||
return fmt.Sprintf("%s LIKE %s", arrayExpr, r.addArg(pattern))
|
||||
case DialectPostgres:
|
||||
return fmt.Sprintf("(%s)::text LIKE %s", arrayExpr, r.addArg(pattern))
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// wrapWithNullCheck wraps a condition with NULL and empty array checks.
|
||||
// This ensures we don't match against NULL or empty JSON arrays.
|
||||
func (r *renderer) wrapWithNullCheck(arrayExpr, condition string) string {
|
||||
var nullCheck string
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
nullCheck = fmt.Sprintf("%s IS NOT NULL AND %s != '[]'", arrayExpr, arrayExpr)
|
||||
case DialectMySQL:
|
||||
nullCheck = fmt.Sprintf("%s IS NOT NULL AND JSON_LENGTH(%s) > 0", arrayExpr, arrayExpr)
|
||||
case DialectPostgres:
|
||||
nullCheck = fmt.Sprintf("%s IS NOT NULL AND jsonb_array_length(%s) > 0", arrayExpr, arrayExpr)
|
||||
default:
|
||||
return condition
|
||||
}
|
||||
return fmt.Sprintf("(%s AND %s)", condition, nullCheck)
|
||||
}
|
||||
|
||||
func (r *renderer) jsonBoolPredicate(field Field) (string, error) {
|
||||
expr := jsonExtractExpr(r.dialect, field)
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
return fmt.Sprintf("%s IS TRUE", expr), nil
|
||||
case DialectMySQL:
|
||||
return fmt.Sprintf("COALESCE(%s, CAST('false' AS JSON)) = CAST('true' AS JSON)", expr), nil
|
||||
case DialectPostgres:
|
||||
return fmt.Sprintf("(%s)::boolean IS TRUE", expr), nil
|
||||
default:
|
||||
return "", errors.Errorf("unsupported dialect %s", r.dialect)
|
||||
}
|
||||
}
|
||||
|
||||
func combineAnd(left, right renderResult) renderResult {
|
||||
if left.unsatisfiable || right.unsatisfiable {
|
||||
return renderResult{sql: "1 = 0", unsatisfiable: true}
|
||||
}
|
||||
if left.trivial {
|
||||
return right
|
||||
}
|
||||
if right.trivial {
|
||||
return left
|
||||
}
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("(%s AND %s)", left.sql, right.sql),
|
||||
}
|
||||
}
|
||||
|
||||
func combineOr(left, right renderResult) renderResult {
|
||||
if left.trivial || right.trivial {
|
||||
return renderResult{trivial: true}
|
||||
}
|
||||
if left.unsatisfiable {
|
||||
return right
|
||||
}
|
||||
if right.unsatisfiable {
|
||||
return left
|
||||
}
|
||||
return renderResult{
|
||||
sql: fmt.Sprintf("(%s OR %s)", left.sql, right.sql),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *renderer) addArg(value any) string {
|
||||
r.placeholderCounter++
|
||||
r.args = append(r.args, value)
|
||||
if r.dialect == DialectPostgres {
|
||||
return fmt.Sprintf("$%d", r.placeholderOffset+r.placeholderCounter)
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
func (r *renderer) addBoolArg(value bool) string {
|
||||
var v any
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
if value {
|
||||
v = 1
|
||||
} else {
|
||||
v = 0
|
||||
}
|
||||
default:
|
||||
v = value
|
||||
}
|
||||
return r.addArg(v)
|
||||
}
|
||||
|
||||
func expectLiteral(expr ValueExpr) (any, error) {
|
||||
lit, ok := expr.(*LiteralValue)
|
||||
if !ok {
|
||||
return nil, errors.New("expression must be a literal")
|
||||
}
|
||||
return lit.Value, nil
|
||||
}
|
||||
|
||||
func expectBool(expr ValueExpr) (bool, error) {
|
||||
lit, err := expectLiteral(expr)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
value, ok := lit.(bool)
|
||||
if !ok {
|
||||
return false, errors.New("boolean literal required")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func expectNumericLiteral(expr ValueExpr) (int64, error) {
|
||||
lit, err := expectLiteral(expr)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return toInt64(lit)
|
||||
}
|
||||
|
||||
func toInt64(value any) (int64, error) {
|
||||
switch v := value.(type) {
|
||||
case int:
|
||||
return int64(v), nil
|
||||
case int32:
|
||||
return int64(v), nil
|
||||
case int64:
|
||||
return v, nil
|
||||
case uint32:
|
||||
return int64(v), nil
|
||||
case uint64:
|
||||
return int64(v), nil
|
||||
case float32:
|
||||
return int64(v), nil
|
||||
case float64:
|
||||
return int64(v), nil
|
||||
default:
|
||||
return 0, errors.Errorf("cannot convert %T to int64", value)
|
||||
}
|
||||
}
|
||||
|
||||
func sqlOperator(op ComparisonOperator) string {
|
||||
return string(op)
|
||||
}
|
||||
|
||||
func qualifyColumn(d DialectName, col Column) string {
|
||||
switch d {
|
||||
case DialectPostgres:
|
||||
return fmt.Sprintf("%s.%s", col.Table, col.Name)
|
||||
default:
|
||||
return fmt.Sprintf("`%s`.`%s`", col.Table, col.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func jsonPath(field Field) string {
|
||||
return "$." + strings.Join(field.JSONPath, ".")
|
||||
}
|
||||
|
||||
func jsonExtractExpr(d DialectName, field Field) string {
|
||||
column := qualifyColumn(d, field.Column)
|
||||
switch d {
|
||||
case DialectSQLite, DialectMySQL:
|
||||
return fmt.Sprintf("JSON_EXTRACT(%s, '%s')", column, jsonPath(field))
|
||||
case DialectPostgres:
|
||||
return buildPostgresJSONAccessor(column, field.JSONPath, true)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func jsonArrayExpr(d DialectName, field Field) string {
|
||||
column := qualifyColumn(d, field.Column)
|
||||
switch d {
|
||||
case DialectSQLite, DialectMySQL:
|
||||
return fmt.Sprintf("JSON_EXTRACT(%s, '%s')", column, jsonPath(field))
|
||||
case DialectPostgres:
|
||||
return buildPostgresJSONAccessor(column, field.JSONPath, false)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func jsonArrayLengthExpr(d DialectName, field Field) string {
|
||||
arrayExpr := jsonArrayExpr(d, field)
|
||||
switch d {
|
||||
case DialectSQLite:
|
||||
return fmt.Sprintf("JSON_ARRAY_LENGTH(COALESCE(%s, JSON_ARRAY()))", arrayExpr)
|
||||
case DialectMySQL:
|
||||
return fmt.Sprintf("JSON_LENGTH(COALESCE(%s, JSON_ARRAY()))", arrayExpr)
|
||||
case DialectPostgres:
|
||||
return fmt.Sprintf("jsonb_array_length(COALESCE(%s, '[]'::jsonb))", arrayExpr)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func buildPostgresJSONAccessor(base string, path []string, terminalText bool) string {
|
||||
expr := base
|
||||
for idx, part := range path {
|
||||
if idx == len(path)-1 && terminalText {
|
||||
expr = fmt.Sprintf("%s->>'%s'", expr, part)
|
||||
} else {
|
||||
expr = fmt.Sprintf("%s->'%s'", expr, part)
|
||||
}
|
||||
}
|
||||
return expr
|
||||
}
|
||||
@@ -1,327 +0,0 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/google/cel-go/cel"
|
||||
"github.com/google/cel-go/ext"
|
||||
)
|
||||
|
||||
// DialectName enumerates supported SQL dialects.
|
||||
type DialectName string
|
||||
|
||||
const (
|
||||
DialectSQLite DialectName = "sqlite"
|
||||
DialectMySQL DialectName = "mysql"
|
||||
DialectPostgres DialectName = "postgres"
|
||||
)
|
||||
|
||||
// FieldType represents the logical type of a field.
|
||||
type FieldType string
|
||||
|
||||
const (
|
||||
FieldTypeString FieldType = "string"
|
||||
FieldTypeInt FieldType = "int"
|
||||
FieldTypeBool FieldType = "bool"
|
||||
FieldTypeTimestamp FieldType = "timestamp"
|
||||
)
|
||||
|
||||
// FieldKind describes how a field is stored.
|
||||
type FieldKind string
|
||||
|
||||
const (
|
||||
FieldKindScalar FieldKind = "scalar"
|
||||
FieldKindBoolColumn FieldKind = "bool_column"
|
||||
FieldKindJSONBool FieldKind = "json_bool"
|
||||
FieldKindJSONList FieldKind = "json_list"
|
||||
FieldKindVirtualAlias FieldKind = "virtual_alias"
|
||||
)
|
||||
|
||||
// Column identifies the backing table column.
|
||||
type Column struct {
|
||||
Table string
|
||||
Name string
|
||||
}
|
||||
|
||||
// Field captures the schema metadata for an exposed CEL identifier.
|
||||
type Field struct {
|
||||
Name string
|
||||
Kind FieldKind
|
||||
Type FieldType
|
||||
Column Column
|
||||
JSONPath []string
|
||||
AliasFor string
|
||||
SupportsContains bool
|
||||
Expressions map[DialectName]string
|
||||
AllowedComparisonOps map[ComparisonOperator]bool
|
||||
}
|
||||
|
||||
// Schema collects CEL environment options and field metadata.
|
||||
type Schema struct {
|
||||
Name string
|
||||
Fields map[string]Field
|
||||
EnvOptions []cel.EnvOption
|
||||
}
|
||||
|
||||
// Field returns the field metadata if present.
|
||||
func (s Schema) Field(name string) (Field, bool) {
|
||||
f, ok := s.Fields[name]
|
||||
return f, ok
|
||||
}
|
||||
|
||||
// ResolveAlias resolves a virtual alias to its target field.
|
||||
func (s Schema) ResolveAlias(name string) (Field, bool) {
|
||||
field, ok := s.Fields[name]
|
||||
if !ok {
|
||||
return Field{}, false
|
||||
}
|
||||
if field.Kind == FieldKindVirtualAlias {
|
||||
target, ok := s.Fields[field.AliasFor]
|
||||
if !ok {
|
||||
return Field{}, false
|
||||
}
|
||||
return target, true
|
||||
}
|
||||
return field, true
|
||||
}
|
||||
|
||||
// NewSchema constructs the memo filter schema and CEL environment.
|
||||
func NewSchema() Schema {
|
||||
fields := map[string]Field{
|
||||
"content": {
|
||||
Name: "content",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeString,
|
||||
Column: Column{Table: "memo", Name: "content"},
|
||||
SupportsContains: true,
|
||||
Expressions: map[DialectName]string{},
|
||||
},
|
||||
"creator": {
|
||||
Name: "creator",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeString,
|
||||
Column: Column{Table: "memo_creator", Name: "username"},
|
||||
Expressions: map[DialectName]string{
|
||||
DialectSQLite: "('users/' || %s)",
|
||||
DialectMySQL: "CONCAT('users/', %s)",
|
||||
DialectPostgres: "('users/' || %s)",
|
||||
},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
"creator_id": {
|
||||
Name: "creator_id",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeInt,
|
||||
Column: Column{Table: "memo", Name: "creator_id"},
|
||||
Expressions: map[DialectName]string{},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
"created_ts": {
|
||||
Name: "created_ts",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeTimestamp,
|
||||
Column: Column{Table: "memo", Name: "created_ts"},
|
||||
Expressions: map[DialectName]string{
|
||||
// MySQL stores created_ts as TIMESTAMP, needs conversion to epoch
|
||||
DialectMySQL: "UNIX_TIMESTAMP(%s)",
|
||||
// PostgreSQL and SQLite store created_ts as BIGINT (epoch), no conversion needed
|
||||
DialectPostgres: "%s",
|
||||
DialectSQLite: "%s",
|
||||
},
|
||||
},
|
||||
"updated_ts": {
|
||||
Name: "updated_ts",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeTimestamp,
|
||||
Column: Column{Table: "memo", Name: "updated_ts"},
|
||||
Expressions: map[DialectName]string{
|
||||
// MySQL stores updated_ts as TIMESTAMP, needs conversion to epoch
|
||||
DialectMySQL: "UNIX_TIMESTAMP(%s)",
|
||||
// PostgreSQL and SQLite store updated_ts as BIGINT (epoch), no conversion needed
|
||||
DialectPostgres: "%s",
|
||||
DialectSQLite: "%s",
|
||||
},
|
||||
},
|
||||
"pinned": {
|
||||
Name: "pinned",
|
||||
Kind: FieldKindBoolColumn,
|
||||
Type: FieldTypeBool,
|
||||
Column: Column{Table: "memo", Name: "pinned"},
|
||||
Expressions: map[DialectName]string{},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
"visibility": {
|
||||
Name: "visibility",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeString,
|
||||
Column: Column{Table: "memo", Name: "visibility"},
|
||||
Expressions: map[DialectName]string{},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
"tags": {
|
||||
Name: "tags",
|
||||
Kind: FieldKindJSONList,
|
||||
Type: FieldTypeString,
|
||||
Column: Column{Table: "memo", Name: "payload"},
|
||||
JSONPath: []string{"tags"},
|
||||
},
|
||||
"tag": {
|
||||
Name: "tag",
|
||||
Kind: FieldKindVirtualAlias,
|
||||
Type: FieldTypeString,
|
||||
AliasFor: "tags",
|
||||
},
|
||||
"has_task_list": {
|
||||
Name: "has_task_list",
|
||||
Kind: FieldKindJSONBool,
|
||||
Type: FieldTypeBool,
|
||||
Column: Column{Table: "memo", Name: "payload"},
|
||||
JSONPath: []string{"property", "hasTaskList"},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
"has_link": {
|
||||
Name: "has_link",
|
||||
Kind: FieldKindJSONBool,
|
||||
Type: FieldTypeBool,
|
||||
Column: Column{Table: "memo", Name: "payload"},
|
||||
JSONPath: []string{"property", "hasLink"},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
"has_code": {
|
||||
Name: "has_code",
|
||||
Kind: FieldKindJSONBool,
|
||||
Type: FieldTypeBool,
|
||||
Column: Column{Table: "memo", Name: "payload"},
|
||||
JSONPath: []string{"property", "hasCode"},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
"has_incomplete_tasks": {
|
||||
Name: "has_incomplete_tasks",
|
||||
Kind: FieldKindJSONBool,
|
||||
Type: FieldTypeBool,
|
||||
Column: Column{Table: "memo", Name: "payload"},
|
||||
JSONPath: []string{"property", "hasIncompleteTasks"},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
envOptions := []cel.EnvOption{
|
||||
cel.Variable("content", cel.StringType),
|
||||
cel.Variable("creator", cel.StringType),
|
||||
cel.Variable("creator_id", cel.IntType),
|
||||
cel.Variable("created_ts", cel.TimestampType),
|
||||
cel.Variable("updated_ts", cel.TimestampType),
|
||||
cel.Variable("pinned", cel.BoolType),
|
||||
cel.Variable("tag", cel.StringType),
|
||||
cel.Variable("tags", cel.ListType(cel.StringType)),
|
||||
cel.Variable("visibility", cel.StringType),
|
||||
cel.Variable("has_task_list", cel.BoolType),
|
||||
cel.Variable("has_link", cel.BoolType),
|
||||
cel.Variable("has_code", cel.BoolType),
|
||||
cel.Variable("has_incomplete_tasks", cel.BoolType),
|
||||
cel.Variable("now", cel.TimestampType),
|
||||
ext.Sets(),
|
||||
cel.ASTValidators(cel.ValidateRegexLiterals()),
|
||||
}
|
||||
|
||||
return Schema{
|
||||
Name: "memo",
|
||||
Fields: fields,
|
||||
EnvOptions: envOptions,
|
||||
}
|
||||
}
|
||||
|
||||
// NewAttachmentSchema constructs the attachment filter schema and CEL environment.
|
||||
func NewAttachmentSchema() Schema {
|
||||
fields := map[string]Field{
|
||||
"filename": {
|
||||
Name: "filename",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeString,
|
||||
Column: Column{Table: "attachment", Name: "filename"},
|
||||
SupportsContains: true,
|
||||
Expressions: map[DialectName]string{},
|
||||
},
|
||||
"mime_type": {
|
||||
Name: "mime_type",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeString,
|
||||
Column: Column{Table: "attachment", Name: "type"},
|
||||
SupportsContains: true,
|
||||
Expressions: map[DialectName]string{},
|
||||
},
|
||||
"create_time": {
|
||||
Name: "create_time",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeTimestamp,
|
||||
Column: Column{Table: "attachment", Name: "created_ts"},
|
||||
Expressions: map[DialectName]string{
|
||||
// MySQL stores created_ts as TIMESTAMP, needs conversion to epoch
|
||||
DialectMySQL: "UNIX_TIMESTAMP(%s)",
|
||||
// PostgreSQL and SQLite store created_ts as BIGINT (epoch), no conversion needed
|
||||
DialectPostgres: "%s",
|
||||
DialectSQLite: "%s",
|
||||
},
|
||||
},
|
||||
"memo_id": {
|
||||
Name: "memo_id",
|
||||
Kind: FieldKindScalar,
|
||||
Type: FieldTypeInt,
|
||||
Column: Column{Table: "attachment", Name: "memo_id"},
|
||||
Expressions: map[DialectName]string{},
|
||||
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||
CompareEq: true,
|
||||
CompareNeq: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
envOptions := []cel.EnvOption{
|
||||
cel.Variable("filename", cel.StringType),
|
||||
cel.Variable("mime_type", cel.StringType),
|
||||
cel.Variable("create_time", cel.TimestampType),
|
||||
cel.Variable("memo_id", cel.AnyType),
|
||||
cel.Variable("now", cel.TimestampType),
|
||||
cel.ASTValidators(cel.ValidateRegexLiterals()),
|
||||
}
|
||||
|
||||
return Schema{
|
||||
Name: "attachment",
|
||||
Fields: fields,
|
||||
EnvOptions: envOptions,
|
||||
}
|
||||
}
|
||||
|
||||
// columnExpr returns the field expression for the given dialect, applying
|
||||
// any schema-specific overrides (e.g. UNIX timestamp conversions).
|
||||
func (f Field) columnExpr(d DialectName) string {
|
||||
base := qualifyColumn(d, f.Column)
|
||||
if expr, ok := f.Expressions[d]; ok && expr != "" {
|
||||
return fmt.Sprintf(expr, base)
|
||||
}
|
||||
return base
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fixedClock returns a deterministic clock for asserting folded `now` values.
|
||||
func fixedClock(epoch int64) func() time.Time {
|
||||
return func() time.Time { return time.Unix(epoch, 0) }
|
||||
}
|
||||
|
||||
func memoEngineAt(t *testing.T, epoch int64) *Engine {
|
||||
t.Helper()
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
engine.nowFunc = fixedClock(epoch)
|
||||
return engine
|
||||
}
|
||||
|
||||
func TestCompileNowVariableFoldsToInjectedClock(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine := memoEngineAt(t, 1750000000)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `created_ts >= now`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(1750000000)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileNowFunctionIsRemoved(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewSchema())
|
||||
require.NoError(t, err)
|
||||
|
||||
// now() was the legacy custom function; it is replaced by the `now` variable.
|
||||
_, err = engine.Compile(context.Background(), `created_ts >= now()`)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestCompileNowMinusDurationFolds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine := memoEngineAt(t, 1750000000)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `created_ts >= now - duration("1h")`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(1750000000 - 3600)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileNowPlusDurationFolds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine := memoEngineAt(t, 1750000000)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `updated_ts < now + duration("24h")`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(1750000000 + 86400)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileAbsoluteTimestampStringFolds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine := memoEngineAt(t, 1750000000)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `created_ts >= timestamp("2025-01-01T00:00:00Z")`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(1735689600)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileTimestampFromEpochIntFolds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// This is the shape the frontend date-range filter emits.
|
||||
engine := memoEngineAt(t, 1750000000)
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `created_ts >= timestamp(1730000000)`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(1730000000)}, stmt.Args)
|
||||
}
|
||||
|
||||
func TestCompileInvalidDurationLiteralErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine := memoEngineAt(t, 1750000000)
|
||||
_, err := engine.Compile(context.Background(), `created_ts >= now - duration("garbage")`)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "duration")
|
||||
}
|
||||
|
||||
func TestCompileAttachmentCreateTimeUsesNow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
engine, err := NewEngine(NewAttachmentSchema())
|
||||
require.NoError(t, err)
|
||||
engine.nowFunc = fixedClock(1750000000)
|
||||
|
||||
stmt, err := engine.CompileToStatement(context.Background(), `create_time >= now - duration("24h")`, RenderOptions{Dialect: DialectSQLite})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []any{int64(1750000000 - 86400)}, stmt.Args)
|
||||
}
|
||||
@@ -1,227 +0,0 @@
|
||||
package httpgetter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
var ErrInternalIP = errors.New("internal IP addresses are not allowed")
|
||||
|
||||
const maxHTMLMetaBytes = 512 * 1024
|
||||
|
||||
var (
|
||||
lookupIPAddr = net.DefaultResolver.LookupIPAddr
|
||||
dialContext = (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext
|
||||
httpClient = newHTTPClient()
|
||||
)
|
||||
|
||||
func newHTTPClient() *http.Client {
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = secureDialContext
|
||||
|
||||
return &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: 5 * time.Second,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if err := validateURL(req.URL.String()); err != nil {
|
||||
return errors.Wrap(err, "redirect to internal IP")
|
||||
}
|
||||
if len(via) >= 10 {
|
||||
return errors.New("too many redirects")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func secureDialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "invalid address")
|
||||
}
|
||||
|
||||
ips, err := resolveAllowedIPs(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var dialErr error
|
||||
for _, ip := range ips {
|
||||
conn, err := dialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
dialErr = err
|
||||
}
|
||||
return nil, dialErr
|
||||
}
|
||||
|
||||
func resolveAllowedIPs(ctx context.Context, host string) ([]net.IP, error) {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if isInternalIP(ip) {
|
||||
return nil, errors.Wrap(ErrInternalIP, ip.String())
|
||||
}
|
||||
return []net.IP{ip}, nil
|
||||
}
|
||||
|
||||
addrs, err := lookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("failed to resolve hostname: %v", err)
|
||||
}
|
||||
|
||||
ips := make([]net.IP, 0, len(addrs))
|
||||
for _, addr := range addrs {
|
||||
ip := addr.IP
|
||||
if ip == nil {
|
||||
continue
|
||||
}
|
||||
if isInternalIP(ip) {
|
||||
return nil, errors.Wrapf(ErrInternalIP, "host=%s, ip=%s", host, ip.String())
|
||||
}
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
if len(ips) == 0 {
|
||||
return nil, errors.New("hostname resolved to no addresses")
|
||||
}
|
||||
|
||||
return ips, nil
|
||||
}
|
||||
|
||||
func isInternalIP(ip net.IP) bool {
|
||||
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()
|
||||
}
|
||||
|
||||
func validateURL(urlStr string) error {
|
||||
u, err := url.Parse(urlStr)
|
||||
if err != nil {
|
||||
return errors.New("invalid URL format")
|
||||
}
|
||||
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return errors.New("only http/https protocols are allowed")
|
||||
}
|
||||
|
||||
host := u.Hostname()
|
||||
if host == "" {
|
||||
return errors.New("empty hostname")
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(host); ip != nil && isInternalIP(ip) {
|
||||
return errors.Wrap(ErrInternalIP, ip.String())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type HTMLMeta struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Image string `json:"image"`
|
||||
}
|
||||
|
||||
func GetHTMLMeta(urlStr string) (*HTMLMeta, error) {
|
||||
if err := validateURL(urlStr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
response, err := httpClient.Get(urlStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
mediatype, err := getMediatype(response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if mediatype != "text/html" {
|
||||
return nil, errors.New("not a HTML page")
|
||||
}
|
||||
|
||||
htmlMeta := extractHTMLMeta(io.LimitReader(response.Body, maxHTMLMetaBytes))
|
||||
enrichSiteMeta(response.Request.URL, htmlMeta)
|
||||
return htmlMeta, nil
|
||||
}
|
||||
|
||||
func extractHTMLMeta(resp io.Reader) *HTMLMeta {
|
||||
tokenizer := html.NewTokenizer(resp)
|
||||
htmlMeta := new(HTMLMeta)
|
||||
|
||||
for {
|
||||
tokenType := tokenizer.Next()
|
||||
if tokenType == html.ErrorToken {
|
||||
break
|
||||
} else if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken {
|
||||
token := tokenizer.Token()
|
||||
if token.DataAtom == atom.Body {
|
||||
break
|
||||
}
|
||||
|
||||
if token.DataAtom == atom.Title {
|
||||
tokenizer.Next()
|
||||
token := tokenizer.Token()
|
||||
htmlMeta.Title = token.Data
|
||||
} else if token.DataAtom == atom.Meta {
|
||||
ogTitle, ok := extractMetaProperty(token, "og:title")
|
||||
if ok {
|
||||
htmlMeta.Title = ogTitle
|
||||
}
|
||||
|
||||
ogDescription, ok := extractMetaProperty(token, "og:description")
|
||||
if ok {
|
||||
htmlMeta.Description = ogDescription
|
||||
}
|
||||
|
||||
ogImage, ok := extractMetaProperty(token, "og:image")
|
||||
if ok {
|
||||
htmlMeta.Image = ogImage
|
||||
}
|
||||
|
||||
description, ok := extractMetaProperty(token, "description")
|
||||
if ok && htmlMeta.Description == "" {
|
||||
htmlMeta.Description = description
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return htmlMeta
|
||||
}
|
||||
|
||||
func extractMetaProperty(token html.Token, prop string) (content string, ok bool) {
|
||||
content, ok = "", false
|
||||
for _, attr := range token.Attr {
|
||||
if (attr.Key == "property" || attr.Key == "name") && strings.EqualFold(attr.Val, prop) {
|
||||
ok = true
|
||||
}
|
||||
if attr.Key == "content" {
|
||||
content = attr.Val
|
||||
}
|
||||
}
|
||||
return content, ok
|
||||
}
|
||||
|
||||
func enrichSiteMeta(url *url.URL, meta *HTMLMeta) {
|
||||
if url.Hostname() == "www.youtube.com" {
|
||||
if url.Path == "/watch" {
|
||||
vid := url.Query().Get("v")
|
||||
if vid != "" {
|
||||
meta.Image = fmt.Sprintf("https://img.youtube.com/vi/%s/mqdefault.jpg", vid)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,189 +0,0 @@
|
||||
package httpgetter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return f(req)
|
||||
}
|
||||
|
||||
func TestGetHTMLMeta(t *testing.T) {
|
||||
originalHTTPClient := httpClient
|
||||
t.Cleanup(func() {
|
||||
httpClient = originalHTTPClient
|
||||
})
|
||||
|
||||
httpClient = &http.Client{
|
||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "http://93.184.216.34/article", req.URL.String())
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
|
||||
Body: io.NopCloser(strings.NewReader(`<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Fallback title</title>
|
||||
<meta name="description" content="Fallback description">
|
||||
<meta property="og:title" content="Open Graph title">
|
||||
<meta property="og:description" content="Open Graph description">
|
||||
<meta property="og:image" content="https://example.com/cover.png">
|
||||
</head>
|
||||
<body>ignored</body>
|
||||
</html>`)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}),
|
||||
}
|
||||
|
||||
metadata, err := GetHTMLMeta("http://93.184.216.34/article")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, HTMLMeta{
|
||||
Title: "Open Graph title",
|
||||
Description: "Open Graph description",
|
||||
Image: "https://example.com/cover.png",
|
||||
}, *metadata)
|
||||
}
|
||||
|
||||
func TestGetHTMLMetaWithNameOnly(t *testing.T) {
|
||||
originalHTTPClient := httpClient
|
||||
t.Cleanup(func() {
|
||||
httpClient = originalHTTPClient
|
||||
})
|
||||
|
||||
httpClient = &http.Client{
|
||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "http://93.184.216.34/blog", req.URL.String())
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
|
||||
Body: io.NopCloser(strings.NewReader(`<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Sample Page</title>
|
||||
<meta name="description" content="This description should appear in the link preview.">
|
||||
</head>
|
||||
<body>Hello</body>
|
||||
</html>`)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}),
|
||||
}
|
||||
|
||||
metadata, err := GetHTMLMeta("http://93.184.216.34/blog")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, HTMLMeta{
|
||||
Title: "Sample Page",
|
||||
Description: "This description should appear in the link preview.",
|
||||
Image: "",
|
||||
}, *metadata)
|
||||
}
|
||||
|
||||
func TestGetHTMLMetaWithNameCaseInsensitive(t *testing.T) {
|
||||
originalHTTPClient := httpClient
|
||||
t.Cleanup(func() {
|
||||
httpClient = originalHTTPClient
|
||||
})
|
||||
|
||||
httpClient = &http.Client{
|
||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "http://93.184.216.34/blog", req.URL.String())
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
|
||||
Body: io.NopCloser(strings.NewReader(`<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Sample Page</title>
|
||||
<meta name="Description" content="Case insensitive description match.">
|
||||
</head>
|
||||
<body>Hello</body>
|
||||
</html>`)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}),
|
||||
}
|
||||
|
||||
metadata, err := GetHTMLMeta("http://93.184.216.34/blog")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, HTMLMeta{
|
||||
Title: "Sample Page",
|
||||
Description: "Case insensitive description match.",
|
||||
Image: "",
|
||||
}, *metadata)
|
||||
}
|
||||
|
||||
func TestGetHTMLMetaForInternal(t *testing.T) {
|
||||
// test for internal IP
|
||||
if _, err := GetHTMLMeta("http://192.168.0.1"); !errors.Is(err, ErrInternalIP) {
|
||||
t.Errorf("Expected error for internal IP, got %v", err)
|
||||
}
|
||||
|
||||
// test for resolved internal IP
|
||||
if _, err := GetHTMLMeta("http://localhost"); !errors.Is(err, ErrInternalIP) {
|
||||
t.Errorf("Expected error for resolved internal IP, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientHasTimeout(t *testing.T) {
|
||||
require.NotZero(t, httpClient.Timeout)
|
||||
}
|
||||
|
||||
func TestSecureDialContextRejectsResolvedInternalIP(t *testing.T) {
|
||||
originalLookupIPAddr := lookupIPAddr
|
||||
originalDialContext := dialContext
|
||||
t.Cleanup(func() {
|
||||
lookupIPAddr = originalLookupIPAddr
|
||||
dialContext = originalDialContext
|
||||
})
|
||||
|
||||
lookupIPAddr = func(context.Context, string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}}, nil
|
||||
}
|
||||
dialContext = func(context.Context, string, string) (net.Conn, error) {
|
||||
t.Fatal("internal IP should be rejected before dialing")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
_, err := secureDialContext(context.Background(), "tcp", "rebind.example:80")
|
||||
require.ErrorIs(t, err, ErrInternalIP)
|
||||
}
|
||||
|
||||
func TestSecureDialContextDialsResolvedIP(t *testing.T) {
|
||||
originalLookupIPAddr := lookupIPAddr
|
||||
originalDialContext := dialContext
|
||||
t.Cleanup(func() {
|
||||
lookupIPAddr = originalLookupIPAddr
|
||||
dialContext = originalDialContext
|
||||
})
|
||||
|
||||
lookupIPAddr = func(context.Context, string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
||||
}
|
||||
|
||||
var dialedAddress string
|
||||
dialContext = func(_ context.Context, _ string, address string) (net.Conn, error) {
|
||||
dialedAddress = address
|
||||
clientConn, serverConn := net.Pipe()
|
||||
t.Cleanup(func() {
|
||||
clientConn.Close()
|
||||
serverConn.Close()
|
||||
})
|
||||
return clientConn, nil
|
||||
}
|
||||
|
||||
conn, err := secureDialContext(context.Background(), "tcp", "rebind.example:80")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, conn)
|
||||
require.Equal(t, "93.184.216.34:80", dialedAddress)
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
package httpgetter
|
||||
@@ -1,45 +0,0 @@
|
||||
package httpgetter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Image struct {
|
||||
Blob []byte
|
||||
Mediatype string
|
||||
}
|
||||
|
||||
func GetImage(urlStr string) (*Image, error) {
|
||||
if _, err := url.Parse(urlStr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
response, err := http.Get(urlStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
mediatype, err := getMediatype(response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !strings.HasPrefix(mediatype, "image/") {
|
||||
return nil, errors.New("wrong image mediatype")
|
||||
}
|
||||
|
||||
bodyBytes, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
image := &Image{
|
||||
Blob: bodyBytes,
|
||||
Mediatype: mediatype,
|
||||
}
|
||||
return image, nil
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package httpgetter
|
||||
|
||||
import (
|
||||
"mime"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func getMediatype(response *http.Response) (string, error) {
|
||||
contentType := response.Header.Get("content-type")
|
||||
mediatype, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return mediatype, nil
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
package idp
|
||||
|
||||
type IdentityProviderUserInfo struct {
|
||||
Identifier string
|
||||
DisplayName string
|
||||
Email string
|
||||
AvatarURL string
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
// Package oauth2 implements the OAuth2 identity provider integration.
|
||||
package oauth2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
"github.com/usememos/memos/internal/idp"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
)
|
||||
|
||||
// IdentityProvider represents an OAuth2 Identity Provider.
|
||||
type IdentityProvider struct {
|
||||
config *storepb.OAuth2Config
|
||||
}
|
||||
|
||||
const userInfoRequestTimeout = 10 * time.Second
|
||||
|
||||
// NewIdentityProvider initializes a new OAuth2 Identity Provider with the given configuration.
|
||||
func NewIdentityProvider(config *storepb.OAuth2Config) (*IdentityProvider, error) {
|
||||
for v, field := range map[string]string{
|
||||
config.ClientId: "clientId",
|
||||
config.ClientSecret: "clientSecret",
|
||||
config.TokenUrl: "tokenUrl",
|
||||
config.UserInfoUrl: "userInfoUrl",
|
||||
config.FieldMapping.Identifier: "fieldMapping.identifier",
|
||||
} {
|
||||
if v == "" {
|
||||
return nil, errors.Errorf(`the field "%s" is empty but required`, field)
|
||||
}
|
||||
}
|
||||
|
||||
return &IdentityProvider{
|
||||
config: config,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ExchangeToken returns the exchanged OAuth2 token using the given authorization code.
|
||||
// If codeVerifier is provided, it will be used for PKCE (Proof Key for Code Exchange) validation.
|
||||
func (p *IdentityProvider) ExchangeToken(ctx context.Context, redirectURL, code, codeVerifier string) (string, error) {
|
||||
conf := &oauth2.Config{
|
||||
ClientID: p.config.ClientId,
|
||||
ClientSecret: p.config.ClientSecret,
|
||||
RedirectURL: redirectURL,
|
||||
Scopes: p.config.Scopes,
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: p.config.AuthUrl,
|
||||
TokenURL: p.config.TokenUrl,
|
||||
AuthStyle: oauth2.AuthStyleAutoDetect,
|
||||
},
|
||||
}
|
||||
|
||||
// Prepare token exchange options
|
||||
opts := []oauth2.AuthCodeOption{}
|
||||
|
||||
// Add PKCE code_verifier if provided
|
||||
if codeVerifier != "" {
|
||||
opts = append(opts, oauth2.SetAuthURLParam("code_verifier", codeVerifier))
|
||||
}
|
||||
|
||||
token, err := conf.Exchange(ctx, code, opts...)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to exchange access token")
|
||||
}
|
||||
|
||||
// Use the standard AccessToken field instead of Extra()
|
||||
// This is more reliable across different OAuth providers
|
||||
if token.AccessToken == "" {
|
||||
return "", errors.New("missing access token from authorization response")
|
||||
}
|
||||
|
||||
return token.AccessToken, nil
|
||||
}
|
||||
|
||||
// UserInfo returns the parsed user information using the given OAuth2 token.
|
||||
func (p *IdentityProvider) UserInfo(ctx context.Context, token string) (*idp.IdentityProviderUserInfo, error) {
|
||||
client := &http.Client{Timeout: userInfoRequestTimeout}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.config.UserInfoUrl, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create http request")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get user information")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
if readErr != nil {
|
||||
return nil, errors.Wrap(readErr, "failed to read error response body")
|
||||
}
|
||||
return nil, errors.Errorf("userinfo request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read response body")
|
||||
}
|
||||
|
||||
var claims map[string]any
|
||||
if err := json.Unmarshal(body, &claims); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to unmarshal response body")
|
||||
}
|
||||
userInfo := &idp.IdentityProviderUserInfo{}
|
||||
if v, ok := claims[p.config.FieldMapping.Identifier].(string); ok {
|
||||
userInfo.Identifier = v
|
||||
}
|
||||
if userInfo.Identifier == "" {
|
||||
return nil, errors.Errorf("the field %q is not found in claims or has empty value", p.config.FieldMapping.Identifier)
|
||||
}
|
||||
|
||||
// Best effort to map optional fields
|
||||
if p.config.FieldMapping.DisplayName != "" {
|
||||
if v, ok := claims[p.config.FieldMapping.DisplayName].(string); ok {
|
||||
userInfo.DisplayName = v
|
||||
}
|
||||
}
|
||||
if userInfo.DisplayName == "" {
|
||||
userInfo.DisplayName = userInfo.Identifier
|
||||
}
|
||||
if p.config.FieldMapping.Email != "" {
|
||||
if v, ok := claims[p.config.FieldMapping.Email].(string); ok {
|
||||
userInfo.Email = v
|
||||
}
|
||||
}
|
||||
if p.config.FieldMapping.AvatarUrl != "" {
|
||||
if v, ok := claims[p.config.FieldMapping.AvatarUrl].(string); ok {
|
||||
userInfo.AvatarURL = v
|
||||
}
|
||||
}
|
||||
return userInfo, nil
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
package oauth2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/usememos/memos/internal/idp"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
)
|
||||
|
||||
func TestNewIdentityProvider(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config *storepb.OAuth2Config
|
||||
containsErr string
|
||||
}{
|
||||
{
|
||||
name: "no tokenUrl",
|
||||
config: &storepb.OAuth2Config{
|
||||
ClientId: "test-client-id",
|
||||
ClientSecret: "test-client-secret",
|
||||
AuthUrl: "",
|
||||
TokenUrl: "",
|
||||
UserInfoUrl: "https://example.com/api/user",
|
||||
FieldMapping: &storepb.FieldMapping{
|
||||
Identifier: "login",
|
||||
},
|
||||
},
|
||||
containsErr: `the field "tokenUrl" is empty but required`,
|
||||
},
|
||||
{
|
||||
name: "no userInfoUrl",
|
||||
config: &storepb.OAuth2Config{
|
||||
ClientId: "test-client-id",
|
||||
ClientSecret: "test-client-secret",
|
||||
AuthUrl: "",
|
||||
TokenUrl: "https://example.com/token",
|
||||
UserInfoUrl: "",
|
||||
FieldMapping: &storepb.FieldMapping{
|
||||
Identifier: "login",
|
||||
},
|
||||
},
|
||||
containsErr: `the field "userInfoUrl" is empty but required`,
|
||||
},
|
||||
{
|
||||
name: "no field mapping identifier",
|
||||
config: &storepb.OAuth2Config{
|
||||
ClientId: "test-client-id",
|
||||
ClientSecret: "test-client-secret",
|
||||
AuthUrl: "",
|
||||
TokenUrl: "https://example.com/token",
|
||||
UserInfoUrl: "https://example.com/api/user",
|
||||
FieldMapping: &storepb.FieldMapping{
|
||||
Identifier: "",
|
||||
},
|
||||
},
|
||||
containsErr: `the field "fieldMapping.identifier" is empty but required`,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(*testing.T) {
|
||||
_, err := NewIdentityProvider(test.config)
|
||||
assert.ErrorContains(t, err, test.containsErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newMockServer(t *testing.T, code, accessToken string, userinfo []byte) *httptest.Server {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
var rawIDToken string
|
||||
mux.HandleFunc("/oauth2/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodPost, r.Method)
|
||||
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
vals, err := url.ParseQuery(string(body))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, code, vals.Get("code"))
|
||||
require.Equal(t, "authorization_code", vals.Get("grant_type"))
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(map[string]any{
|
||||
"access_token": accessToken,
|
||||
"token_type": "Bearer",
|
||||
"expires_in": 3600,
|
||||
"id_token": rawIDToken,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
mux.HandleFunc("/oauth2/userinfo", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := w.Write(userinfo)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
s := httptest.NewServer(mux)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func TestIdentityProvider(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
const (
|
||||
testClientID = "test-client-id"
|
||||
testCode = "test-code"
|
||||
testAccessToken = "test-access-token"
|
||||
testSubject = "123456789"
|
||||
testName = "John Doe"
|
||||
testEmail = "john.doe@example.com"
|
||||
)
|
||||
userInfo, err := json.Marshal(
|
||||
map[string]any{
|
||||
"sub": testSubject,
|
||||
"name": testName,
|
||||
"email": testEmail,
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
s := newMockServer(t, testCode, testAccessToken, userInfo)
|
||||
|
||||
oauth2, err := NewIdentityProvider(
|
||||
&storepb.OAuth2Config{
|
||||
ClientId: testClientID,
|
||||
ClientSecret: "test-client-secret",
|
||||
TokenUrl: fmt.Sprintf("%s/oauth2/token", s.URL),
|
||||
UserInfoUrl: fmt.Sprintf("%s/oauth2/userinfo", s.URL),
|
||||
FieldMapping: &storepb.FieldMapping{
|
||||
Identifier: "sub",
|
||||
DisplayName: "name",
|
||||
Email: "email",
|
||||
},
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
redirectURL := "https://example.com/oauth/callback"
|
||||
// Test without PKCE (backward compatibility)
|
||||
oauthToken, err := oauth2.ExchangeToken(ctx, redirectURL, testCode, "")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, testAccessToken, oauthToken)
|
||||
|
||||
userInfoResult, err := oauth2.UserInfo(ctx, oauthToken)
|
||||
require.NoError(t, err)
|
||||
|
||||
wantUserInfo := &idp.IdentityProviderUserInfo{
|
||||
Identifier: testSubject,
|
||||
DisplayName: testName,
|
||||
Email: testEmail,
|
||||
}
|
||||
assert.Equal(t, wantUserInfo, userInfoResult)
|
||||
}
|
||||
|
||||
func TestIdentityProviderExchangeTokenClientAuthentication(t *testing.T) {
|
||||
const (
|
||||
clientID = "test-client-id"
|
||||
clientSecret = "test-client-secret"
|
||||
code = "test-code"
|
||||
accessToken = "test-access-token"
|
||||
codeVerifier = "test-code-verifier"
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
acceptBasicAuth bool
|
||||
expectedRequests int
|
||||
}{
|
||||
{
|
||||
name: "client secret basic",
|
||||
acceptBasicAuth: true,
|
||||
expectedRequests: 1,
|
||||
},
|
||||
{
|
||||
name: "client secret post fallback",
|
||||
acceptBasicAuth: false,
|
||||
expectedRequests: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
requestCount := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
require.NoError(t, r.ParseForm())
|
||||
require.Equal(t, code, r.Form.Get("code"))
|
||||
require.Equal(t, codeVerifier, r.Form.Get("code_verifier"))
|
||||
|
||||
username, password, hasBasicAuth := r.BasicAuth()
|
||||
if test.acceptBasicAuth {
|
||||
require.True(t, hasBasicAuth)
|
||||
require.Equal(t, clientID, username)
|
||||
require.Equal(t, clientSecret, password)
|
||||
require.Empty(t, r.Form.Get("client_id"))
|
||||
require.Empty(t, r.Form.Get("client_secret"))
|
||||
} else if hasBasicAuth {
|
||||
http.Error(w, `{"error":"invalid_client"}`, http.StatusUnauthorized)
|
||||
return
|
||||
} else {
|
||||
require.Equal(t, clientID, r.Form.Get("client_id"))
|
||||
require.Equal(t, clientSecret, r.Form.Get("client_secret"))
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
|
||||
"access_token": accessToken,
|
||||
"token_type": "Bearer",
|
||||
}))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
provider, err := NewIdentityProvider(&storepb.OAuth2Config{
|
||||
ClientId: clientID,
|
||||
ClientSecret: clientSecret,
|
||||
TokenUrl: server.URL,
|
||||
UserInfoUrl: "https://example.com/oauth2/userinfo",
|
||||
FieldMapping: &storepb.FieldMapping{Identifier: "sub"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
token, err := provider.ExchangeToken(context.Background(), "https://example.com/auth/callback", code, codeVerifier)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, accessToken, token)
|
||||
assert.Equal(t, test.expectedRequests, requestCount)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentityProviderUserInfoUsesContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer s.Close()
|
||||
|
||||
oauth2, err := NewIdentityProvider(
|
||||
&storepb.OAuth2Config{
|
||||
ClientId: "test-client-id",
|
||||
ClientSecret: "test-client-secret",
|
||||
TokenUrl: "https://example.com/oauth2/token",
|
||||
UserInfoUrl: s.URL,
|
||||
FieldMapping: &storepb.FieldMapping{
|
||||
Identifier: "sub",
|
||||
},
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = oauth2.UserInfo(ctx, "test-access-token")
|
||||
require.Error(t, err)
|
||||
assert.ErrorContains(t, err, "failed to get user information")
|
||||
}
|
||||
|
||||
func TestIdentityProviderUserInfoRejectsNon2xx(t *testing.T) {
|
||||
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "upstream failure", http.StatusBadGateway)
|
||||
}))
|
||||
defer s.Close()
|
||||
|
||||
oauth2, err := NewIdentityProvider(
|
||||
&storepb.OAuth2Config{
|
||||
ClientId: "test-client-id",
|
||||
ClientSecret: "test-client-secret",
|
||||
TokenUrl: "https://example.com/oauth2/token",
|
||||
UserInfoUrl: s.URL,
|
||||
FieldMapping: &storepb.FieldMapping{
|
||||
Identifier: "sub",
|
||||
},
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = oauth2.UserInfo(context.Background(), "test-access-token")
|
||||
require.Error(t, err)
|
||||
assert.ErrorContains(t, err, "userinfo request failed with status 502")
|
||||
assert.ErrorContains(t, err, "upstream failure")
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package ast
|
||||
|
||||
import (
|
||||
gast "github.com/yuin/goldmark/ast"
|
||||
)
|
||||
|
||||
// MentionNode represents an @mention in the markdown AST.
|
||||
type MentionNode struct {
|
||||
gast.BaseInline
|
||||
|
||||
// Username without the @ prefix.
|
||||
Username []byte
|
||||
}
|
||||
|
||||
// KindMention is the NodeKind for MentionNode.
|
||||
var KindMention = gast.NewNodeKind("Mention")
|
||||
|
||||
// Kind returns KindMention.
|
||||
func (*MentionNode) Kind() gast.NodeKind {
|
||||
return KindMention
|
||||
}
|
||||
|
||||
// Dump implements Node.Dump for debugging.
|
||||
func (n *MentionNode) Dump(source []byte, level int) {
|
||||
gast.DumpHelper(n, source, level, map[string]string{
|
||||
"Username": string(n.Username),
|
||||
}, nil)
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package ast
|
||||
|
||||
import (
|
||||
gast "github.com/yuin/goldmark/ast"
|
||||
)
|
||||
|
||||
// TagNode represents a #tag in the markdown AST.
|
||||
type TagNode struct {
|
||||
gast.BaseInline
|
||||
|
||||
// Tag name without the # prefix
|
||||
Tag []byte
|
||||
}
|
||||
|
||||
// KindTag is the NodeKind for TagNode.
|
||||
var KindTag = gast.NewNodeKind("Tag")
|
||||
|
||||
// Kind returns KindTag.
|
||||
func (*TagNode) Kind() gast.NodeKind {
|
||||
return KindTag
|
||||
}
|
||||
|
||||
// Dump implements Node.Dump for debugging.
|
||||
func (n *TagNode) Dump(source []byte, level int) {
|
||||
gast.DumpHelper(n, source, level, map[string]string{
|
||||
"Tag": string(n.Tag),
|
||||
}, nil)
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package extensions
|
||||
|
||||
import (
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/parser"
|
||||
"github.com/yuin/goldmark/util"
|
||||
|
||||
mparser "github.com/usememos/memos/internal/markdown/parser"
|
||||
)
|
||||
|
||||
type mentionExtension struct{}
|
||||
|
||||
// MentionExtension is a goldmark extension for @mention syntax.
|
||||
var MentionExtension = &mentionExtension{}
|
||||
|
||||
// Extend extends the goldmark parser with mention support.
|
||||
func (*mentionExtension) Extend(m goldmark.Markdown) {
|
||||
m.Parser().AddOptions(
|
||||
parser.WithInlineParsers(
|
||||
// Priority 200 - run before standard link parser (500).
|
||||
util.Prioritized(mparser.NewMentionParser(), 200),
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package extensions
|
||||
|
||||
import (
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/parser"
|
||||
"github.com/yuin/goldmark/util"
|
||||
|
||||
mparser "github.com/usememos/memos/internal/markdown/parser"
|
||||
)
|
||||
|
||||
type tagExtension struct{}
|
||||
|
||||
// TagExtension is a goldmark extension for #tag syntax.
|
||||
var TagExtension = &tagExtension{}
|
||||
|
||||
// Extend extends the goldmark parser with tag support.
|
||||
func (*tagExtension) Extend(m goldmark.Markdown) {
|
||||
m.Parser().AddOptions(
|
||||
parser.WithInlineParsers(
|
||||
// Priority 200 - run before standard link parser (500)
|
||||
util.Prioritized(mparser.NewTagParser(), 200),
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -1,489 +0,0 @@
|
||||
package markdown
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/yuin/goldmark"
|
||||
gast "github.com/yuin/goldmark/ast"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
east "github.com/yuin/goldmark/extension/ast"
|
||||
"github.com/yuin/goldmark/parser"
|
||||
"github.com/yuin/goldmark/text"
|
||||
|
||||
mast "github.com/usememos/memos/internal/markdown/ast"
|
||||
"github.com/usememos/memos/internal/markdown/extensions"
|
||||
"github.com/usememos/memos/internal/markdown/renderer"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
)
|
||||
|
||||
// ExtractedData contains all metadata extracted from markdown in a single pass.
|
||||
type ExtractedData struct {
|
||||
Tags []string
|
||||
Mentions []string
|
||||
Property *storepb.MemoPayload_Property
|
||||
}
|
||||
|
||||
// Service handles markdown metadata extraction.
|
||||
// It uses goldmark to parse markdown and extract tags, properties, and snippets.
|
||||
// HTML rendering is primarily done on frontend using markdown-it, but backend provides
|
||||
// RenderHTML for RSS feeds and other server-side rendering needs.
|
||||
type Service interface {
|
||||
// ExtractAll extracts tags, properties, and references in a single parse (most efficient)
|
||||
ExtractAll(content []byte) (*ExtractedData, error)
|
||||
|
||||
// ExtractTags returns all #tags found in content
|
||||
ExtractTags(content []byte) ([]string, error)
|
||||
|
||||
// ExtractProperties computes boolean properties
|
||||
ExtractProperties(content []byte) (*storepb.MemoPayload_Property, error)
|
||||
|
||||
// RenderMarkdown renders goldmark AST back to markdown text
|
||||
RenderMarkdown(content []byte) (string, error)
|
||||
|
||||
// RenderHTML renders markdown content to HTML
|
||||
RenderHTML(content []byte) (string, error)
|
||||
|
||||
// GenerateSnippet creates plain text summary
|
||||
GenerateSnippet(content []byte, maxLength int) (string, error)
|
||||
|
||||
// ValidateContent checks for syntax errors
|
||||
ValidateContent(content []byte) error
|
||||
|
||||
// RenameTag renames all occurrences of oldTag to newTag in content
|
||||
RenameTag(content []byte, oldTag, newTag string) (string, error)
|
||||
}
|
||||
|
||||
// service implements the Service interface.
|
||||
type service struct {
|
||||
md goldmark.Markdown
|
||||
}
|
||||
|
||||
// Option configures the markdown service.
|
||||
type Option func(*config)
|
||||
|
||||
type config struct {
|
||||
enableTags bool
|
||||
enableMentions bool
|
||||
}
|
||||
|
||||
// WithTagExtension enables #tag parsing.
|
||||
func WithTagExtension() Option {
|
||||
return func(c *config) {
|
||||
c.enableTags = true
|
||||
}
|
||||
}
|
||||
|
||||
// WithMentionExtension enables @mention parsing.
|
||||
func WithMentionExtension() Option {
|
||||
return func(c *config) {
|
||||
c.enableMentions = true
|
||||
}
|
||||
}
|
||||
|
||||
// NewService creates a new markdown service with the given options.
|
||||
func NewService(opts ...Option) Service {
|
||||
cfg := &config{}
|
||||
for _, opt := range opts {
|
||||
opt(cfg)
|
||||
}
|
||||
|
||||
exts := []goldmark.Extender{
|
||||
extension.GFM, // GitHub Flavored Markdown (tables, strikethrough, task lists, autolinks)
|
||||
}
|
||||
|
||||
// Add custom extensions based on config
|
||||
if cfg.enableTags {
|
||||
exts = append(exts, extensions.TagExtension)
|
||||
}
|
||||
if cfg.enableMentions {
|
||||
exts = append(exts, extensions.MentionExtension)
|
||||
}
|
||||
|
||||
md := goldmark.New(
|
||||
goldmark.WithExtensions(exts...),
|
||||
goldmark.WithParserOptions(
|
||||
parser.WithAutoHeadingID(), // Generate heading IDs
|
||||
),
|
||||
)
|
||||
|
||||
return &service{
|
||||
md: md,
|
||||
}
|
||||
}
|
||||
|
||||
// parse is an internal helper to parse content into AST.
|
||||
func (s *service) parse(content []byte) (gast.Node, error) {
|
||||
reader := text.NewReader(content)
|
||||
doc := s.md.Parser().Parse(reader)
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
func isTagNodeInLinkOrImage(n gast.Node) bool {
|
||||
for parent := n.Parent(); parent != nil; parent = parent.Parent() {
|
||||
switch parent.Kind() {
|
||||
case gast.KindLink, gast.KindImage:
|
||||
return true
|
||||
default:
|
||||
// Keep walking ancestors.
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func asMemoTagNode(n gast.Node) (*mast.TagNode, bool) {
|
||||
tagNode, ok := n.(*mast.TagNode)
|
||||
if !ok || isTagNodeInLinkOrImage(n) {
|
||||
return nil, false
|
||||
}
|
||||
return tagNode, true
|
||||
}
|
||||
|
||||
// ExtractTags returns all #tags found in content.
|
||||
func (s *service) ExtractTags(content []byte) ([]string, error) {
|
||||
root, err := s.parse(content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tags []string
|
||||
|
||||
// Walk the AST to find tag nodes
|
||||
err = gast.Walk(root, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return gast.WalkContinue, nil
|
||||
}
|
||||
|
||||
if tagNode, ok := asMemoTagNode(n); ok {
|
||||
tags = append(tags, string(tagNode.Tag))
|
||||
}
|
||||
|
||||
return gast.WalkContinue, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Deduplicate tags while preserving original case
|
||||
return uniquePreserveCase(tags), nil
|
||||
}
|
||||
|
||||
// extractHeadingText extracts plain text content from a heading node.
|
||||
func extractHeadingText(n gast.Node, source []byte) string {
|
||||
var buf strings.Builder
|
||||
for child := n.FirstChild(); child != nil; child = child.NextSibling() {
|
||||
extractTextFromNode(child, source, &buf)
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// extractTextFromNode recursively extracts plain text from a node and its children.
|
||||
func extractTextFromNode(n gast.Node, source []byte, buf *strings.Builder) {
|
||||
if textNode, ok := n.(*gast.Text); ok {
|
||||
buf.Write(textNode.Segment.Value(source))
|
||||
return
|
||||
}
|
||||
for child := n.FirstChild(); child != nil; child = child.NextSibling() {
|
||||
extractTextFromNode(child, source, buf)
|
||||
}
|
||||
}
|
||||
|
||||
// ExtractProperties computes boolean properties about the content.
|
||||
func (s *service) ExtractProperties(content []byte) (*storepb.MemoPayload_Property, error) {
|
||||
root, err := s.parse(content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prop := &storepb.MemoPayload_Property{}
|
||||
firstBlockChecked := false
|
||||
|
||||
err = gast.Walk(root, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return gast.WalkContinue, nil
|
||||
}
|
||||
|
||||
// Check if the first block-level child of the document is an H1 heading.
|
||||
if !firstBlockChecked && n.Parent() != nil && n.Parent().Kind() == gast.KindDocument {
|
||||
firstBlockChecked = true
|
||||
if heading, ok := n.(*gast.Heading); ok && heading.Level == 1 {
|
||||
prop.Title = extractHeadingText(n, content)
|
||||
}
|
||||
}
|
||||
|
||||
switch n.Kind() {
|
||||
case gast.KindLink:
|
||||
prop.HasLink = true
|
||||
|
||||
case gast.KindCodeBlock, gast.KindFencedCodeBlock, gast.KindCodeSpan:
|
||||
prop.HasCode = true
|
||||
|
||||
case east.KindTaskCheckBox:
|
||||
prop.HasTaskList = true
|
||||
if checkBox, ok := n.(*east.TaskCheckBox); ok {
|
||||
if !checkBox.IsChecked {
|
||||
prop.HasIncompleteTasks = true
|
||||
}
|
||||
}
|
||||
default:
|
||||
// No special handling for other node types
|
||||
}
|
||||
|
||||
return gast.WalkContinue, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return prop, nil
|
||||
}
|
||||
|
||||
// RenderMarkdown renders goldmark AST back to markdown text.
|
||||
func (s *service) RenderMarkdown(content []byte) (string, error) {
|
||||
root, err := s.parse(content)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
mdRenderer := renderer.NewMarkdownRenderer()
|
||||
return mdRenderer.Render(root, content), nil
|
||||
}
|
||||
|
||||
// RenderHTML renders markdown content to HTML using goldmark's built-in HTML renderer.
|
||||
func (s *service) RenderHTML(content []byte) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := s.md.Convert(content, &buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
// GenerateSnippet creates a plain text summary from markdown content.
|
||||
func (s *service) GenerateSnippet(content []byte, maxLength int) (string, error) {
|
||||
root, err := s.parse(content)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
var lastNodeWasBlock bool
|
||||
|
||||
err = gast.Walk(root, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
|
||||
if entering {
|
||||
// Skip code blocks entirely (but keep inline code spans for snippet text)
|
||||
switch n.Kind() {
|
||||
case gast.KindCodeBlock, gast.KindFencedCodeBlock:
|
||||
return gast.WalkSkipChildren, nil
|
||||
default:
|
||||
// Continue walking for other node types
|
||||
}
|
||||
|
||||
// Add space before block elements (except first)
|
||||
switch n.Kind() {
|
||||
case gast.KindParagraph, gast.KindHeading, gast.KindListItem, east.KindTableCell, east.KindTableRow, east.KindTableHeader:
|
||||
if buf.Len() > 0 && lastNodeWasBlock {
|
||||
buf.WriteByte(' ')
|
||||
}
|
||||
default:
|
||||
// No space needed for other node types
|
||||
}
|
||||
}
|
||||
|
||||
if !entering {
|
||||
// Mark that we just exited a block element
|
||||
switch n.Kind() {
|
||||
case gast.KindParagraph, gast.KindHeading, gast.KindListItem, east.KindTableCell, east.KindTableRow, east.KindTableHeader:
|
||||
lastNodeWasBlock = true
|
||||
default:
|
||||
// Not a block element
|
||||
}
|
||||
return gast.WalkContinue, nil
|
||||
}
|
||||
|
||||
lastNodeWasBlock = false
|
||||
|
||||
// Extract text from various node types
|
||||
switch node := n.(type) {
|
||||
case *gast.Text:
|
||||
segment := node.Segment
|
||||
buf.Write(segment.Value(content))
|
||||
if node.SoftLineBreak() {
|
||||
buf.WriteByte(' ')
|
||||
}
|
||||
case *gast.AutoLink:
|
||||
buf.Write(node.URL(content))
|
||||
return gast.WalkSkipChildren, nil
|
||||
case *mast.TagNode:
|
||||
buf.WriteByte('#')
|
||||
buf.Write(node.Tag)
|
||||
default:
|
||||
// Ignore other node types.
|
||||
}
|
||||
|
||||
// Stop walking if we've exceeded double the max length
|
||||
// (we'll truncate precisely later)
|
||||
if buf.Len() > maxLength*2 {
|
||||
return gast.WalkStop, nil
|
||||
}
|
||||
|
||||
return gast.WalkContinue, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
snippet := buf.String()
|
||||
|
||||
// Truncate at word boundary if needed
|
||||
if len(snippet) > maxLength {
|
||||
snippet = truncateAtWord(snippet, maxLength)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(snippet), nil
|
||||
}
|
||||
|
||||
// ValidateContent checks if the markdown content is valid.
|
||||
func (s *service) ValidateContent(content []byte) error {
|
||||
// Try to parse the content
|
||||
_, err := s.parse(content)
|
||||
return err
|
||||
}
|
||||
|
||||
// ExtractAll extracts tags, properties, and references in a single parse for efficiency.
|
||||
func (s *service) ExtractAll(content []byte) (*ExtractedData, error) {
|
||||
root, err := s.parse(content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data := &ExtractedData{
|
||||
Tags: []string{},
|
||||
Mentions: []string{},
|
||||
Property: &storepb.MemoPayload_Property{},
|
||||
}
|
||||
|
||||
firstBlockChecked := false
|
||||
|
||||
// Single walk to collect all data
|
||||
err = gast.Walk(root, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return gast.WalkContinue, nil
|
||||
}
|
||||
|
||||
if tagNode, ok := asMemoTagNode(n); ok {
|
||||
data.Tags = append(data.Tags, string(tagNode.Tag))
|
||||
}
|
||||
if mentionNode, ok := n.(*mast.MentionNode); ok {
|
||||
data.Mentions = append(data.Mentions, strings.ToLower(string(mentionNode.Username)))
|
||||
}
|
||||
|
||||
// Check if the first block-level child of the document is an H1 heading.
|
||||
if !firstBlockChecked && n.Parent() != nil && n.Parent().Kind() == gast.KindDocument {
|
||||
firstBlockChecked = true
|
||||
if heading, ok := n.(*gast.Heading); ok && heading.Level == 1 {
|
||||
data.Property.Title = extractHeadingText(n, content)
|
||||
}
|
||||
}
|
||||
|
||||
// Extract properties based on node kind
|
||||
switch n.Kind() {
|
||||
case gast.KindLink:
|
||||
data.Property.HasLink = true
|
||||
|
||||
case gast.KindCodeBlock, gast.KindFencedCodeBlock, gast.KindCodeSpan:
|
||||
data.Property.HasCode = true
|
||||
|
||||
case east.KindTaskCheckBox:
|
||||
data.Property.HasTaskList = true
|
||||
if checkBox, ok := n.(*east.TaskCheckBox); ok {
|
||||
if !checkBox.IsChecked {
|
||||
data.Property.HasIncompleteTasks = true
|
||||
}
|
||||
}
|
||||
default:
|
||||
// No special handling for other node types
|
||||
}
|
||||
|
||||
return gast.WalkContinue, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Deduplicate tags while preserving original case
|
||||
data.Tags = uniquePreserveCase(data.Tags)
|
||||
data.Mentions = uniquePreserveCase(data.Mentions)
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// RenameTag renames all occurrences of oldTag to newTag in content.
|
||||
func (s *service) RenameTag(content []byte, oldTag, newTag string) (string, error) {
|
||||
root, err := s.parse(content)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Walk the AST to find and rename tag nodes
|
||||
err = gast.Walk(root, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return gast.WalkContinue, nil
|
||||
}
|
||||
|
||||
if tagNode, ok := asMemoTagNode(n); ok {
|
||||
if string(tagNode.Tag) == oldTag {
|
||||
tagNode.Tag = []byte(newTag)
|
||||
}
|
||||
}
|
||||
|
||||
return gast.WalkContinue, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Render back to markdown using the already-parsed AST
|
||||
mdRenderer := renderer.NewMarkdownRenderer()
|
||||
return mdRenderer.Render(root, content), nil
|
||||
}
|
||||
|
||||
// uniquePreserveCase returns unique strings from input while preserving case.
|
||||
func uniquePreserveCase(strs []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var result []string
|
||||
|
||||
for _, s := range strs {
|
||||
if _, exists := seen[s]; !exists {
|
||||
seen[s] = struct{}{}
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// truncateAtWord truncates a string at the last word boundary before maxLength.
|
||||
// maxLength is treated as a rune (character) count to properly handle UTF-8 multi-byte characters.
|
||||
func truncateAtWord(s string, maxLength int) string {
|
||||
// Convert to runes to properly handle multi-byte UTF-8 characters
|
||||
runes := []rune(s)
|
||||
if len(runes) <= maxLength {
|
||||
return s
|
||||
}
|
||||
|
||||
// Truncate to max length (by character count, not byte count)
|
||||
truncated := string(runes[:maxLength])
|
||||
|
||||
// Find last space to avoid cutting in the middle of a word
|
||||
lastSpace := strings.LastIndexAny(truncated, " \t\n\r")
|
||||
if lastSpace > 0 {
|
||||
truncated = truncated[:lastSpace]
|
||||
}
|
||||
|
||||
return truncated + " ..."
|
||||
}
|
||||
@@ -1,664 +0,0 @@
|
||||
package markdown
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewService(t *testing.T) {
|
||||
svc := NewService()
|
||||
assert.NotNil(t, svc)
|
||||
}
|
||||
|
||||
func TestValidateContent(t *testing.T) {
|
||||
svc := NewService()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid markdown",
|
||||
content: "# Hello\n\nThis is **bold** text.",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty content",
|
||||
content: "",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "complex markdown",
|
||||
content: "# Title\n\n- List item 1\n- List item 2\n\n```go\ncode block\n```",
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := svc.ValidateContent([]byte(tt.content))
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateSnippet(t *testing.T) {
|
||||
svc := NewService()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
maxLength int
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "simple text",
|
||||
content: "Hello world",
|
||||
maxLength: 100,
|
||||
expected: "Hello world",
|
||||
},
|
||||
{
|
||||
name: "text with formatting",
|
||||
content: "This is **bold** and *italic* text.",
|
||||
maxLength: 100,
|
||||
expected: "This is bold and italic text.",
|
||||
},
|
||||
{
|
||||
name: "truncate long text",
|
||||
content: "This is a very long piece of text that should be truncated at a word boundary.",
|
||||
maxLength: 30,
|
||||
expected: "This is a very long piece of ...",
|
||||
},
|
||||
{
|
||||
name: "heading and paragraph",
|
||||
content: "# My Title\n\nThis is the first paragraph.",
|
||||
maxLength: 100,
|
||||
expected: "My Title This is the first paragraph.",
|
||||
},
|
||||
{
|
||||
name: "code block removed",
|
||||
content: "Text before\n\n```go\ncode\n```\n\nText after",
|
||||
maxLength: 100,
|
||||
expected: "Text before Text after",
|
||||
},
|
||||
{
|
||||
name: "list items",
|
||||
content: "- Item 1\n- Item 2\n- Item 3",
|
||||
maxLength: 100,
|
||||
expected: "Item 1 Item 2 Item 3",
|
||||
},
|
||||
{
|
||||
name: "inline code preserved",
|
||||
content: "`console.log('hello')`",
|
||||
maxLength: 100,
|
||||
expected: "console.log('hello')",
|
||||
},
|
||||
{
|
||||
name: "text with inline code",
|
||||
content: "Use `fmt.Println` to print output.",
|
||||
maxLength: 100,
|
||||
expected: "Use fmt.Println to print output.",
|
||||
},
|
||||
{
|
||||
name: "image alt text",
|
||||
content: "",
|
||||
maxLength: 100,
|
||||
expected: "alt text",
|
||||
},
|
||||
{
|
||||
name: "strikethrough text",
|
||||
content: "~~deleted text~~",
|
||||
maxLength: 100,
|
||||
expected: "deleted text",
|
||||
},
|
||||
{
|
||||
name: "blockquote",
|
||||
content: "> quoted text",
|
||||
maxLength: 100,
|
||||
expected: "quoted text",
|
||||
},
|
||||
{
|
||||
name: "table cells spaced",
|
||||
content: "| a | b |\n|---|---|\n| 1 | 2 |",
|
||||
maxLength: 100,
|
||||
expected: "a b 1 2",
|
||||
},
|
||||
{
|
||||
name: "plain URL autolink",
|
||||
content: "https://usememos.com",
|
||||
maxLength: 100,
|
||||
expected: "https://usememos.com",
|
||||
},
|
||||
{
|
||||
name: "text with plain URL",
|
||||
content: "Check out https://usememos.com for more info.",
|
||||
maxLength: 100,
|
||||
expected: "Check out https://usememos.com for more info.",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
snippet, err := svc.GenerateSnippet([]byte(tt.content), tt.maxLength)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, snippet)
|
||||
})
|
||||
}
|
||||
|
||||
// Test with tag extension enabled (matches production config).
|
||||
svcWithTags := NewService(WithTagExtension())
|
||||
tagTests := []struct {
|
||||
name string
|
||||
content string
|
||||
maxLength int
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "tag only",
|
||||
content: "#todo",
|
||||
maxLength: 100,
|
||||
expected: "#todo",
|
||||
},
|
||||
{
|
||||
name: "text with tags",
|
||||
content: "Remember to #review the #code",
|
||||
maxLength: 100,
|
||||
expected: "Remember to #review the #code",
|
||||
},
|
||||
}
|
||||
for _, tt := range tagTests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
snippet, err := svcWithTags.GenerateSnippet([]byte(tt.content), tt.maxLength)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, snippet)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractProperties(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
hasLink bool
|
||||
hasCode bool
|
||||
hasTasks bool
|
||||
hasInc bool
|
||||
title string
|
||||
}{
|
||||
{
|
||||
name: "plain text",
|
||||
content: "Just plain text",
|
||||
hasLink: false,
|
||||
hasCode: false,
|
||||
hasTasks: false,
|
||||
hasInc: false,
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "with link",
|
||||
content: "Check out [this link](https://example.com)",
|
||||
hasLink: true,
|
||||
hasCode: false,
|
||||
hasTasks: false,
|
||||
hasInc: false,
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "with inline code",
|
||||
content: "Use `console.log()` to debug",
|
||||
hasLink: false,
|
||||
hasCode: true,
|
||||
hasTasks: false,
|
||||
hasInc: false,
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "with code block",
|
||||
content: "```go\nfunc main() {}\n```",
|
||||
hasLink: false,
|
||||
hasCode: true,
|
||||
hasTasks: false,
|
||||
hasInc: false,
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "with completed task",
|
||||
content: "- [x] Completed task",
|
||||
hasLink: false,
|
||||
hasCode: false,
|
||||
hasTasks: true,
|
||||
hasInc: false,
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "with incomplete task",
|
||||
content: "- [ ] Todo item",
|
||||
hasLink: false,
|
||||
hasCode: false,
|
||||
hasTasks: true,
|
||||
hasInc: true,
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "mixed tasks",
|
||||
content: "- [x] Done\n- [ ] Not done",
|
||||
hasLink: false,
|
||||
hasCode: false,
|
||||
hasTasks: true,
|
||||
hasInc: true,
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "everything",
|
||||
content: "# Title\n\n[Link](url)\n\n`code`\n\n- [ ] Task",
|
||||
hasLink: true,
|
||||
hasCode: true,
|
||||
hasTasks: true,
|
||||
hasInc: true,
|
||||
title: "Title",
|
||||
},
|
||||
{
|
||||
name: "h1 as first node extracts title",
|
||||
content: "# My Article Title\n\nBody text here.",
|
||||
title: "My Article Title",
|
||||
},
|
||||
{
|
||||
name: "h2 as first node does not extract title",
|
||||
content: "## Sub Heading\n\nBody text.",
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "h1 not first node does not extract title",
|
||||
content: "Some text\n\n# Heading Later",
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "h1 with inline formatting extracts plain text",
|
||||
content: "# Title with **bold** and *italic*\n\nBody.",
|
||||
title: "Title with bold and italic",
|
||||
},
|
||||
{
|
||||
name: "empty content has no title",
|
||||
content: "",
|
||||
title: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
svc := NewService()
|
||||
|
||||
props, err := svc.ExtractProperties([]byte(tt.content))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.hasLink, props.HasLink, "HasLink")
|
||||
assert.Equal(t, tt.hasCode, props.HasCode, "HasCode")
|
||||
assert.Equal(t, tt.hasTasks, props.HasTaskList, "HasTaskList")
|
||||
assert.Equal(t, tt.hasInc, props.HasIncompleteTasks, "HasIncompleteTasks")
|
||||
assert.Equal(t, tt.title, props.Title, "Title")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAllTitle(t *testing.T) {
|
||||
svc := NewService(WithTagExtension())
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
title string
|
||||
}{
|
||||
{
|
||||
name: "h1 first node",
|
||||
content: "# Article Title\n\nContent with #tag",
|
||||
title: "Article Title",
|
||||
},
|
||||
{
|
||||
name: "no h1",
|
||||
content: "Just text with #tag",
|
||||
title: "",
|
||||
},
|
||||
{
|
||||
name: "h1 not first",
|
||||
content: "Intro\n\n# Late Heading",
|
||||
title: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data, err := svc.ExtractAll([]byte(tt.content))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.title, data.Property.Title, "Title")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAllMentions(t *testing.T) {
|
||||
svc := NewService(WithTagExtension(), WithMentionExtension())
|
||||
|
||||
data, err := svc.ExtractAll([]byte("Hi @Alice and @bob. Email support@example.com should stay plain. #tag"))
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{"alice", "bob"}, data.Mentions)
|
||||
assert.ElementsMatch(t, []string{"tag"}, data.Tags)
|
||||
|
||||
maxLengthUsername := "a" + strings.Repeat("b", 62)
|
||||
data, err = svc.ExtractAll([]byte("@" + maxLengthUsername))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{maxLengthUsername}, data.Mentions)
|
||||
|
||||
data, err = svc.ExtractAll([]byte("@" + maxLengthUsername + "c"))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, data.Mentions)
|
||||
}
|
||||
|
||||
func TestExtractAllSkipsTagsInsideLinks(t *testing.T) {
|
||||
svc := NewService(WithTagExtension())
|
||||
|
||||
data, err := svc.ExtractAll([]byte(
|
||||
"[release #notes](https://example.com/releases#release-notes)\n\n" +
|
||||
"\n\n" +
|
||||
"Outside #memo-tag",
|
||||
))
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.ElementsMatch(t, []string{"memo-tag"}, data.Tags)
|
||||
assert.True(t, data.Property.HasLink)
|
||||
}
|
||||
|
||||
func TestExtractTags(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
withExt bool
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "no tags",
|
||||
content: "Just plain text",
|
||||
withExt: false,
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "single tag",
|
||||
content: "Text with #tag",
|
||||
withExt: true,
|
||||
expected: []string{"tag"},
|
||||
},
|
||||
{
|
||||
name: "multiple tags",
|
||||
content: "Text with #tag1 and #tag2",
|
||||
withExt: true,
|
||||
expected: []string{"tag1", "tag2"},
|
||||
},
|
||||
{
|
||||
name: "duplicate tags",
|
||||
content: "#work is important. #Work #WORK",
|
||||
withExt: true,
|
||||
expected: []string{"work", "Work", "WORK"},
|
||||
},
|
||||
{
|
||||
name: "tags with hyphens and underscores",
|
||||
content: "Tags: #work-notes #2024_plans",
|
||||
withExt: true,
|
||||
expected: []string{"work-notes", "2024_plans"},
|
||||
},
|
||||
{
|
||||
name: "tags at end of sentence",
|
||||
content: "This is important #urgent.",
|
||||
withExt: true,
|
||||
expected: []string{"urgent"},
|
||||
},
|
||||
{
|
||||
name: "headings not tags",
|
||||
content: "## Heading\n\n# Title\n\nText with #realtag",
|
||||
withExt: true,
|
||||
expected: []string{"realtag"},
|
||||
},
|
||||
{
|
||||
name: "numeric tag",
|
||||
content: "Issue #123",
|
||||
withExt: true,
|
||||
expected: []string{"123"},
|
||||
},
|
||||
{
|
||||
name: "tag in list",
|
||||
content: "- Item 1 #todo\n- Item 2 #done",
|
||||
withExt: true,
|
||||
expected: []string{"todo", "done"},
|
||||
},
|
||||
{
|
||||
name: "autolink URL fragment not tag",
|
||||
content: "https://github.com/dmtrKovalenko/fff#pi-agent-extension\n\nProject #memo-tag",
|
||||
withExt: true,
|
||||
expected: []string{"memo-tag"},
|
||||
},
|
||||
{
|
||||
name: "markdown link text and fragment not tags",
|
||||
content: "[release #notes](https://example.com/releases#release-notes) Outside #memo-tag",
|
||||
withExt: true,
|
||||
expected: []string{"memo-tag"},
|
||||
},
|
||||
{
|
||||
name: "reference link text and fragment not tags",
|
||||
content: "[reference #anchor][docs]\n\n[docs]: https://example.com/docs#reference-anchor\n\nOutside #memo-tag",
|
||||
withExt: true,
|
||||
expected: []string{"memo-tag"},
|
||||
},
|
||||
{
|
||||
name: "image alt text and fragment not tags",
|
||||
content: "\n\nOutside #memo-tag",
|
||||
withExt: true,
|
||||
expected: []string{"memo-tag"},
|
||||
},
|
||||
{
|
||||
name: "no extension enabled",
|
||||
content: "Text with #tag",
|
||||
withExt: false,
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "Chinese tag",
|
||||
content: "Text with #测试",
|
||||
withExt: true,
|
||||
expected: []string{"测试"},
|
||||
},
|
||||
{
|
||||
name: "Chinese tag followed by punctuation",
|
||||
content: "Text #测试。 More text",
|
||||
withExt: true,
|
||||
expected: []string{"测试"},
|
||||
},
|
||||
{
|
||||
name: "mixed Chinese and ASCII tag",
|
||||
content: "#测试test123 content",
|
||||
withExt: true,
|
||||
expected: []string{"测试test123"},
|
||||
},
|
||||
{
|
||||
name: "Japanese tag",
|
||||
content: "#日本語 content",
|
||||
withExt: true,
|
||||
expected: []string{"日本語"},
|
||||
},
|
||||
{
|
||||
name: "Korean tag",
|
||||
content: "#한국어 content",
|
||||
withExt: true,
|
||||
expected: []string{"한국어"},
|
||||
},
|
||||
{
|
||||
name: "hierarchical tag with Chinese",
|
||||
content: "#work/测试/项目",
|
||||
withExt: true,
|
||||
expected: []string{"work/测试/项目"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var svc Service
|
||||
if tt.withExt {
|
||||
svc = NewService(WithTagExtension())
|
||||
} else {
|
||||
svc = NewService()
|
||||
}
|
||||
|
||||
tags, err := svc.ExtractTags([]byte(tt.content))
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, tt.expected, tags)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameTagSkipsTagsInsideLinks(t *testing.T) {
|
||||
svc := NewService(WithTagExtension())
|
||||
|
||||
result, err := svc.RenameTag(
|
||||
[]byte("[release #notes](https://example.com/releases#release-notes)\n\nOutside #notes"),
|
||||
"notes",
|
||||
"done",
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "[release #notes](https://example.com/releases#release-notes)\n\nOutside #done", result)
|
||||
}
|
||||
|
||||
func TestUniquePreserveCase(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
input: []string{},
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "unique items",
|
||||
input: []string{"tag1", "tag2", "tag3"},
|
||||
expected: []string{"tag1", "tag2", "tag3"},
|
||||
},
|
||||
{
|
||||
name: "duplicates",
|
||||
input: []string{"tag", "TAG", "Tag"},
|
||||
expected: []string{"tag", "TAG", "Tag"},
|
||||
},
|
||||
{
|
||||
name: "mixed",
|
||||
input: []string{"Work", "work", "Important", "work"},
|
||||
expected: []string{"Work", "work", "Important"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := uniquePreserveCase(tt.input)
|
||||
assert.ElementsMatch(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateAtWord(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
maxLength int
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "no truncation needed",
|
||||
input: "short",
|
||||
maxLength: 10,
|
||||
expected: "short",
|
||||
},
|
||||
{
|
||||
name: "exact length",
|
||||
input: "exactly ten",
|
||||
maxLength: 11,
|
||||
expected: "exactly ten",
|
||||
},
|
||||
{
|
||||
name: "truncate at word",
|
||||
input: "this is a long sentence",
|
||||
maxLength: 10,
|
||||
expected: "this is a ...",
|
||||
},
|
||||
{
|
||||
name: "truncate very long word",
|
||||
input: "supercalifragilisticexpialidocious",
|
||||
maxLength: 10,
|
||||
expected: "supercalif ...",
|
||||
},
|
||||
{
|
||||
name: "CJK characters without spaces",
|
||||
input: "这是一个很长的中文句子没有空格的情况下也要正确处理",
|
||||
maxLength: 15,
|
||||
expected: "这是一个很长的中文句子没有空格 ...",
|
||||
},
|
||||
{
|
||||
name: "mixed CJK and Latin",
|
||||
input: "这是中文mixed with English文字",
|
||||
maxLength: 10,
|
||||
expected: "这是中文mixed ...",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncateAtWord(tt.input, tt.maxLength)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark tests.
|
||||
func BenchmarkGenerateSnippet(b *testing.B) {
|
||||
svc := NewService()
|
||||
content := []byte(`# Large Document
|
||||
|
||||
This is a large document with multiple paragraphs and formatting.
|
||||
|
||||
## Section 1
|
||||
|
||||
Here is some **bold** text and *italic* text with [links](https://example.com).
|
||||
|
||||
- List item 1
|
||||
- List item 2
|
||||
- List item 3
|
||||
|
||||
## Section 2
|
||||
|
||||
More content here with ` + "`inline code`" + ` and other elements.
|
||||
|
||||
` + "```go\nfunc example() {\n return true\n}\n```")
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := svc.GenerateSnippet(content, 200)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkExtractProperties(b *testing.B) {
|
||||
svc := NewService()
|
||||
content := []byte("# Title\n\n[Link](url)\n\n`code`\n\n- [ ] Task\n- [x] Done")
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := svc.ExtractProperties(content)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user