feat(auth): bootstrap identity providers from secret files

- scan /etc/secrets for memos-idp-*.json after migrations and demo seeding - reconcile protobuf JSON identity providers by stable UID - validate all discovered providers before applying updates
2026-07-13 00:08:15 +08:00
parent 6c17e87cf6
commit ad6d009767
4 changed files with 391 additions and 0 deletions
@@ -0,0 +1,42 @@
+# Identity Provider Bootstrap
+
+Memos automatically reconciles OAuth2 identity providers from JSON files before the server starts. This is useful when a deployment platform provides configuration as mounted secret files.
+
+By default, Memos scans `/etc/secrets` for files named `memos-idp-*.json`:
+
+```bash
+memos
+```
+
+For example, `/etc/secrets/memos-idp-primary.json` can contain:
+
+```json
+{
+  "uid": "primary-sso",
+  "name": "Company SSO",
+  "type": "OAUTH2",
+  "identifierFilter": "",
+  "config": {
+    "oauth2Config": {
+      "clientId": "client-id",
+      "clientSecret": "client-secret",
+      "authUrl": "https://idp.example.com/oauth/authorize",
+      "tokenUrl": "https://idp.example.com/oauth/token",
+      "userInfoUrl": "https://idp.example.com/oauth/userinfo",
+      "scopes": ["profile", "email"],
+      "fieldMapping": {
+        "identifier": "sub",
+        "displayName": "name",
+        "email": "email",
+        "avatarUrl": "picture"
+      }
+    }
+  }
+}
+```
+
+Each file contains exactly one `memos.store.IdentityProvider` encoded as protobuf JSON. The database-generated numeric `id` must be omitted; `uid` is the provider's stable identifier.
+
+After database migrations and demo seeding, Memos reads all matching files in filename order and validates every provider before writing anything. Each provider is then created or updated by its stable `uid`; providers omitted from the files are left unchanged. Reapplying the files is safe and updates credentials on restart, which supports secret rotation. Duplicate provider UIDs across files are rejected.
+
+A missing directory or a directory without matching files is a normal no-op. If a matching file is unreadable, oversized, or invalid, Memos does not start. Files with other names are ignored so the directory can safely contain unrelated secrets. Keep files containing `clientSecret` outside source control and provide them through the deployment platform's secret-management facility.
@@ -0,0 +1,165 @@
+package store
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"google.golang.org/protobuf/encoding/protojson"
+
+	"github.com/usememos/memos/internal/base"
+	storepb "github.com/usememos/memos/proto/gen/store"
+)
+
+const (
+	maxIdentityProviderBootstrapSize    = 1 << 20
+	defaultIdentityProviderBootstrapDir = "/etc/secrets"
+	identityProviderBootstrapPrefix     = "memos-idp-"
+	identityProviderBootstrapSuffix     = ".json"
+)
+
+// ApplyIdentityProviderBootstrapDir validates and reconciles identity providers
+// from memos-idp-*.json files in a directory. Providers not named in those files
+// are left unchanged.
+func (s *Store) ApplyIdentityProviderBootstrapDir(ctx context.Context, dir string) error {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return errors.Wrap(err, "failed to read identity provider bootstrap directory")
+	}
+
+	providers := []*storepb.IdentityProvider{}
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasPrefix(entry.Name(), identityProviderBootstrapPrefix) || !strings.HasSuffix(entry.Name(), identityProviderBootstrapSuffix) {
+			continue
+		}
+		provider, err := readIdentityProviderBootstrap(filepath.Join(dir, entry.Name()))
+		if err != nil {
+			return errors.Wrapf(err, "invalid identity provider bootstrap file %q", entry.Name())
+		}
+		providers = append(providers, provider)
+	}
+	if err := validateIdentityProviderBootstrap(providers); err != nil {
+		return err
+	}
+
+	for _, provider := range providers {
+		existing, err := s.GetIdentityProvider(ctx, &FindIdentityProvider{UID: &provider.Uid})
+		if err != nil {
+			return errors.Wrapf(err, "failed to look up identity provider %q", provider.Uid)
+		}
+		if existing == nil {
+			if _, err := s.CreateIdentityProvider(ctx, provider); err != nil {
+				return errors.Wrapf(err, "failed to create identity provider %q", provider.Uid)
+			}
+			continue
+		}
+		if existing.Type != provider.Type {
+			return errors.Errorf("identity provider %q has type %s, expected %s", provider.Uid, existing.Type, provider.Type)
+		}
+
+		title := provider.Name
+		identifierFilter := provider.IdentifierFilter
+		if _, err := s.UpdateIdentityProvider(ctx, &UpdateIdentityProviderV1{
+			ID:               existing.Id,
+			Type:             existing.Type,
+			Name:             &title,
+			IdentifierFilter: &identifierFilter,
+			Config:           provider.Config,
+		}); err != nil {
+			return errors.Wrapf(err, "failed to update identity provider %q", provider.Uid)
+		}
+	}
+
+	return nil
+}
+
+func readIdentityProviderBootstrap(path string) (*storepb.IdentityProvider, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read identity provider bootstrap file")
+	}
+	defer file.Close()
+
+	content, err := io.ReadAll(io.LimitReader(file, maxIdentityProviderBootstrapSize+1))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read identity provider bootstrap file")
+	}
+	if len(content) > maxIdentityProviderBootstrapSize {
+		return nil, errors.Errorf("identity provider bootstrap file exceeds %d bytes", maxIdentityProviderBootstrapSize)
+	}
+
+	provider := &storepb.IdentityProvider{}
+	if err := protojson.Unmarshal(content, provider); err != nil {
+		return nil, errors.Wrap(err, "failed to decode identity provider bootstrap file")
+	}
+	return provider, nil
+}
+
+func validateIdentityProviderBootstrap(providers []*storepb.IdentityProvider) error {
+	seenUIDs := make(map[string]struct{}, len(providers))
+	for i, provider := range providers {
+		if provider.Id != 0 {
+			return errors.Errorf("identityProviders[%d].id must be omitted", i)
+		}
+		if !base.UIDMatcher.MatchString(provider.Uid) {
+			return errors.Errorf("identityProviders[%d].uid is invalid", i)
+		}
+		if _, exists := seenUIDs[provider.Uid]; exists {
+			return errors.Errorf("identityProviders[%d].uid duplicates %q", i, provider.Uid)
+		}
+		seenUIDs[provider.Uid] = struct{}{}
+		if strings.TrimSpace(provider.Name) == "" {
+			return errors.Errorf("identityProviders[%d].name is required", i)
+		}
+		if provider.Type != storepb.IdentityProvider_OAUTH2 {
+			return errors.Errorf("identityProviders[%d].type must be OAUTH2", i)
+		}
+		if err := validateBootstrapOAuth2Config(provider.Config.GetOauth2Config(), i); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func validateBootstrapOAuth2Config(config *storepb.OAuth2Config, index int) error {
+	if config == nil {
+		return errors.Errorf("identityProviders[%d].config.oauth2Config is required", index)
+	}
+	required := []struct {
+		name  string
+		value string
+	}{
+		{name: "clientId", value: config.ClientId},
+		{name: "clientSecret", value: config.ClientSecret},
+		{name: "authUrl", value: config.AuthUrl},
+		{name: "tokenUrl", value: config.TokenUrl},
+		{name: "userInfoUrl", value: config.UserInfoUrl},
+	}
+	if config.FieldMapping == nil {
+		return errors.Errorf("identityProviders[%d].config.oauth2Config.fieldMapping is required", index)
+	}
+	required = append(required, struct {
+		name  string
+		value string
+	}{name: "fieldMapping.identifier", value: config.FieldMapping.Identifier})
+	for _, field := range required {
+		if strings.TrimSpace(field.value) == "" {
+			return errors.Errorf("identityProviders[%d].config.oauth2Config.%s is required", index, field.name)
+		}
+	}
+	if len(config.Scopes) == 0 {
+		return errors.Errorf("identityProviders[%d].config.oauth2Config.scopes is required", index)
+	}
+	for scopeIndex, scope := range config.Scopes {
+		if strings.TrimSpace(scope) == "" {
+			return errors.Errorf("identityProviders[%d].config.oauth2Config.scopes[%d] must not be empty", index, scopeIndex)
+		}
+	}
+	return nil
+}
@@ -0,0 +1,180 @@
+package store_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/internal/profile"
+	"github.com/usememos/memos/store"
+	"github.com/usememos/memos/store/db/sqlite"
+)
+
+func TestApplyIdentityProviderBootstrapCreatesAndUpdatesProvider(t *testing.T) {
+	ctx := context.Background()
+	stores := newIdentityProviderBootstrapTestStore(t)
+	bootstrapDir := t.TempDir()
+	bootstrapPath := filepath.Join(bootstrapDir, "memos-idp-primary.json")
+
+	writeIdentityProviderBootstrap(t, bootstrapPath, "Initial SSO", "initial-secret")
+	require.NoError(t, stores.ApplyIdentityProviderBootstrapDir(ctx, bootstrapDir))
+
+	provider, err := stores.GetIdentityProvider(ctx, &store.FindIdentityProvider{UID: ptr("primary-sso")})
+	require.NoError(t, err)
+	require.NotNil(t, provider)
+	assert.Equal(t, "Initial SSO", provider.Name)
+	assert.Equal(t, "initial-secret", provider.Config.GetOauth2Config().ClientSecret)
+	providerID := provider.Id
+
+	writeIdentityProviderBootstrap(t, bootstrapPath, "Updated SSO", "rotated-secret")
+	require.NoError(t, stores.ApplyIdentityProviderBootstrapDir(ctx, bootstrapDir))
+
+	provider, err = stores.GetIdentityProvider(ctx, &store.FindIdentityProvider{UID: ptr("primary-sso")})
+	require.NoError(t, err)
+	require.NotNil(t, provider)
+	assert.Equal(t, providerID, provider.Id)
+	assert.Equal(t, "Updated SSO", provider.Name)
+	assert.Equal(t, "rotated-secret", provider.Config.GetOauth2Config().ClientSecret)
+}
+
+func TestApplyIdentityProviderBootstrapValidatesEntireFileBeforeWriting(t *testing.T) {
+	ctx := context.Background()
+	stores := newIdentityProviderBootstrapTestStore(t)
+	bootstrapDir := t.TempDir()
+	writeIdentityProviderBootstrap(t, filepath.Join(bootstrapDir, "memos-idp-a-valid.json"), "Valid Provider", "must-not-appear-in-errors")
+	invalidContent := `{
+  "uid": "invalid/provider",
+  "name": "Invalid Provider",
+  "type": "OAUTH2",
+  "config": {
+    "oauth2Config": {
+      "clientId": "client-id",
+      "clientSecret": "another-secret",
+      "authUrl": "https://example.com/authorize",
+      "tokenUrl": "https://example.com/token",
+      "userInfoUrl": "https://example.com/userinfo",
+      "scopes": ["profile"],
+      "fieldMapping": {"identifier": "sub"}
+    }
+  }
+}`
+	require.NoError(t, os.WriteFile(filepath.Join(bootstrapDir, "memos-idp-z-invalid.json"), []byte(invalidContent), 0600))
+
+	err := stores.ApplyIdentityProviderBootstrapDir(ctx, bootstrapDir)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "identityProviders[1].uid is invalid")
+	assert.NotContains(t, err.Error(), "must-not-appear-in-errors")
+	assert.NotContains(t, err.Error(), "another-secret")
+
+	providers, err := stores.ListIdentityProviders(ctx, &store.FindIdentityProvider{})
+	require.NoError(t, err)
+	assert.Empty(t, providers)
+}
+
+func TestApplyIdentityProviderBootstrapRejectsUnknownFields(t *testing.T) {
+	stores := newIdentityProviderBootstrapTestStore(t)
+	bootstrapDir := t.TempDir()
+	bootstrapPath := filepath.Join(bootstrapDir, "memos-idp-invalid.json")
+	require.NoError(t, os.WriteFile(bootstrapPath, []byte(`{"uid":"primary-sso","unexpected":true}`), 0600))
+
+	err := stores.ApplyIdentityProviderBootstrapDir(context.Background(), bootstrapDir)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, `unknown field "unexpected"`)
+}
+
+func TestApplyIdentityProviderBootstrapDirRejectsDuplicateUIDsAcrossFiles(t *testing.T) {
+	stores := newIdentityProviderBootstrapTestStore(t)
+	bootstrapDir := t.TempDir()
+	writeIdentityProviderBootstrap(t, filepath.Join(bootstrapDir, "memos-idp-first.json"), "First Provider", "first-secret")
+	writeIdentityProviderBootstrap(t, filepath.Join(bootstrapDir, "memos-idp-second.json"), "Second Provider", "second-secret")
+
+	err := stores.ApplyIdentityProviderBootstrapDir(context.Background(), bootstrapDir)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, `uid duplicates "primary-sso"`)
+
+	providers, listErr := stores.ListIdentityProviders(context.Background(), &store.FindIdentityProvider{})
+	require.NoError(t, listErr)
+	assert.Empty(t, providers)
+}
+
+func TestApplyIdentityProviderBootstrapDirIgnoresUnrelatedSecretFiles(t *testing.T) {
+	stores := newIdentityProviderBootstrapTestStore(t)
+	bootstrapDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(bootstrapDir, ".env"), []byte("DATABASE_PASSWORD=secret"), 0600))
+	writeIdentityProviderBootstrap(t, filepath.Join(bootstrapDir, "memos-idp-primary.json"), "Primary SSO", "oauth-secret")
+
+	require.NoError(t, stores.ApplyIdentityProviderBootstrapDir(context.Background(), bootstrapDir))
+	provider, err := stores.GetIdentityProvider(context.Background(), &store.FindIdentityProvider{UID: ptr("primary-sso")})
+	require.NoError(t, err)
+	require.NotNil(t, provider)
+}
+
+func TestApplyIdentityProviderBootstrapDirIgnoresDirectoryWithoutMatchingFiles(t *testing.T) {
+	stores := newIdentityProviderBootstrapTestStore(t)
+	bootstrapDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(bootstrapDir, "database-password"), []byte("secret"), 0600))
+
+	require.NoError(t, stores.ApplyIdentityProviderBootstrapDir(context.Background(), bootstrapDir))
+}
+
+func TestApplyIdentityProviderBootstrapDirIgnoresMissingDirectory(t *testing.T) {
+	stores := newIdentityProviderBootstrapTestStore(t)
+	bootstrapDir := filepath.Join(t.TempDir(), "missing")
+
+	require.NoError(t, stores.ApplyIdentityProviderBootstrapDir(context.Background(), bootstrapDir))
+}
+
+func newIdentityProviderBootstrapTestStore(t *testing.T) *store.Store {
+	t.Helper()
+	p := &profile.Profile{
+		Data:   t.TempDir(),
+		Driver: "sqlite",
+		DSN:    filepath.Join(t.TempDir(), "bootstrap.db"),
+	}
+	driver, err := sqlite.NewDB(p)
+	require.NoError(t, err)
+	stores := store.New(driver, p)
+	require.NoError(t, stores.Migrate(context.Background()))
+	t.Cleanup(func() {
+		require.NoError(t, stores.Close())
+	})
+	return stores
+}
+
+func writeIdentityProviderBootstrap(t *testing.T, path, title, secret string) {
+	t.Helper()
+	content := `{
+  "uid": "primary-sso",
+  "name": "` + title + `",
+  "type": "OAUTH2",
+  "identifierFilter": "",
+  "config": {
+    "oauth2Config": {
+      "clientId": "client-id",
+      "clientSecret": "` + secret + `",
+      "authUrl": "https://example.com/authorize",
+      "tokenUrl": "https://example.com/token",
+      "userInfoUrl": "https://example.com/userinfo",
+      "scopes": ["profile", "email"],
+      "fieldMapping": {
+        "identifier": "sub",
+        "displayName": "name",
+        "email": "email",
+        "avatarUrl": "picture"
+      }
+    }
+  }
+}`
+	require.False(t, strings.Contains(title, `"`))
+	require.False(t, strings.Contains(secret, `"`))
+	require.NoError(t, os.WriteFile(path, []byte(content), 0600))
+}
+
+func ptr[T any](value T) *T {
+	return &value
+}
@@ -28,6 +28,7 @@ import (
 // 2. checkMinimumUpgradeVersion: Verify installation can be upgraded (reject pre-0.22 installations)
 // 3. Migrate (prod mode): Apply incremental migrations from current to target version
 // 4. Migrate (demo mode): Seed database with demo data
+// 5. Reconcile identity providers from /etc/secrets/memos-idp-*.json
 //
 // Version Tracking:
 // - New installations: Schema version set in system_setting immediately
@@ -131,6 +132,9 @@ func (s *Store) Migrate(ctx context.Context) error {
 			return errors.Wrap(err, "failed to seed")
 		}
 	}
+	if err := s.ApplyIdentityProviderBootstrapDir(ctx, defaultIdentityProviderBootstrapDir); err != nil {
+		return errors.Wrap(err, "failed to apply identity provider bootstrap")
+	}

 	return nil
 }