diff --git a/.github/workflows/backend-linter.yml b/.github/workflows/backend-linter.yml index 4346b2c9..a3284962 100644 --- a/.github/workflows/backend-linter.yml +++ b/.github/workflows/backend-linter.yml @@ -35,7 +35,7 @@ jobs: - name: Run Golangci-lint uses: golangci/golangci-lint-action@v9.2.1 with: - version: v2.11.4 - args: --build-tags=exclude_frontend + version: v2.12.2 + args: --config=.golangci.yml working-directory: backend only-new-issues: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 4ecbbf8c..224b3b8a 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -31,7 +31,7 @@ jobs: working-directory: backend run: | set -e -o pipefail - go test -tags=exclude_frontend -v ./... | tee /tmp/TestResults.log + go test -tags=exclude_frontend,unit -v ./... | tee /tmp/TestResults.log - uses: actions/upload-artifact@v7 if: always() with: diff --git a/.vscode/settings.json b/.vscode/settings.json index 5e688181..ce7b561f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,4 @@ { - "go.buildTags": "e2etest", + "go.buildTags": "e2etest unit", "prettier.documentSelectors": ["**/*.svelte"] } diff --git a/AGENTS.md b/AGENTS.md index d994ef2f..2e46f9a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,10 +12,10 @@ the binary for production). This file lists what isn't obvious from reading the ## Build / test / lint ```sh -# backend/ — the exclude_frontend tag is mandatory locally; CI uses it too -go test -tags=exclude_frontend ./... # unit/integration tests -go test -tags=exclude_frontend -run TestName ./internal/... # a single test -golangci-lint run --build-tags=exclude_frontend # lint (config: backend/.golangci.yml) +# backend/ — the exclude_frontend and unit tags are mandatory locally; CI uses them too +go test -tags=exclude_frontend,unit ./... # unit/integration tests +go test -tags=exclude_frontend,unit -run TestName ./internal/... # a single test +golangci-lint run # lint (config: backend/.golangci.yml - includes build tags) # frontend/ (or root) pnpm check # svelte-check — the ONLY frontend type gate (no unit tests exist) @@ -32,9 +32,8 @@ cd ../.. && pnpm test # = playwright test in tests/ ## Critical gotchas -- **`exclude_frontend` build tag.** `backend/frontend/` embeds `dist/` via `go:embed`. Without - a built frontend present, plain `go run`/`go test`/`golangci-lint` fail to compile. Always pass - `-tags exclude_frontend` for backend dev/test/lint. Production builds omit it to embed the SPA. +- **`exclude_frontend` and `unit` build tags.** Without them plain `go run`/`go test`/`golangci-lint` fail + to run. Always pass `-tags exclude_frontend,unit` for backend dev/test/lint. - **Never edit generated files:** `frontend/src/lib/paraglide/**` (Paraglide i18n output) and `backend/frontend/dist/**`. For i18n, only edit `frontend/messages/en.json`; other locales come from Crowdin. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 304fdefe..12820898 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,4 +125,4 @@ If you make any changes to the application, you have to rebuild the test environ In the backend we are using unit tests with the built-in Go testing framework. The tests are located in the same folder as the code they are testing and have the `_test.go` suffix. -To run the tests, simply run `go test ./...` from the root of the `backend` folder. +To run the tests, simply run `go test -tags=exclude_frontend,unit ./...` from the root of the `backend` folder. diff --git a/backend/.golangci.yml b/backend/.golangci.yml index c9ed33e5..5b0f1919 100644 --- a/backend/.golangci.yml +++ b/backend/.golangci.yml @@ -2,6 +2,9 @@ version: "2" run: tests: true timeout: 5m + build-tags: + - unit + - exclude_frontend linters: default: none enable: diff --git a/backend/Makefile b/backend/Makefile new file mode 100644 index 00000000..9bbb6eb5 --- /dev/null +++ b/backend/Makefile @@ -0,0 +1,11 @@ +.PHONY: test +test: + go test -tags exclude_frontend,unit ./... + +.PHONY: test-race +test-race: + CGO_ENABLED=1 go test -race -tags exclude_frontend,unit ./... + +.PHONY: lint +lint: + golangci-lint run -c .golangci.yml diff --git a/backend/go.mod b/backend/go.mod index 86740638..c024d8ad 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -24,9 +24,8 @@ require ( github.com/go-webauthn/webauthn v0.17.4 github.com/golang-migrate/migrate/v4 v4.19.1 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-uuid v1.0.3 - github.com/italypaleale/francis v0.1.0-beta.6 - github.com/italypaleale/go-kit v0.0.0-20260705144331-cc6661f9a8cf + github.com/italypaleale/francis v0.1.0-beta.10 + github.com/italypaleale/go-kit v0.0.0-20260708054611-e276b65dd3be github.com/italypaleale/go-sql-utils v0.2.4 github.com/jackc/pgx/v5 v5.10.0 github.com/jinzhu/copier v0.4.0 @@ -182,7 +181,7 @@ require ( github.com/prometheus/procfs v0.21.1 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.60.0 // indirect - github.com/quic-go/webtransport-go v0.10.0 // indirect + github.com/quic-go/webtransport-go v0.11.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 60642734..85da7ebd 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -263,10 +263,10 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/italypaleale/francis v0.1.0-beta.6 h1:88XGnFMwQEMsGSJvI9vrGGvp//WKipXH8YPnfNHDStQ= -github.com/italypaleale/francis v0.1.0-beta.6/go.mod h1:3QQ1iSRGzvxJZ5gfVlVR1RyPDUAgzyN+m2vHeeR7OVs= -github.com/italypaleale/go-kit v0.0.0-20260705144331-cc6661f9a8cf h1:8tk4010URtPgVqwZb7CDiNsuQI71qc8t7VSS4IaxoTY= -github.com/italypaleale/go-kit v0.0.0-20260705144331-cc6661f9a8cf/go.mod h1:pl0r3F+thZIyDsyDo8aOUsAIVcsRuAeP1bB4GuAHLoY= +github.com/italypaleale/francis v0.1.0-beta.10 h1:LCYVwkZAkakv7g5ZS6TVIRH7hozr1A0eg24LXWWBJYE= +github.com/italypaleale/francis v0.1.0-beta.10/go.mod h1:vqKhwdLs5Sx+n6JCNknEKAODtEU51E9/LC1q9JAG3zk= +github.com/italypaleale/go-kit v0.0.0-20260708054611-e276b65dd3be h1:jgu+Mdsda++LqPxz8cj8vvgiFINQ8PhFB4Q1VZpyPjs= +github.com/italypaleale/go-kit v0.0.0-20260708054611-e276b65dd3be/go.mod h1:pl0r3F+thZIyDsyDo8aOUsAIVcsRuAeP1bB4GuAHLoY= github.com/italypaleale/go-sql-utils v0.2.4 h1:6CN8y3qEdNzvYlS/JK6N65E8cL9F8a6OBCJjzaQIv3c= github.com/italypaleale/go-sql-utils v0.2.4/go.mod h1:BJStxMfB6fzYVcOe0oZQCjGIPZQu76UBmg1Wuy6Z/7I= github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ= @@ -444,8 +444,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0= github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk= -github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI= -github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow= +github.com/quic-go/webtransport-go v0.11.1 h1:rrFQMO+7/52ZDJ04fsrjIaWqn6q1z1MYo9iVFq6JtbA= +github.com/quic-go/webtransport-go v0.11.1/go.mod h1:SHgEzUFVyj+9WUSuGB1P6Zd351Pww2leWV3SwlTovkA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= diff --git a/backend/internal/bootstrap/actors_bootstrap.go b/backend/internal/bootstrap/actors_bootstrap.go index 9ffa1485..7a697d45 100644 --- a/backend/internal/bootstrap/actors_bootstrap.go +++ b/backend/internal/bootstrap/actors_bootstrap.go @@ -18,7 +18,6 @@ import ( "github.com/pocket-id/pocket-id/backend/internal/common" "github.com/pocket-id/pocket-id/backend/internal/job" "github.com/pocket-id/pocket-id/backend/internal/middleware" - "github.com/pocket-id/pocket-id/backend/internal/service" "github.com/pocket-id/pocket-id/backend/internal/storage" "github.com/pocket-id/pocket-id/backend/internal/utils/crypto" ) @@ -28,7 +27,7 @@ type NewActorsOpts struct { Postgres *pgxpool.Pool EnvConfig *common.EnvConfigSchema - AppConfig *service.AppConfigService + InstanceID string HttpClient *http.Client DB *gorm.DB FileStorage storage.FileStorage @@ -52,20 +51,6 @@ func NewActors(o NewActorsOpts) (*local.Host, map[string]*ratelimit.RateLimitSer local.WithShutdownGracePeriod(10 * time.Second), } - // Add all cron jobs - cronjobs, err := o.getCronJobs() - if err != nil { - return nil, nil, err - } - opts = append(opts, cronjobs...) - - // Add the rate limiters - rateLimiters, rateLimiterOpts, err := o.getRateLimiters() - if err != nil { - return nil, nil, err - } - opts = append(opts, rateLimiterOpts...) - // Add the database connection providerOpt, err := o.getProvider() if err != nil { @@ -79,6 +64,18 @@ func NewActors(o NewActorsOpts) (*local.Host, map[string]*ratelimit.RateLimitSer return nil, nil, fmt.Errorf("failed to create actor host: %w", err) } + // Add all cron jobs + err = o.registerCronJobs(h) + if err != nil { + return nil, nil, err + } + + // Add the rate limiters + rateLimiters, err := o.registerRateLimiters(h) + if err != nil { + return nil, nil, err + } + // Bind a service for each rate limiter so the middleware can invoke them rateLimitServices := make(map[string]*ratelimit.RateLimitService, len(rateLimiters)) for name, rl := range rateLimiters { @@ -90,8 +87,9 @@ func NewActors(o NewActorsOpts) (*local.Host, map[string]*ratelimit.RateLimitSer // Derive a PSK from the global encryption key func (o *NewActorsOpts) getPSK() ([]byte, error) { + // This is tied to the instance ID of the Pocket ID deployment/cluster // Note: changing the key derivation or the seed is a breaking change - return crypto.DeriveKey(o.EnvConfig.EncryptionKey, "pocketid/actors-psk") + return crypto.DeriveKey(o.EnvConfig.EncryptionKey, "pocketid/actors-psk/"+o.InstanceID) } func (o *NewActorsOpts) getProvider() (local.HostOption, error) { @@ -111,37 +109,44 @@ func (o *NewActorsOpts) getProvider() (local.HostOption, error) { } } -func (o *NewActorsOpts) getCronJobs() (opts []local.HostOption, err error) { +func (o *NewActorsOpts) registerCronJobs(host *local.Host) (err error) { // In test mode, we do not register anything if common.EnvConfig.AppEnv == "test" { - return opts, nil + return nil } // Register the analytics job - analyticsJob, err := job.GetAnalyticsJob(o.AppConfig, o.HttpClient) + analyticsJob, err := job.GetAnalyticsJob(o.HttpClient, o.InstanceID) if err != nil { - return nil, fmt.Errorf("failed to get analytics cron job: %w", err) + return fmt.Errorf("failed to get analytics cron job: %w", err) } + + // This could be nil if analytics are disabled if analyticsJob != nil { - // This could be nil if analytics are disabled - opts = append(opts, local.WithBuiltInActor(analyticsJob)) + err = host.RegisterBuiltInActor(analyticsJob) + if err != nil { + return fmt.Errorf("error registering built-in actor for analytics job: %w", err) + } } // Register the file cleanup jobs fileCleanupJobs, err := job.GetFileCleanupJobs(o.DB, o.FileStorage) if err != nil { - return nil, fmt.Errorf("failed to get file cleanup cron jobs: %w", err) + return fmt.Errorf("failed to get file cleanup cron jobs: %w", err) } for _, j := range fileCleanupJobs { - opts = append(opts, local.WithBuiltInActor(j)) + err = host.RegisterBuiltInActor(j) + if err != nil { + return fmt.Errorf("error registering built-in actor for cleanup job: %w", err) + } } - return opts, nil + return nil } -// getRateLimiters creates a built-in rate-limit actor for each middleware policy and returns both the created actors (keyed by policy name) and the host options to register them +// registerRateLimiters creates a built-in rate-limit actor for each middleware policy and returns both the created actors (keyed by policy name) and the host options to register them // Unlike cron jobs, rate limiters keep no durable state, so they are registered in every environment -func (o *NewActorsOpts) getRateLimiters() (actors map[string]*ratelimit.RateLimit, opts []local.HostOption, err error) { +func (o *NewActorsOpts) registerRateLimiters(host *local.Host) (actors map[string]*ratelimit.RateLimit, err error) { policies := middleware.RateLimitPolicies() actors = make(map[string]*ratelimit.RateLimit, len(policies)) for _, p := range policies { @@ -152,11 +157,15 @@ func (o *NewActorsOpts) getRateLimiters() (actors map[string]*ratelimit.RateLimi ratelimit.WithBurst(p.Burst), ) if err != nil { - return nil, nil, fmt.Errorf("error creating rate limiter %q: %w", p.Name, err) + return nil, fmt.Errorf("error creating rate limiter %q: %w", p.Name, err) } actors[p.Name] = rl - opts = append(opts, local.WithBuiltInActor(rl)) + + err = host.RegisterBuiltInActor(rl) + if err != nil { + return nil, fmt.Errorf("error registering built-in actor for rate limiter '%s': %w", p.Name, err) + } } - return actors, opts, nil + return actors, nil } diff --git a/backend/internal/bootstrap/actors_bootstrap_test.go b/backend/internal/bootstrap/actors_bootstrap_test.go index 6a07d45e..5b59a78a 100644 --- a/backend/internal/bootstrap/actors_bootstrap_test.go +++ b/backend/internal/bootstrap/actors_bootstrap_test.go @@ -14,13 +14,15 @@ func TestNewActorsOptsGetPSKUsesStableValue(t *testing.T) { EnvConfig: &common.EnvConfigSchema{ EncryptionKey: []byte("test-encryption-key"), }, + // Constant value for this test + InstanceID: "ee05c3eb-8129-47a6-a1c7-849998b6f876", } - expectedHex := "651300d35d48998d0fa66ac89091bcde8ed0fd0aa35fbb849f068410c64807e9" + expectedHex := "db09067fa194c3731bf77b6415a1c5d903f03d4557605ba3236b31f6eddfc8d7" expected, err := hex.DecodeString(expectedHex) require.NoError(t, err) actual, err := opts.getPSK() require.NoError(t, err) - require.Equal(t, expected, actual) + require.Equalf(t, expected, actual, "actual result: %s", actual) } diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go index b781e705..53505426 100644 --- a/backend/internal/bootstrap/bootstrap.go +++ b/backend/internal/bootstrap/bootstrap.go @@ -9,10 +9,12 @@ import ( _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/italypaleale/francis/host/local" "github.com/italypaleale/go-kit/servicerunner" "gorm.io/gorm" "github.com/pocket-id/pocket-id/backend/internal/common" + "github.com/pocket-id/pocket-id/backend/internal/instanceid" "github.com/pocket-id/pocket-id/backend/internal/job" "github.com/pocket-id/pocket-id/backend/internal/service" "github.com/pocket-id/pocket-id/backend/internal/storage" @@ -46,6 +48,13 @@ func Bootstrap(ctx context.Context) error { }() } + // Load the instance ID + // This is stored in the "kv" table, and generated on first startup + instanceID, err := instanceid.Load(ctx, db) + if err != nil { + return fmt.Errorf("failed to initialize instance ID: %w", err) + } + // Init storage fileStorage, err := InitStorage(ctx, db) if err != nil { @@ -64,8 +73,34 @@ func Bootstrap(ctx context.Context) error { return fmt.Errorf("failed to create job scheduler: %w", err) } + // Init the actors + // The actor host is created and started before the services, so services can depend on it once it's ready + actorsOpts := NewActorsOpts{ + Postgres: pg, + + EnvConfig: &common.EnvConfig, + InstanceID: instanceID, + HttpClient: httpClient, + DB: db, + FileStorage: fileStorage, + } + if pg == nil { + actorsOpts.SQLite, err = db.DB() + if err != nil { + return fmt.Errorf("failed to get *sql.DB connection from Gorm: %w", err) + } + } + actors, rateLimitServices, err := NewActors(actorsOpts) + if err != nil { + return fmt.Errorf("failed to initialize actors: %w", err) + } + + // Run the actor host as a background service and get a "ready" signal that other services can wait on + actorsRun, actorsReady := actorsRunServiceFn(actors) + services = append(services, actorsRun) + // Create all services - svc, err := initServices(ctx, db, httpClient, imageExtensions, fileStorage, scheduler) + svc, err := initServices(ctx, db, instanceID, httpClient, imageExtensions, fileStorage, scheduler) if err != nil { return fmt.Errorf("failed to initialize services: %w", err) } @@ -93,28 +128,6 @@ func Bootstrap(ctx context.Context) error { return nil }) - // Init the actors - actorsOpts := NewActorsOpts{ - Postgres: pg, - - EnvConfig: &common.EnvConfig, - AppConfig: svc.appConfigService, - HttpClient: httpClient, - DB: db, - FileStorage: fileStorage, - } - if pg == nil { - actorsOpts.SQLite, err = db.DB() - if err != nil { - return fmt.Errorf("failed to get *sql.DB connection from Gorm: %w", err) - } - } - actors, rateLimitServices, err := NewActors(actorsOpts) - if err != nil { - return fmt.Errorf("failed to initialize actors: %w", err) - } - services = append(services, actors.Run) - // Register scheduled jobs, only in non-test mode if common.EnvConfig.AppEnv != "test" { err = registerScheduledJobs(ctx, db, svc, scheduler) @@ -131,7 +144,9 @@ func Bootstrap(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to initialize router: %w", err) } - services = append(services, router) + + // The router must wait on the actor host being ready, since the rate-limit middleware invokes actors + services = append(services, actorsReady.Await(router)) // Run all background services // This call blocks until the context is canceled @@ -146,6 +161,37 @@ func Bootstrap(ctx context.Context) error { return nil } +// actorsRunServiceFn wraps the actor host's Run method in a background service and returns a "ready" signal that other services can wait on +func actorsRunServiceFn(actors *local.Host) (servicerunner.Service, *servicerunner.Ready) { + actorsReady := servicerunner.NewReady() + fn := func(ctx context.Context) error { + runErrCh := make(chan error, 1) + go func() { + runErrCh <- actors.Run(ctx) + }() + + // Wait for the right signal + select { + case <-actors.Ready(): + // Actor host is ready, signal actorsReady + actorsReady.Signal() + case runErr := <-runErrCh: + // Run returned with an error + return runErr + case <-ctx.Done(): + // Context canceled + return ctx.Err() + } + + // Now the actor host is running + // This goroutine must stay up until the actor host returns + // Here, context cancellation will surface through this channel too + return <-runErrCh + } + + return fn, actorsReady +} + func InitStorage(ctx context.Context, db *gorm.DB) (fileStorage storage.FileStorage, err error) { switch common.EnvConfig.FileBackend { case storage.TypeFileSystem: diff --git a/backend/internal/bootstrap/services_bootstrap.go b/backend/internal/bootstrap/services_bootstrap.go index 3a670b9a..cfc8ac8d 100644 --- a/backend/internal/bootstrap/services_bootstrap.go +++ b/backend/internal/bootstrap/services_bootstrap.go @@ -44,7 +44,7 @@ type services struct { } // Initializes all services -func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, imageExtensions map[string]string, fileStorage storage.FileStorage, scheduler *job.Scheduler) (svc *services, err error) { +func initServices(ctx context.Context, db *gorm.DB, instanceID string, httpClient *http.Client, imageExtensions map[string]string, fileStorage storage.FileStorage, scheduler *job.Scheduler) (svc *services, err error) { svc = &services{} svc.appConfigService, err = service.NewAppConfigService(ctx, db) @@ -63,7 +63,7 @@ func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, ima svc.geoLiteService = service.NewGeoLiteService(httpClient) svc.auditLogService = service.NewAuditLogService(db, svc.appConfigService, svc.emailService, svc.geoLiteService) - svc.jwtService, err = service.NewJwtService(ctx, db, svc.appConfigService) + svc.jwtService, err = service.NewJwtService(ctx, db, instanceID, svc.appConfigService) if err != nil { return nil, fmt.Errorf("failed to create JWT service: %w", err) } diff --git a/backend/internal/cmds/encryption_key_rotate.go b/backend/internal/cmds/encryption_key_rotate.go index ab51f944..d28a535f 100644 --- a/backend/internal/cmds/encryption_key_rotate.go +++ b/backend/internal/cmds/encryption_key_rotate.go @@ -12,8 +12,8 @@ import ( "github.com/pocket-id/pocket-id/backend/internal/bootstrap" "github.com/pocket-id/pocket-id/backend/internal/common" + "github.com/pocket-id/pocket-id/backend/internal/instanceid" datatype "github.com/pocket-id/pocket-id/backend/internal/model/types" - "github.com/pocket-id/pocket-id/backend/internal/service" "github.com/pocket-id/pocket-id/backend/internal/utils" jwkutils "github.com/pocket-id/pocket-id/backend/internal/utils/jwk" ) @@ -35,7 +35,12 @@ func init() { return err } - return encryptionKeyRotate(cmd.Context(), flags, db, &common.EnvConfig) + instanceID, err := instanceid.Load(cmd.Context(), db) + if err != nil { + return err + } + + return encryptionKeyRotate(cmd.Context(), flags, db, instanceID, &common.EnvConfig) }, } @@ -45,7 +50,7 @@ func init() { rootCmd.AddCommand(encryptionKeyRotateCmd) } -func encryptionKeyRotate(ctx context.Context, flags encryptionKeyRotateFlags, db *gorm.DB, envConfig *common.EnvConfigSchema) error { +func encryptionKeyRotate(ctx context.Context, flags encryptionKeyRotateFlags, db *gorm.DB, instanceID string, envConfig *common.EnvConfigSchema) error { oldKey := envConfig.EncryptionKey newKey := []byte(flags.NewKey) if len(newKey) == 0 { @@ -67,12 +72,6 @@ func encryptionKeyRotate(ctx context.Context, flags encryptionKeyRotateFlags, db } } - appConfigService, err := service.NewAppConfigService(ctx, db) - if err != nil { - return fmt.Errorf("failed to create app config service: %w", err) - } - instanceID := appConfigService.GetDbConfig().InstanceID.Value - // Derive the encryption keys used for the JWK encryption oldKek, err := jwkutils.LoadKeyEncryptionKey(&common.EnvConfigSchema{EncryptionKey: oldKey}, instanceID) if err != nil { diff --git a/backend/internal/cmds/encryption_key_rotate_test.go b/backend/internal/cmds/encryption_key_rotate_test.go index 8d27e975..a90749ee 100644 --- a/backend/internal/cmds/encryption_key_rotate_test.go +++ b/backend/internal/cmds/encryption_key_rotate_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/require" "github.com/pocket-id/pocket-id/backend/internal/common" + "github.com/pocket-id/pocket-id/backend/internal/instanceid" datatype "github.com/pocket-id/pocket-id/backend/internal/model/types" - "github.com/pocket-id/pocket-id/backend/internal/service" jwkutils "github.com/pocket-id/pocket-id/backend/internal/utils/jwk" testingutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing" ) @@ -25,9 +25,8 @@ func TestEncryptionKeyRotate(t *testing.T) { db := testingutils.NewDatabaseForTest(t) - appConfigService, err := service.NewAppConfigService(t.Context(), db) + instanceID, err := instanceid.Load(t.Context(), db) require.NoError(t, err) - instanceID := appConfigService.GetDbConfig().InstanceID.Value oldKek, err := jwkutils.LoadKeyEncryptionKey(envConfig, instanceID) require.NoError(t, err) @@ -61,7 +60,8 @@ func TestEncryptionKeyRotate(t *testing.T) { NewKey: string(newKey), Yes: true, } - require.NoError(t, encryptionKeyRotate(t.Context(), flags, db, envConfig)) + err = encryptionKeyRotate(t.Context(), flags, db, instanceID, envConfig) + require.NoError(t, err) newKek, err := jwkutils.LoadKeyEncryptionKey(&common.EnvConfigSchema{EncryptionKey: newKey}, instanceID) require.NoError(t, err) diff --git a/backend/internal/cmds/key_rotate.go b/backend/internal/cmds/key_rotate.go index cd1c652e..077ea2c9 100644 --- a/backend/internal/cmds/key_rotate.go +++ b/backend/internal/cmds/key_rotate.go @@ -13,7 +13,7 @@ import ( "github.com/pocket-id/pocket-id/backend/internal/bootstrap" "github.com/pocket-id/pocket-id/backend/internal/common" - "github.com/pocket-id/pocket-id/backend/internal/service" + "github.com/pocket-id/pocket-id/backend/internal/instanceid" "github.com/pocket-id/pocket-id/backend/internal/utils" jwkutils "github.com/pocket-id/pocket-id/backend/internal/utils/jwk" ) @@ -36,7 +36,12 @@ func init() { return err } - return keyRotate(cmd.Context(), flags, db, &common.EnvConfig) + instanceID, err := instanceid.Load(cmd.Context(), db) + if err != nil { + return err + } + + return keyRotate(cmd.Context(), flags, db, instanceID, &common.EnvConfig) }, } @@ -47,7 +52,7 @@ func init() { rootCmd.AddCommand(keyRotateCmd) } -func keyRotate(ctx context.Context, flags keyRotateFlags, db *gorm.DB, envConfig *common.EnvConfigSchema) error { +func keyRotate(ctx context.Context, flags keyRotateFlags, db *gorm.DB, instanceID string, envConfig *common.EnvConfigSchema) error { // Validate the flags switch strings.ToUpper(flags.Alg) { case jwa.RS256().String(), jwa.RS384().String(), jwa.RS512().String(), @@ -83,14 +88,8 @@ func keyRotate(ctx context.Context, flags keyRotateFlags, db *gorm.DB, envConfig } } - // Init the services we need - appConfigService, err := service.NewAppConfigService(ctx, db) - if err != nil { - return fmt.Errorf("failed to create app config service: %w", err) - } - // Get the key provider - keyProvider, err := jwkutils.GetKeyProvider(db, envConfig, appConfigService.GetDbConfig().InstanceID.Value) + keyProvider, err := jwkutils.GetKeyProvider(db, envConfig, instanceID) if err != nil { return fmt.Errorf("failed to get key provider: %w", err) } diff --git a/backend/internal/cmds/key_rotate_test.go b/backend/internal/cmds/key_rotate_test.go index fb4f5548..6e051f48 100644 --- a/backend/internal/cmds/key_rotate_test.go +++ b/backend/internal/cmds/key_rotate_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/pocket-id/pocket-id/backend/internal/common" - "github.com/pocket-id/pocket-id/backend/internal/service" + "github.com/pocket-id/pocket-id/backend/internal/instanceid" jwkutils "github.com/pocket-id/pocket-id/backend/internal/utils/jwk" testingutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing" ) @@ -81,17 +81,16 @@ func testKeyRotateWithDatabaseStorage(t *testing.T, flags keyRotateFlags, wantEr // Create test database db := testingutils.NewDatabaseForTest(t) - // Initialize app config service and create instance - appConfigService, err := service.NewAppConfigService(t.Context(), db) + // Load the instance ID + instanceID, err := instanceid.Load(t.Context(), db) require.NoError(t, err) - instanceID := appConfigService.GetDbConfig().InstanceID.Value // Get key provider keyProvider, err := jwkutils.GetKeyProvider(db, envConfig, instanceID) require.NoError(t, err) // Run the key rotation - err = keyRotate(t.Context(), flags, db, envConfig) + err = keyRotate(t.Context(), flags, db, instanceID, envConfig) if wantErr { require.Error(t, err) diff --git a/backend/internal/instanceid/instanceid.go b/backend/internal/instanceid/instanceid.go new file mode 100644 index 00000000..b4b6e0e5 --- /dev/null +++ b/backend/internal/instanceid/instanceid.go @@ -0,0 +1,50 @@ +package instanceid + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +// Load the instance ID for the Pocket ID cluster from the "kv" table +// If no instance ID exists yet, a new one is generated and persisted atomically +func Load(parentCtx context.Context, db *gorm.DB) (string, error) { + // Candidate value used only if there's no instance ID stored yet + newInstanceID := uuid.NewString() + + // We use a raw query because gorm can't build it for us in this atomic way + // The syntax is valid for both SQLite and Postgres + var value string + ctx, cancel := context.WithTimeout(parentCtx, 10*time.Second) + defer cancel() + err := db. + WithContext(ctx). + Raw( + `INSERT INTO kv (key, value) + VALUES ('instance_id', ?) + ON CONFLICT (key) DO UPDATE SET + value = CASE + WHEN kv.value = '' THEN excluded.value + ELSE kv.value + END + RETURNING value`, + newInstanceID, + ). + Scan(&value). + Error + if err != nil { + return "", fmt.Errorf("failed to load instance ID from the database: %w", err) + } + + // We should have an instance ID now + // This case should never happen + if value == "" { + return "", errors.New("retrieved instance ID is unexpectedly empty") + } + + return value, nil +} diff --git a/backend/internal/instanceid/instanceid_test.go b/backend/internal/instanceid/instanceid_test.go new file mode 100644 index 00000000..b850efd2 --- /dev/null +++ b/backend/internal/instanceid/instanceid_test.go @@ -0,0 +1,190 @@ +package instanceid + +import ( + "sync" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + "github.com/pocket-id/pocket-id/backend/internal/model" + testutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing" +) + +// readInstanceID returns the value stored in the kv table under the "instance_id" key, together with the number of rows that match (to detect duplicates) +func readInstanceID(t *testing.T, db *gorm.DB) (value string, count int) { + t.Helper() + var rows []model.KV + err := db.Where("key = ?", "instance_id").Find(&rows).Error + require.NoError(t, err) + if len(rows) == 0 { + return "", 0 + } + require.NotNil(t, rows[0].Value, "instance_id value should not be NULL") + return *rows[0].Value, len(rows) +} + +func TestLoad(t *testing.T) { + t.Run("generates and persists a new instance ID when none exists", func(t *testing.T) { + db := testutils.NewDatabaseForTest(t) + + id, err := Load(t.Context(), db) + require.NoError(t, err) + require.NotEmpty(t, id) + + // The generated ID should be a valid UUID + _, err = uuid.Parse(id) + require.NoError(t, err, "generated instance ID should be a valid UUID") + + // The value should have been persisted in the kv table + stored, count := readInstanceID(t, db) + require.Equal(t, 1, count) + require.Equal(t, id, stored) + }) + + t.Run("returns the existing instance ID on subsequent calls", func(t *testing.T) { + db := testutils.NewDatabaseForTest(t) + + first, err := Load(t.Context(), db) + require.NoError(t, err) + require.NotEmpty(t, first) + + // A second call must return the same ID and must not create a duplicate row + second, err := Load(t.Context(), db) + require.NoError(t, err) + require.Equal(t, first, second) + + stored, count := readInstanceID(t, db) + require.Equal(t, 1, count) + require.Equal(t, first, stored) + }) + + t.Run("returns an instance ID that was already stored", func(t *testing.T) { + db := testutils.NewDatabaseForTest(t) + + // Seed an instance ID directly in the database, e.g. as written by the migration that moved it out of the app config table + existing := "existing-instance-id" + err := db.Create(&model.KV{Key: "instance_id", Value: &existing}).Error + require.NoError(t, err) + + id, err := Load(t.Context(), db) + require.NoError(t, err) + require.Equal(t, existing, id) + + stored, count := readInstanceID(t, db) + require.Equal(t, 1, count) + require.Equal(t, existing, stored) + }) + + t.Run("concurrent calls without an existing instance ID converge on a single value", func(t *testing.T) { + // This needs a database that allows concurrent access, so the goroutines race + db := testutils.NewConcurrentDatabaseForTest(t) + + const parallel = 25 + + // Each goroutine writes to its own index, so no locking is needed to collect the results + ids := make([]string, parallel) + loadErrs := make([]error, parallel) + + // The start channel is a barrier so all goroutines race their Load call at once + start := make(chan struct{}) + + var wg sync.WaitGroup + wg.Add(parallel) + for i := range parallel { + go func() { + defer wg.Done() + <-start + ids[i], loadErrs[i] = Load(t.Context(), db) + }() + } + close(start) + wg.Wait() + + // No concurrent call should have failed + for i, err := range loadErrs { + require.NoErrorf(t, err, "goroutine %d errored", i) + } + + // Every goroutine must have observed the same, non-empty instance ID, meaning only one of them actually generated and persisted a value + first := ids[0] + require.NotEmpty(t, first) + for _, id := range ids { + require.Equal(t, first, id) + } + + // Only a single row must have been persisted, and it must match the shared value + stored, count := readInstanceID(t, db) + require.Equal(t, 1, count) + require.Equal(t, first, stored) + }) +} + +// TestMigrateFromAppConfig validates the database migration that moves the instance ID out of the legacy app_config_variables table and into the kv table +// The value used to be stored as an app config variable under the "instanceId" key +func TestMigrateFromAppConfig(t *testing.T) { + // Version of the migration right before "20260708130000_move_instance_id_to_kv", which performs the move + // We seed the legacy table right after this version, so the following migration has data to operate on + const versionBeforeMove uint = 20260708120000 + + // seedAppConfigInstanceID inserts a legacy "instanceId" row into the app_config_variables table + seedAppConfigInstanceID := func(value string) func(t *testing.T, db *gorm.DB) { + return func(t *testing.T, db *gorm.DB) { + t.Helper() + err := db.Exec(`INSERT INTO app_config_variables ("key", "value") VALUES ('instanceId', ?)`, value).Error + require.NoError(t, err) + } + } + + // countLegacyInstanceID returns the number of "instanceId" rows left in the app_config_variables table + countLegacyInstanceID := func(t *testing.T, db *gorm.DB) int64 { + t.Helper() + var count int64 + err := db.Table("app_config_variables").Where(`"key" = ?`, "instanceId").Count(&count).Error + require.NoError(t, err) + return count + } + + t.Run("moves an existing instance ID from app_config_variables into the kv table", func(t *testing.T) { + legacyID := uuid.NewString() + db := testutils.NewDatabaseForTestWithMigrationSeed(t, versionBeforeMove, seedAppConfigInstanceID(legacyID)) + + // The migration should have copied the value into the kv table under the "instance_id" key + stored, count := readInstanceID(t, db) + require.Equal(t, 1, count) + require.Equal(t, legacyID, stored) + + // The legacy row must have been removed from app_config_variables + require.Zero(t, countLegacyInstanceID(t, db)) + + // Load must return the migrated value without generating a new one + id, err := Load(t.Context(), db) + require.NoError(t, err) + require.Equal(t, legacyID, id) + + stored, count = readInstanceID(t, db) + require.Equal(t, 1, count) + require.Equal(t, legacyID, stored) + }) + + t.Run("keeps the existing kv value when both tables have an instance ID", func(t *testing.T) { + legacyID := uuid.NewString() + db := testutils.NewDatabaseForTestWithMigrationSeed(t, versionBeforeMove, func(t *testing.T, db *gorm.DB) { + t.Helper() + // An instance ID is already present in the kv table before the move runs + err := db.Exec(`INSERT INTO kv ("key", "value") VALUES ('instance_id', ?)`, "kv-instance-id").Error + require.NoError(t, err) + // And a different value lingers in the legacy app_config_variables table + seedAppConfigInstanceID(legacyID)(t, db) + }) + + // The migration must not clobber the value already stored in the kv table + stored, count := readInstanceID(t, db) + require.Equal(t, 1, count) + require.Equal(t, "kv-instance-id", stored) + + // The legacy row must still be removed regardless of the conflict + require.Zero(t, countLegacyInstanceID(t, db)) + }) +} diff --git a/backend/internal/job/analytics_job.go b/backend/internal/job/analytics_job.go index 6a9adfb3..e4ef4987 100644 --- a/backend/internal/job/analytics_job.go +++ b/backend/internal/job/analytics_job.go @@ -13,13 +13,12 @@ import ( "github.com/italypaleale/francis/builtin/cronjob" "github.com/pocket-id/pocket-id/backend/internal/common" - "github.com/pocket-id/pocket-id/backend/internal/service" ) const heartbeatUrl = "https://analytics.pocket-id.org/heartbeat" // GetAnalyticsJob returns the CronJob actor -func GetAnalyticsJob(appConfig *service.AppConfigService, httpClient *http.Client) (*cronjob.CronJob, error) { +func GetAnalyticsJob(httpClient *http.Client, instanceID string) (*cronjob.CronJob, error) { // Skip if analytics are disabled or not in production environment if common.EnvConfig.AnalyticsDisabled || !common.EnvConfig.AppEnv.IsProduction() { return nil, nil @@ -28,7 +27,7 @@ func GetAnalyticsJob(appConfig *service.AppConfigService, httpClient *http.Clien job := &AnalyticsJob{ httpClient: httpClient, } - err := job.createBody(appConfig) + err := job.createBody(instanceID) if err != nil { return nil, fmt.Errorf("error pre-computing request body: %w", err) } @@ -56,13 +55,13 @@ type AnalyticsJob struct { } // createBody pre-computes the body for all requests -func (j *AnalyticsJob) createBody(appConfig *service.AppConfigService) error { +func (j *AnalyticsJob) createBody(instanceID string) error { body, err := json.Marshal(struct { Version string `json:"version"` InstanceID string `json:"instance_id"` }{ Version: common.Version, - InstanceID: appConfig.GetDbConfig().InstanceID.Value, + InstanceID: instanceID, }) if err != nil { return fmt.Errorf("failed to marshal heartbeat body: %w", err) diff --git a/backend/internal/middleware/auth_middleware_test.go b/backend/internal/middleware/auth_middleware_test.go index acdb344b..08379af2 100644 --- a/backend/internal/middleware/auth_middleware_test.go +++ b/backend/internal/middleware/auth_middleware_test.go @@ -13,6 +13,7 @@ import ( "github.com/pocket-id/pocket-id/backend/internal/apikey" "github.com/pocket-id/pocket-id/backend/internal/common" + "github.com/pocket-id/pocket-id/backend/internal/instanceid" "github.com/pocket-id/pocket-id/backend/internal/model" datatype "github.com/pocket-id/pocket-id/backend/internal/model/types" "github.com/pocket-id/pocket-id/backend/internal/service" @@ -35,7 +36,10 @@ func TestWithApiKeyAuthDisabled(t *testing.T) { appConfigService, err := service.NewAppConfigService(t.Context(), db) require.NoError(t, err) - jwtService, err := service.NewJwtService(t.Context(), db, appConfigService) + instanceID, err := instanceid.Load(t.Context(), db) + require.NoError(t, err) + + jwtService, err := service.NewJwtService(t.Context(), db, instanceID, appConfigService) require.NoError(t, err) userService := service.NewUserService(db, jwtService, nil, nil, appConfigService, nil, nil, nil, nil) diff --git a/backend/internal/middleware/rate_limit_test.go b/backend/internal/middleware/rate_limit_test.go index eba3885e..863b6204 100644 --- a/backend/internal/middleware/rate_limit_test.go +++ b/backend/internal/middleware/rate_limit_test.go @@ -23,15 +23,16 @@ func startRateLimitServices(t *testing.T, policies ...RateLimitPolicy) map[strin t.Helper() limiters := make(map[string]*ratelimit.RateLimit, len(policies)) - opts := make([]local.HostOption, 0, len(policies)) - for _, p := range policies { - rl, err := ratelimit.New(p.Name, ratelimit.WithRate(p.Rate), ratelimit.WithPer(p.Per), ratelimit.WithBurst(p.Burst)) - require.NoError(t, err) - limiters[p.Name] = rl - opts = append(opts, local.WithBuiltInActor(rl)) - } + h := testutils.NewActorHostForTest(t, func(t *testing.T, h *local.Host) { + for _, p := range policies { + rl, err := ratelimit.New(p.Name, ratelimit.WithRate(p.Rate), ratelimit.WithPer(p.Per), ratelimit.WithBurst(p.Burst)) + require.NoError(t, err) + limiters[p.Name] = rl - h := testutils.NewActorHostForTest(t, opts...) + err = h.RegisterBuiltInActor(rl) + require.NoError(t, err) + } + }) services := make(map[string]*ratelimit.RateLimitService, len(limiters)) svc := h.Service() diff --git a/backend/internal/model/app_config.go b/backend/internal/model/app_config.go index 3e008478..76d508d5 100644 --- a/backend/internal/model/app_config.go +++ b/backend/internal/model/app_config.go @@ -44,8 +44,6 @@ type AppConfig struct { AllowUserSignups AppConfigVariable `key:"allowUserSignups,public"` // Public SignupDefaultUserGroupIDs AppConfigVariable `key:"signupDefaultUserGroupIDs"` SignupDefaultCustomClaims AppConfigVariable `key:"signupDefaultCustomClaims"` - // Internal - InstanceID AppConfigVariable `key:"instanceId,internal"` // Internal // Email RequireUserEmail AppConfigVariable `key:"requireUserEmail,public"` // Public SmtpHost AppConfigVariable `key:"smtpHost"` diff --git a/backend/internal/model/app_config_test.go b/backend/internal/model/app_config_test.go index aac70a62..4f28eb99 100644 --- a/backend/internal/model/app_config_test.go +++ b/backend/internal/model/app_config_test.go @@ -100,7 +100,7 @@ func TestAppConfigStructMatchesUpdateDto(t *testing.T) { // Verify every AppConfig field has a matching DTO field with the same name for fieldName, keyName := range appConfigFields { - if strings.HasSuffix(fieldName, "ImageType") || keyName == "instanceId" { + if strings.HasSuffix(fieldName, "ImageType") { // Skip internal fields that shouldn't be in the DTO continue } diff --git a/backend/internal/service/app_config_service.go b/backend/internal/service/app_config_service.go index a493796d..d2670fa9 100644 --- a/backend/internal/service/app_config_service.go +++ b/backend/internal/service/app_config_service.go @@ -10,7 +10,6 @@ import ( "sync/atomic" "time" - "github.com/hashicorp/go-uuid" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -40,11 +39,6 @@ func NewAppConfigService(ctx context.Context, db *gorm.DB) (service *AppConfigSe return nil, fmt.Errorf("failed to initialize app config service: %w", err) } - err = service.initInstanceID(ctx) - if err != nil { - return nil, fmt.Errorf("failed to initialize instance ID: %w", err) - } - return service, nil } @@ -74,8 +68,6 @@ func (s *AppConfigService) getDefaultDbConfig() *model.AppConfig { SignupDefaultUserGroupIDs: model.AppConfigVariable{Value: "[]"}, SignupDefaultCustomClaims: model.AppConfigVariable{Value: "[]"}, AccentColor: model.AppConfigVariable{Value: "default"}, - // Internal - InstanceID: model.AppConfigVariable{Value: ""}, // Email RequireUserEmail: model.AppConfigVariable{Value: "true"}, SmtpHost: model.AppConfigVariable{}, @@ -422,23 +414,3 @@ func (s *AppConfigService) loadDbConfigFromEnv(ctx context.Context, tx *gorm.DB) return dest, nil } - -func (s *AppConfigService) initInstanceID(ctx context.Context) error { - // Check if the instance ID is already set - instanceID := s.GetDbConfig().InstanceID.Value - if instanceID != "" { - return nil - } - - newInstanceID, err := uuid.GenerateUUID() - if err != nil { - return fmt.Errorf("failed to generate new instance ID: %w", err) - } - - err = s.UpdateAppConfigValues(ctx, "instanceId", newInstanceID) - if err != nil { - return fmt.Errorf("failed to update instance ID in the database: %w", err) - } - - return nil -} diff --git a/backend/internal/service/e2etest_service.go b/backend/internal/service/e2etest_service.go index 908d644c..bd32bc38 100644 --- a/backend/internal/service/e2etest_service.go +++ b/backend/internal/service/e2etest_service.go @@ -630,12 +630,22 @@ func (s *TestService) ResetAppConfig(ctx context.Context) error { return err } - // Manually set instance ID - err = s.appConfigService.UpdateAppConfigValues(ctx, "instanceId", "test-instance-id") + // Manually set the instance ID used to derive the JWK encryption key, so the seeded JWK can be decrypted + // Persist the fixed test value so it survives an export/import round-trip + const testInstanceID = "test-instance-id" + err = s.db.WithContext(ctx). + Exec( + `INSERT INTO kv (key, value) VALUES ('instance_id', ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`, + testInstanceID, + ). + Error if err != nil { return err } + // The instance ID is loaded once at startup, so we also set it directly on the JWT service so it takes effect immediately + s.jwtService.instanceID = testInstanceID + // Reload the app config from the database after resetting the values err = s.appConfigService.LoadDbConfig(ctx) if err != nil { diff --git a/backend/internal/service/jwt_service.go b/backend/internal/service/jwt_service.go index 48a1a252..07ce8b21 100644 --- a/backend/internal/service/jwt_service.go +++ b/backend/internal/service/jwt_service.go @@ -48,13 +48,14 @@ type JwtService struct { privateKey jwk.Key keyId string appConfigService *AppConfigService + instanceID string jwksEncoded []byte } -func NewJwtService(ctx context.Context, db *gorm.DB, appConfigService *AppConfigService) (*JwtService, error) { +func NewJwtService(ctx context.Context, db *gorm.DB, instanceID string, appConfigService *AppConfigService) (*JwtService, error) { service := &JwtService{} - err := service.init(ctx, db, appConfigService, &common.EnvConfig) + err := service.init(ctx, db, instanceID, appConfigService, &common.EnvConfig) if err != nil { return nil, err } @@ -62,10 +63,11 @@ func NewJwtService(ctx context.Context, db *gorm.DB, appConfigService *AppConfig return service, nil } -func (s *JwtService) init(ctx context.Context, db *gorm.DB, appConfigService *AppConfigService, envConfig *common.EnvConfigSchema) (err error) { +func (s *JwtService) init(ctx context.Context, db *gorm.DB, instanceID string, appConfigService *AppConfigService, envConfig *common.EnvConfigSchema) (err error) { s.appConfigService = appConfigService s.envConfig = envConfig s.db = db + s.instanceID = instanceID // Ensure keys are generated or loaded return s.LoadOrGenerateKey(ctx) @@ -73,7 +75,7 @@ func (s *JwtService) init(ctx context.Context, db *gorm.DB, appConfigService *Ap func (s *JwtService) LoadOrGenerateKey(ctx context.Context) error { // Get the key provider - keyProvider, err := jwkutils.GetKeyProvider(s.db, s.envConfig, s.appConfigService.GetDbConfig().InstanceID.Value) + keyProvider, err := jwkutils.GetKeyProvider(s.db, s.envConfig, s.instanceID) if err != nil { return fmt.Errorf("failed to get key provider: %w", err) } diff --git a/backend/internal/service/jwt_service_test.go b/backend/internal/service/jwt_service_test.go index e892d8ef..b0eae143 100644 --- a/backend/internal/service/jwt_service_test.go +++ b/backend/internal/service/jwt_service_test.go @@ -1,7 +1,6 @@ package service import ( - "context" "crypto/ecdsa" "crypto/ed25519" "crypto/elliptic" @@ -19,6 +18,7 @@ import ( "gorm.io/gorm" "github.com/pocket-id/pocket-id/backend/internal/common" + "github.com/pocket-id/pocket-id/backend/internal/instanceid" "github.com/pocket-id/pocket-id/backend/internal/model" jwkutils "github.com/pocket-id/pocket-id/backend/internal/utils/jwk" testutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing" @@ -35,23 +35,33 @@ func newTestEnvConfig() *common.EnvConfigSchema { } } -func initJwtService(t *testing.T, db *gorm.DB, appConfig *AppConfigService, envConfig *common.EnvConfigSchema) *JwtService { +func initJwtService(t *testing.T, db *gorm.DB, instanceID string, appConfig *AppConfigService, envConfig *common.EnvConfigSchema) *JwtService { t.Helper() service := &JwtService{} - err := service.init(t.Context(), db, appConfig, envConfig) + err := service.init(t.Context(), db, instanceID, appConfig, envConfig) require.NoError(t, err, "Failed to initialize JWT service") return service } -func setupJwtService(t *testing.T, appConfig *AppConfigService) (*JwtService, *gorm.DB, *common.EnvConfigSchema) { +func setupJwtService(t *testing.T, instanceID string, appConfig *AppConfigService) (*JwtService, *gorm.DB, *common.EnvConfigSchema) { t.Helper() db := testutils.NewDatabaseForTest(t) envConfig := newTestEnvConfig() - return initJwtService(t, db, appConfig, envConfig), db, envConfig + service := initJwtService(t, db, instanceID, appConfig, envConfig) + return service, db, envConfig +} + +func newInstanceID(t *testing.T, db *gorm.DB) string { + t.Helper() + + instanceID, err := instanceid.Load(t.Context(), db) + require.NoError(t, err) + + return instanceID } func newTestDbAndEnv(t *testing.T) (*gorm.DB, *common.EnvConfigSchema) { @@ -60,10 +70,10 @@ func newTestDbAndEnv(t *testing.T) (*gorm.DB, *common.EnvConfigSchema) { return testutils.NewDatabaseForTest(t), newTestEnvConfig() } -func saveKeyToDatabase(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigSchema, appConfig *AppConfigService, key jwk.Key) string { +func saveKeyToDatabase(t *testing.T, db *gorm.DB, instanceID string, envConfig *common.EnvConfigSchema, appConfig *AppConfigService, key jwk.Key) string { t.Helper() - keyProvider, err := jwkutils.GetKeyProvider(db, envConfig, appConfig.GetDbConfig().InstanceID.Value) + keyProvider, err := jwkutils.GetKeyProvider(db, envConfig, instanceID) require.NoError(t, err, "Failed to init key provider") err = keyProvider.SaveKey(t.Context(), key) @@ -84,15 +94,16 @@ func TestJwtService_Init(t *testing.T) { t.Run("should generate new key when none exists", func(t *testing.T) { db := testutils.NewDatabaseForTest(t) mockEnvConfig := newTestEnvConfig() + instanceID := newInstanceID(t, db) // Initialize the JWT service - service := initJwtService(t, db, mockConfig, mockEnvConfig) + service := initJwtService(t, db, instanceID, mockConfig, mockEnvConfig) // Verify the private key was set require.NotNil(t, service.privateKey, "Private key should be set") // Verify the key has been persisted in the database - keyProvider, err := jwkutils.GetKeyProvider(db, mockEnvConfig, mockConfig.GetDbConfig().InstanceID.Value) + keyProvider, err := jwkutils.GetKeyProvider(db, mockEnvConfig, instanceID) require.NoError(t, err, "Failed to init key provider") key, err := keyProvider.LoadKey(t.Context()) require.NoError(t, err, "Failed to load key from provider") @@ -111,16 +122,17 @@ func TestJwtService_Init(t *testing.T) { t.Run("should load existing JWK key", func(t *testing.T) { db := testutils.NewDatabaseForTest(t) mockEnvConfig := newTestEnvConfig() + instanceID := newInstanceID(t, db) // First create a service to generate a key - firstService := initJwtService(t, db, mockConfig, mockEnvConfig) + firstService := initJwtService(t, db, instanceID, mockConfig, mockEnvConfig) // Get the key ID of the first service origKeyID, ok := firstService.privateKey.KeyID() require.True(t, ok) // Now create a new service that should load the existing key - secondService := initJwtService(t, db, mockConfig, mockEnvConfig) + secondService := initJwtService(t, db, instanceID, mockConfig, mockEnvConfig) // Verify the loaded key has the same ID as the original loadedKeyID, ok := secondService.privateKey.KeyID() @@ -131,12 +143,13 @@ func TestJwtService_Init(t *testing.T) { t.Run("should load existing JWK for ECDSA keys", func(t *testing.T) { db := testutils.NewDatabaseForTest(t) mockEnvConfig := newTestEnvConfig() + instanceID := newInstanceID(t, db) // Create a new JWK and save it to the database - origKeyID := createECDSAKeyJWK(t, db, mockEnvConfig, mockConfig) + origKeyID := createECDSAKeyJWK(t, db, instanceID, mockEnvConfig, mockConfig) // Now create a new service that should load the existing key - svc := initJwtService(t, db, mockConfig, mockEnvConfig) + svc := initJwtService(t, db, instanceID, mockConfig, mockEnvConfig) // Ensure loaded key has the right algorithm alg, ok := svc.privateKey.Algorithm() @@ -152,12 +165,13 @@ func TestJwtService_Init(t *testing.T) { t.Run("should load existing JWK for EdDSA keys", func(t *testing.T) { db := testutils.NewDatabaseForTest(t) mockEnvConfig := newTestEnvConfig() + instanceID := newInstanceID(t, db) // Create a new JWK and save it to the database - origKeyID := createEdDSAKeyJWK(t, db, mockEnvConfig, mockConfig) + origKeyID := createEdDSAKeyJWK(t, db, instanceID, mockEnvConfig, mockConfig) // Now create a new service that should load the existing key - svc := initJwtService(t, db, mockConfig, mockEnvConfig) + svc := initJwtService(t, db, instanceID, mockConfig, mockEnvConfig) // Ensure loaded key has the right algorithm and curve alg, ok := svc.privateKey.Algorithm() @@ -181,9 +195,12 @@ func TestJwtService_GetPublicJWK(t *testing.T) { mockConfig := NewTestAppConfigService(&model.AppConfig{ SessionDuration: model.AppConfigVariable{Value: "60"}, // 60 minutes }) + db := testutils.NewDatabaseForTest(t) + mockEnvConfig := newTestEnvConfig() + instanceID := newInstanceID(t, db) t.Run("returns public key when private key is initialized", func(t *testing.T) { - service, _, _ := setupJwtService(t, mockConfig) + service, _, _ := setupJwtService(t, instanceID, mockConfig) // Get the JWK (public key) publicKey, err := service.GetPublicJWK() @@ -208,14 +225,11 @@ func TestJwtService_GetPublicJWK(t *testing.T) { }) t.Run("returns public key when ECDSA private key is initialized", func(t *testing.T) { - db := testutils.NewDatabaseForTest(t) - mockEnvConfig := newTestEnvConfig() - // Create an ECDSA key and save it in the database - originalKeyID := createECDSAKeyJWK(t, db, mockEnvConfig, mockConfig) + originalKeyID := createECDSAKeyJWK(t, db, instanceID, mockEnvConfig, mockConfig) // Create a JWT service that loads the ECDSA key - service := initJwtService(t, db, mockConfig, mockEnvConfig) + service := initJwtService(t, db, instanceID, mockConfig, mockEnvConfig) // Get the JWK (public key) publicKey, err := service.GetPublicJWK() @@ -248,10 +262,10 @@ func TestJwtService_GetPublicJWK(t *testing.T) { mockEnvConfig := newTestEnvConfig() // Create an EdDSA key and save it in the database - originalKeyID := createEdDSAKeyJWK(t, db, mockEnvConfig, mockConfig) + originalKeyID := createEdDSAKeyJWK(t, db, instanceID, mockEnvConfig, mockConfig) // Create a JWT service that loads the EdDSA key - service := initJwtService(t, db, mockConfig, mockEnvConfig) + service := initJwtService(t, db, instanceID, mockConfig, mockEnvConfig) // Get the JWK (public key) publicKey, err := service.GetPublicJWK() @@ -299,9 +313,11 @@ func TestGenerateVerifyAccessToken(t *testing.T) { mockConfig := NewTestAppConfigService(&model.AppConfig{ SessionDuration: model.AppConfigVariable{Value: "60"}, // 60 minutes }) + db, envConfig := newTestDbAndEnv(t) + instanceID := newInstanceID(t, db) t.Run("generates token for regular user", func(t *testing.T) { - service, _, _ := setupJwtService(t, mockConfig) + service, _, _ := setupJwtService(t, instanceID, mockConfig) user := model.User{ Base: model.Base{ID: "user123"}, @@ -342,7 +358,7 @@ func TestGenerateVerifyAccessToken(t *testing.T) { }) t.Run("generates token for admin user", func(t *testing.T) { - service, _, _ := setupJwtService(t, mockConfig) + service, _, _ := setupJwtService(t, instanceID, mockConfig) adminUser := model.User{ Base: model.Base{ID: "admin123"}, @@ -367,7 +383,7 @@ func TestGenerateVerifyAccessToken(t *testing.T) { }) t.Run("sets authentication method references claim when provided", func(t *testing.T) { - service, _, _ := setupJwtService(t, mockConfig) + service, _, _ := setupJwtService(t, instanceID, mockConfig) user := model.User{ Base: model.Base{ID: "user-with-auth-method"}, @@ -388,7 +404,7 @@ func TestGenerateVerifyAccessToken(t *testing.T) { customMockConfig := NewTestAppConfigService(&model.AppConfig{ SessionDuration: model.AppConfigVariable{Value: "30"}, // 30 minutes }) - service, _, _ := setupJwtService(t, customMockConfig) + service, _, _ := setupJwtService(t, instanceID, customMockConfig) user := model.User{ Base: model.Base{ID: "user456"}, @@ -408,9 +424,8 @@ func TestGenerateVerifyAccessToken(t *testing.T) { }) t.Run("works with Ed25519 keys", func(t *testing.T) { - db, envConfig := newTestDbAndEnv(t) - origKeyID := createEdDSAKeyJWK(t, db, envConfig, mockConfig) - service := initJwtService(t, db, mockConfig, envConfig) + origKeyID := createEdDSAKeyJWK(t, db, instanceID, envConfig, mockConfig) + service := initJwtService(t, db, instanceID, mockConfig, envConfig) loadedKeyID, ok := service.privateKey.KeyID() require.True(t, ok) @@ -447,9 +462,8 @@ func TestGenerateVerifyAccessToken(t *testing.T) { }) t.Run("works with P-256 keys", func(t *testing.T) { - db, envConfig := newTestDbAndEnv(t) - origKeyID := createECDSAKeyJWK(t, db, envConfig, mockConfig) - service := initJwtService(t, db, mockConfig, envConfig) + origKeyID := createECDSAKeyJWK(t, db, instanceID, envConfig, mockConfig) + service := initJwtService(t, db, instanceID, mockConfig, envConfig) loadedKeyID, ok := service.privateKey.KeyID() require.True(t, ok) @@ -486,9 +500,8 @@ func TestGenerateVerifyAccessToken(t *testing.T) { }) t.Run("works with RSA-4096 keys", func(t *testing.T) { - db, envConfig := newTestDbAndEnv(t) - origKeyID := createRSA4096KeyJWK(t, db, envConfig, mockConfig) - service := initJwtService(t, db, mockConfig, envConfig) + origKeyID := createRSA4096KeyJWK(t, db, instanceID, envConfig, mockConfig) + service := initJwtService(t, db, instanceID, mockConfig, envConfig) loadedKeyID, ok := service.privateKey.KeyID() require.True(t, ok) @@ -526,9 +539,6 @@ func TestGenerateVerifyAccessToken(t *testing.T) { } func TestTokenTypeValidator(t *testing.T) { - // Create a context for the validator function - ctx := context.Background() - t.Run("succeeds when token type matches expected type", func(t *testing.T) { // Create a token with the expected type token := jwt.New() @@ -539,7 +549,7 @@ func TestTokenTypeValidator(t *testing.T) { validator := TokenTypeValidator(AccessTokenJWTType) // Validate the token - err = validator(ctx, token) + err = validator(t.Context(), token) assert.NoError(t, err, "Validator should accept token with matching type") }) @@ -553,7 +563,7 @@ func TestTokenTypeValidator(t *testing.T) { validator := TokenTypeValidator(AccessTokenJWTType) // Validate the token - err = validator(ctx, token) + err = validator(t.Context(), token) require.Error(t, err, "Validator should reject token with non-matching type") assert.Contains(t, err.Error(), "invalid token type: expected access-token, got other-token") }) @@ -566,19 +576,19 @@ func TestTokenTypeValidator(t *testing.T) { validator := TokenTypeValidator(AccessTokenJWTType) // Validate the token - err := validator(ctx, token) + err := validator(t.Context(), token) require.Error(t, err, "Validator should reject token without type claim") assert.Contains(t, err.Error(), "failed to get token type claim") }) } -func importKey(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigSchema, appConfig *AppConfigService, privateKeyRaw any) string { +func importKey(t *testing.T, db *gorm.DB, instanceID string, envConfig *common.EnvConfigSchema, appConfig *AppConfigService, privateKeyRaw any) string { t.Helper() privateKey, err := jwkutils.ImportRawKey(privateKeyRaw, "", "") require.NoError(t, err, "Failed to import private key") - return saveKeyToDatabase(t, db, envConfig, appConfig, privateKey) + return saveKeyToDatabase(t, db, instanceID, envConfig, appConfig, privateKey) } // Because generating a RSA-406 key isn't immediate, we pre-compute one @@ -587,7 +597,7 @@ var ( rsaKeyPrecomputeOnce sync.Once ) -func createRSA4096KeyJWK(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigSchema, appConfig *AppConfigService) string { +func createRSA4096KeyJWK(t *testing.T, db *gorm.DB, instanceID string, envConfig *common.EnvConfigSchema, appConfig *AppConfigService) string { t.Helper() rsaKeyPrecomputeOnce.Do(func() { @@ -599,10 +609,10 @@ func createRSA4096KeyJWK(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigS }) // Import as JWK and save it - return importKey(t, db, envConfig, appConfig, rsaKeyPrecomputed) + return importKey(t, db, instanceID, envConfig, appConfig, rsaKeyPrecomputed) } -func createECDSAKeyJWK(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigSchema, appConfig *AppConfigService) string { +func createECDSAKeyJWK(t *testing.T, db *gorm.DB, instanceID string, envConfig *common.EnvConfigSchema, appConfig *AppConfigService) string { t.Helper() // Generate a new P-256 ECDSA key @@ -610,11 +620,11 @@ func createECDSAKeyJWK(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigSch require.NoError(t, err, "Failed to generate ECDSA key") // Import as JWK and save it - return importKey(t, db, envConfig, appConfig, privateKeyRaw) + return importKey(t, db, instanceID, envConfig, appConfig, privateKeyRaw) } // Helper function to create an Ed25519 key and save it as JWK -func createEdDSAKeyJWK(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigSchema, appConfig *AppConfigService) string { +func createEdDSAKeyJWK(t *testing.T, db *gorm.DB, instanceID string, envConfig *common.EnvConfigSchema, appConfig *AppConfigService) string { t.Helper() // Generate a new Ed25519 key pair @@ -622,5 +632,5 @@ func createEdDSAKeyJWK(t *testing.T, db *gorm.DB, envConfig *common.EnvConfigSch require.NoError(t, err, "Failed to generate Ed25519 key") // Import as JWK and save it - return importKey(t, db, envConfig, appConfig, privateKeyRaw) + return importKey(t, db, instanceID, envConfig, appConfig, privateKeyRaw) } diff --git a/backend/internal/service/one_time_access_service_test.go b/backend/internal/service/one_time_access_service_test.go index 44a189fe..8d2964fb 100644 --- a/backend/internal/service/one_time_access_service_test.go +++ b/backend/internal/service/one_time_access_service_test.go @@ -15,7 +15,8 @@ import ( func TestExchangeOneTimeAccessTokenRejectsDisabledUser(t *testing.T) { db := testutils.NewDatabaseForTest(t) appConfig := NewTestAppConfigService((&AppConfigService{}).getDefaultDbConfig()) - jwtService := initJwtService(t, db, appConfig, newTestEnvConfig()) + instanceID := newInstanceID(t, db) + jwtService := initJwtService(t, db, instanceID, appConfig, newTestEnvConfig()) auditLogService := NewAuditLogService(db, appConfig, nil, &GeoLiteService{}) oneTimeAccessService := NewOneTimeAccessService(db, nil, jwtService, auditLogService, nil, appConfig) diff --git a/backend/internal/utils/testing/actors.go b/backend/internal/utils/testing/actors.go index 00b2a6a2..34d8c544 100644 --- a/backend/internal/utils/testing/actors.go +++ b/backend/internal/utils/testing/actors.go @@ -1,3 +1,5 @@ +//go:build unit + // This file is only imported by unit tests package testing @@ -18,25 +20,27 @@ import ( const testActorHostPSK = "pocket-id-test-actor-host-psk-32bytes" // NewActorHostForTest starts a single-host Francis cluster backed by the in-memory provider, runs it, and waits until it is ready to serve invocations -// Pass built-in actors with local.WithBuiltInActor; the host is stopped when the test ends +// The register callback, if not nil, runs after the host is created but before it starts, so callers can register actors with host.RegisterActor/host.RegisterBuiltInActor (must be called before the host is running) +// The host is stopped when the test ends // The in-memory provider keeps no state on disk, so the test never touches a real database -func NewActorHostForTest(t *testing.T, opts ...local.HostOption) *local.Host { +func NewActorHostForTest(t *testing.T, register func(t *testing.T, h *local.Host)) *local.Host { t.Helper() - // The defaults come first so callers can register their built-in actors (and override any default) through opts - hostOpts := append( - []local.HostOption{ - local.WithAddress(freeLoopbackAddr(t)), - local.WithRuntimePSKs([]byte(testActorHostPSK)), - local.WithStandaloneMemoryProvider(standalone.StandaloneMemoryOptions{}), - local.WithShutdownGracePeriod(time.Second), - }, - opts..., - ) + hostOpts := []local.HostOption{ + local.WithAddress(freeLoopbackAddr(t)), + local.WithRuntimePSKs([]byte(testActorHostPSK)), + local.WithStandaloneMemoryProvider(standalone.StandaloneMemoryOptions{}), + local.WithShutdownGracePeriod(time.Second), + } h, err := local.NewHost(hostOpts...) require.NoError(t, err) + // Register built-in actors before the host starts + if register != nil { + register(t, h) + } + // Run the host in the background and stop it when the test ends ctx, cancel := context.WithCancel(context.Background()) errCh := make(chan error, 1) diff --git a/backend/internal/utils/testing/database.go b/backend/internal/utils/testing/database.go index 1842c067..02b2bc2e 100644 --- a/backend/internal/utils/testing/database.go +++ b/backend/internal/utils/testing/database.go @@ -1,8 +1,13 @@ +//go:build unit + // This file is only imported by unit tests package testing import ( + "errors" + "log/slog" + "path/filepath" "testing" "time" @@ -11,6 +16,7 @@ import ( "github.com/golang-migrate/migrate/v4" sqliteMigrate "github.com/golang-migrate/migrate/v4/database/sqlite3" "github.com/golang-migrate/migrate/v4/source/iofs" + sqlitekit "github.com/italypaleale/go-sql-utils/sqlite" "github.com/libtnb/sqlite" "github.com/stretchr/testify/require" "gorm.io/gorm" @@ -28,28 +34,43 @@ func init() { // NewDatabaseForTest returns a new instance of GORM connected to an in-memory SQLite database. // Each database connection is unique for the test. // All migrations are automatically performed. +// Note: the in-memory database is limited to a single connection, so it cannot be used to test concurrent access: use NewConcurrentDatabaseForTest for that. func NewDatabaseForTest(t *testing.T) *gorm.DB { t.Helper() + db := openInMemoryTestDB(t) + runMigrations(t, db, 0, nil) + return db +} + +// NewDatabaseForTestWithMigrationSeed behaves like NewDatabaseForTest, but pauses the migrations right after the migration whose version is stopAfterVersion has been applied. +// It then invokes seed, so the test can insert data into the intermediate schema, before applying the remaining migrations. +// This is meant to test data migrations, which operate on data that already exists in the database. +func NewDatabaseForTestWithMigrationSeed(t *testing.T, stopAfterVersion uint, seed func(t *testing.T, db *gorm.DB)) *gorm.DB { + t.Helper() + db := openInMemoryTestDB(t) + runMigrations(t, db, stopAfterVersion, seed) + return db +} + +// NewConcurrentDatabaseForTest returns a new instance of GORM connected to a temporary file-based SQLite database +// Unlike NewDatabaseForTest, which forces a single connection to an in-memory database, this one can be used to test concurrent access to the database. +// All migrations are automatically performed. +func NewConcurrentDatabaseForTest(t *testing.T) *gorm.DB { + t.Helper() + db := openFileTestDB(t) + runMigrations(t, db, 0, nil) + return db +} + +// openInMemoryTestDB opens a GORM instance backed by an in-memory SQLite database, unique to the test. +func openInMemoryTestDB(t *testing.T) *gorm.DB { + t.Helper() // Get a name for this in-memory database that is specific to the test dbName := utils.CreateSha256Hash(t.Name()) // Connect to a new in-memory SQL database - db, err := gorm.Open( - sqlite.Open("file:"+dbName+"?mode=memory"), - &gorm.Config{ - TranslateError: true, - Logger: logger.New( - testLoggerAdapter{t: t}, - logger.Config{ - SlowThreshold: 200 * time.Millisecond, - LogLevel: logger.Info, - IgnoreRecordNotFoundError: false, - ParameterizedQueries: false, - Colorful: false, - }, - ), - }) + db, err := gorm.Open(sqlite.Open("file:"+dbName+"?mode=memory"), newTestGormConfig(t)) require.NoError(t, err, "Failed to connect to test database") sqlDB, err := db.DB() @@ -59,6 +80,31 @@ func NewDatabaseForTest(t *testing.T) *gorm.DB { // The other workaround, of using shared caches, doesn't work well with multiple write transactions trying to happen at once sqlDB.SetMaxOpenConns(1) + return db +} + +// openFileTestDB opens a GORM instance backed by a temporary file-based SQLite database, configured like the production database +func openFileTestDB(t *testing.T) *gorm.DB { + t.Helper() + + dbPath := filepath.Join(t.TempDir(), "test.db") + connString, _, _, err := sqlitekit.ParseConnectionString(dbPath, slog.Default()) + require.NoError(t, err, "Failed to parse connection string") + + db, err := gorm.Open(sqlite.Open(connString), newTestGormConfig(t)) + require.NoError(t, err, "Failed to connect to test database") + + return db +} + +// runMigrations applies the embedded SQLite migrations to db. +// If seed is not nil, migrations are first applied up to and including stopAfterVersion, then seed is invoked so the test can insert data into the intermediate schema, before the remaining migrations are applied. +func runMigrations(t *testing.T, db *gorm.DB, stopAfterVersion uint, seed func(t *testing.T, db *gorm.DB)) { + t.Helper() + + sqlDB, err := db.DB() + require.NoError(t, err, "Failed to get sql.DB") + // Perform migrations with the embedded migrations driver, err := sqliteMigrate.WithInstance(sqlDB, &sqliteMigrate.Config{ NoTxWrap: true, @@ -68,12 +114,40 @@ func NewDatabaseForTest(t *testing.T) *gorm.DB { require.NoError(t, err, "Failed to create embedded migration source") m, err := migrate.NewWithInstance("iofs", source, "pocket-id", driver) require.NoError(t, err, "Failed to create migration instance") + + // If the test wants to seed data partway through the migrations, apply migrations up to and including stopAfterVersion first, then let it seed + if seed != nil { + err = m.Migrate(stopAfterVersion) + require.NoErrorf(t, err, "Failed to perform migrations up to version %d", stopAfterVersion) + seed(t, db) + } + + // Apply all the remaining migrations + // ErrNoChange means we were already at the latest version, which is not an error here err = m.Up() - require.NoError(t, err, "Failed to perform migrations") + if !errors.Is(err, migrate.ErrNoChange) { + require.NoError(t, err, "Failed to perform migrations") + } _, err = sqlDB.Exec("PRAGMA foreign_keys = OFF;") require.NoError(t, err, "Failed to disable foreign keys") +} - return db +// newTestGormConfig returns the GORM configuration shared by the test databases, wiring the logger to the test's output. +func newTestGormConfig(t *testing.T) *gorm.Config { + t.Helper() + return &gorm.Config{ + TranslateError: true, + Logger: logger.New( + testLoggerAdapter{t: t}, + logger.Config{ + SlowThreshold: 200 * time.Millisecond, + LogLevel: logger.Info, + IgnoreRecordNotFoundError: false, + ParameterizedQueries: false, + Colorful: false, + }, + ), + } } // Implements gorm's logger.Writer interface diff --git a/backend/internal/utils/testing/round_tripper.go b/backend/internal/utils/testing/round_tripper.go index d8b758e4..54cf1ec4 100644 --- a/backend/internal/utils/testing/round_tripper.go +++ b/backend/internal/utils/testing/round_tripper.go @@ -1,3 +1,5 @@ +//go:build unit + // This file is only imported by unit tests package testing diff --git a/backend/resources/migrations/postgres/20260708130000_move_instance_id_to_kv.down.sql b/backend/resources/migrations/postgres/20260708130000_move_instance_id_to_kv.down.sql new file mode 100644 index 00000000..a57bbe4b --- /dev/null +++ b/backend/resources/migrations/postgres/20260708130000_move_instance_id_to_kv.down.sql @@ -0,0 +1,8 @@ +-- Move the instance ID back into the standalone app config table +INSERT INTO app_config_variables ("key", "value") +SELECT 'instanceId', "value" + FROM kv + WHERE "key" = 'instance_id' AND "value" IS NOT NULL + ON CONFLICT ("key") DO NOTHING; + +DELETE FROM kv WHERE "key" = 'instance_id'; diff --git a/backend/resources/migrations/postgres/20260708130000_move_instance_id_to_kv.up.sql b/backend/resources/migrations/postgres/20260708130000_move_instance_id_to_kv.up.sql new file mode 100644 index 00000000..69243c04 --- /dev/null +++ b/backend/resources/migrations/postgres/20260708130000_move_instance_id_to_kv.up.sql @@ -0,0 +1,8 @@ +-- Move the instance ID out of the standalone app config table and into the "kv" table +INSERT INTO kv ("key", "value") +SELECT 'instance_id', "value" + FROM app_config_variables + WHERE "key" = 'instanceId' + ON CONFLICT ("key") DO NOTHING; + +DELETE FROM app_config_variables WHERE "key" = 'instanceId'; diff --git a/backend/resources/migrations/sqlite/20260708130000_move_instance_id_to_kv.down.sql b/backend/resources/migrations/sqlite/20260708130000_move_instance_id_to_kv.down.sql new file mode 100644 index 00000000..b0cc3293 --- /dev/null +++ b/backend/resources/migrations/sqlite/20260708130000_move_instance_id_to_kv.down.sql @@ -0,0 +1,14 @@ +PRAGMA foreign_keys=OFF; +BEGIN; + +-- Move the instance ID back into the standalone app config table +INSERT INTO app_config_variables ("key", "value") +SELECT 'instanceId', "value" + FROM kv + WHERE "key" = 'instance_id' AND "value" IS NOT NULL + ON CONFLICT ("key") DO NOTHING; + +DELETE FROM kv WHERE "key" = 'instance_id'; + +COMMIT; +PRAGMA foreign_keys=ON; diff --git a/backend/resources/migrations/sqlite/20260708130000_move_instance_id_to_kv.up.sql b/backend/resources/migrations/sqlite/20260708130000_move_instance_id_to_kv.up.sql new file mode 100644 index 00000000..0ab233e9 --- /dev/null +++ b/backend/resources/migrations/sqlite/20260708130000_move_instance_id_to_kv.up.sql @@ -0,0 +1,14 @@ +PRAGMA foreign_keys=OFF; +BEGIN; + +-- Move the instance ID out of the standalone app config table and into the "kv" table +INSERT INTO kv ("key", "value") +SELECT 'instance_id', "value" + FROM app_config_variables + WHERE "key" = 'instanceId' + ON CONFLICT ("key") DO NOTHING; + +DELETE FROM app_config_variables WHERE "key" = 'instanceId'; + +COMMIT; +PRAGMA foreign_keys=ON; diff --git a/tests/resources/export/database.json b/tests/resources/export/database.json index aeb7ba51..37ebf554 100644 --- a/tests/resources/export/database.json +++ b/tests/resources/export/database.json @@ -1,6 +1,6 @@ { "provider": "sqlite", - "version": 20260708120000, + "version": 20260708130000, "tableOrder": ["users", "user_groups", "oidc_clients", "signup_tokens", "apis", "api_permissions", "oidc_clients_allowed_api_permissions"], "tables": { "apis": [ @@ -66,13 +66,11 @@ "user_id": "f4b89dc2-62fb-46bf-9f5f-c34f4eafe93e" } ], - "app_config_variables": [ - { - "key": "instanceId", - "value": "test-instance-id" - } - ], "kv": [ + { + "key": "instance_id", + "value": "test-instance-id" + }, { "key": "jwt_private_key.json", "value": "7d/5hl7diJ2rnFL14hEAQf9tzpu29aqXQ8jpJ2iqqKUNFZpdOkEpud0CmRv4H3r8yyk2u/Gqqj9klSy58DJkYXGF5PAYgLyoBIb7L3JXWRbxg4cQ3QJCug13l2OTmpAKoVc+rmX8c3j3h1sNqyJ+7Ql5sS0jSeyiYgIsFNCdnK5alBDyvtcpe/QDpklmP4JCeVpvmf2rLGplk3g5UO5ydJ8UiDXxfDmi+gF6NKJvrGnnah8Ar3G/x88z+tTJtp0DIQFwxXwUM2XZqzEVGm8K2r0w5o9/Keh6bBBaiuH2C78ZOaijGV3DovhR+e9J0cYUYGwT42MZMx9fSWQ/lvWGGnf+Uq3MXJfjWSREfhkp8KTQwR9F7+dnVJWswOEk7jPR8I7hCWTMxJyvaFX3wgAXIVmhrgXZQQbYOqTt56IoqUl0xOJku8dA8opg2UcLlmmuOh6+hfkXKsiiS/H/9c1BVIGj1fCOiT6IePh4wKKSTbwJnPD5EKmdJpgTsUpjcDnXQKY4ReO0UpdRdKxwRDDLeQuG6j+ljGxR9GPudCU9Nmci6rFVI6n5LWYkQxBA1O73RpmXRZPDzntDfpXMEonkmSvOoxaCK2Id7CRKMdqvR0kEouwnhk5WSFtsfi3sA0pkXzPFxwZeWM8vFtbffZOZzXaOhxCOfcj1NClZohlZhyc4jvkxmrpY7PSaAzih0AmHI7y0LYFi6fZu/K4EheVa1+KF55nWZ8ARikHMWKAKkyExkTak7xyN884TDmzURRaPlQg4jzQte5WMNjAG/hlHibdMBNvgwiYd49ZxteJ8ABdbiXVRl+2JGbdjl2ubpQZwOn7bJKlqO56bIwsZ+e4+pXsuOGdBahkHrUjtMEmH3DZbGc6CJLbcmdhdpApLQRRcLAazxJhzAwJ47FRYsHsj57LnYNvmcKdIxw8rxCdLUuzz95uw0T3ankEO5J9sjem+HMEuKdwXK1UcuOn2rjR8Sd/BuvQmeso27dFbPXqXYNS90Ml45YyTvcKSiopD181oZR703TFUSpR7dsiqROMr+p/2jN9h6a8WbQ8xpksyclaQByY/M77AssbXnG6wfhRsntNIINCZLbBnjXOyz6ZHIC5K4tSTdcnWaiYPeRPQmnw9UUvHAcNU2yMWsy0eU377yDS0WstTxOdQutTdkczl8kv5Lo26JiEK7mSIuRK19ffF9Zz8FG8+eKv5zdyIPjyQRDYBysUoDv5huKe2eoxJu/MWS2Pql/ZtUGeD6Ozm3mCvh0vQ9ceagBkY6Ocm3du0ziAKP29Ri0mjg4DizVorbLzsh+EQH/s2Pi9MnjUZDlEmuLl2Xfp7/w4j/8u0N0tVR70VDFuGdKpTjFY3vS8EJrPtyMTM51x1D9rb8gIql8aR/rJw4YF+huxg1mv5n6+tGVqg5msbPmF12eJijP4lkmaRwIpLW5pJTtaDkUj7uOeu1mm4k+Dt5nh0/0jPHzrv6bcTCcbV7UjMHDoTXXqEpFAAJ66rHR7zdAJu+YKsnTIZyLmOpcowq7LL8G9qTvV0OSpyQWUIavRSgbDHFqEqRs+JU94jAzkq8nCY5MTd9m5sIv9InfdT3k+pwpsE/FKge8nghFLtbUrafGkzTky8SE2druvVcIvbfXMfLIKRUYjJgnWc0gQzF5J6pzXM7D2r/RG6JDzASqjlbURq6v9bhNerlOVdMujWKEEVcKWIzlbt4RkihRjM8AUqIZQOyicGQ+4yfIjAHw5viuABONYs3OIWULnFqJxdvS9rNKhfxSjIq9cfqyzevq2xrRoMXEonobh6M3bD2Vang8OAeVeD1OXWPERi4pepCYFS9RJ/Xa/UWxptsqSNuGcb3fAzQSmLpXLGdWRoKXvSe7EYgc0bGcLOjSTu5RURKo+EF9i4KT9EJauf6VXw5dTf/CCIJRXE1bWzXhSCFYntohYhX2ldOCDYpi/jFBC6Vtkw0ud3/xq8Nmhd5gUk+SpngByCZH3Pm3H+jvlbMpiqkDkm1v74hDX13Xhrcw2eWyuqKBVoRCCniUvwpYNbGvBfjC6Hcizv0Aybciwj+4nybt5EPoEUm6S6Gs7fG7QpPdvrzpAxX70MlmdkF/gwyuhbEeJhLK+WL7qAsN5CvHPzVbsIf90x+nGTtMJPgpxVr0tJMj+vprXV4WxutfARBiOnqe58MhA857sd+MzKBgKnoLOBRTiC3qc/0/ULwbG2HCCD7nmwzz7M4nUuMvo8rgS7z0BF68OClT8X3JwSXbL5Wg=="