From e01ad4c196679b2b6647cb35545676bedd3daf84 Mon Sep 17 00:00:00 2001 From: Khaled FERJANI Date: Mon, 13 Apr 2026 13:20:49 +0200 Subject: [PATCH 1/2] feat(nextcloud): trigger endpoint for bulk migrations Twake users migrating from Nextcloud need a way to import their entire file tree into their Cozy. Konnectors and stack workers are bounded by job timeouts, so the heavy lifting has to live outside the stack. This endpoint is the user-facing entry point: it validates the credentials synchronously, persists the account, creates a tracking document, and hands the work off to a separate migration service via RabbitMQ. The service drives the existing /remote/nextcloud/:account/* routes and updates the tracking document, which the Settings UI watches over realtime. --- docs/nextcloud.md | 88 ++++ docs/rabbitmq.md | 40 ++ model/nextcloud/migration.go | 151 +++++++ model/nextcloud/nextcloud.go | 38 +- pkg/consts/doctype.go | 3 + pkg/couchdb/index.go | 6 +- pkg/rabbitmq/contracts.go | 24 +- web/remote/app_token_verification_test.go | 114 ++++++ web/remote/migration.go | 237 +++++++++++ web/remote/migration_test.go | 471 ++++++++++++++++++++++ web/routing.go | 2 +- 11 files changed, 1163 insertions(+), 11 deletions(-) create mode 100644 model/nextcloud/migration.go create mode 100644 web/remote/app_token_verification_test.go create mode 100644 web/remote/migration.go create mode 100644 web/remote/migration_test.go diff --git a/docs/nextcloud.md b/docs/nextcloud.md index 307d5b126ef..436470a8daf 100644 --- a/docs/nextcloud.md +++ b/docs/nextcloud.md @@ -579,3 +579,91 @@ Authorization: Bearer eyJhbG... ```http HTTP/1.1 204 No Content ``` + +## POST /remote/nextcloud/migration + +This route triggers a one-shot bulk migration of a user's Nextcloud files into +their Cozy. The Stack validates the credentials, persists an +`io.cozy.accounts` document, creates an `io.cozy.nextcloud.migrations` +tracking document in `pending` state, and publishes a +`nextcloud.migration.requested` command to the `migration` RabbitMQ exchange. +The actual transfer is performed by an external migration service that +consumes the command and drives the existing `/remote/nextcloud/:account/*` +routes, updating the tracking document as it progresses. + +Before persisting anything, the Stack probes the supplied credentials against +the Nextcloud instance via the OCS `user_status` endpoint, so wrong passwords +and unreachable hosts surface synchronously instead of being deferred to the +migration service. The probe also resolves the WebDAV user ID, which is +cached on the account document so the migration service does not need to +re-fetch it. + +When an existing `io.cozy.accounts` document for the same `account_type: +"nextcloud"` + `auth.url` + `auth.login` triplet is found, it is reused with +its stored password and `webdav_user_id` refreshed from the request. Only one +migration can be in flight per instance at a time: if a `pending` or `running` +tracking document already exists, the Stack returns `409 Conflict`. Failed +migrations do not block new attempts. + +**Note:** a permission on `POST io.cozy.nextcloud.migrations` is required to +use this route. + +### Request + +```http +POST /remote/nextcloud/migration HTTP/1.1 +Host: cozy.example.net +Authorization: Bearer eyJhbG... +Content-Type: application/json +``` + +```json +{ + "nextcloud_url": "https://nextcloud.example.com", + "nextcloud_login": "alice", + "nextcloud_app_password": "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx", + "source_path": "/" +} +``` + +`source_path` is optional and defaults to `/`. The `nextcloud_app_password` +should be a Nextcloud app password, not the user's main account password. + +### Response + +```http +HTTP/1.1 201 Created +Content-Type: application/vnd.api+json +``` + +```json +{ + "data": { + "id": "d4e5f6a7b8c94d0ea1b2c3d4e5f6a7b8", + "type": "io.cozy.nextcloud.migrations", + "attributes": { + "status": "pending", + "target_dir": "/Nextcloud", + "progress": { + "files_imported": 0, + "files_total": 0, + "bytes_imported": 0, + "bytes_total": 0 + }, + "errors": [], + "skipped": [], + "started_at": null, + "finished_at": null + } + } +} +``` + +#### Status codes + +- 201 Created, when the migration has been queued and the tracking document is returned +- 401 Unauthorized, when the Nextcloud credentials are rejected by the remote host +- 409 Conflict, when a `pending` or `running` migration already exists +- 500 Internal Server Error, when the conflict check, account upsert, or tracking document creation fails +- 502 Bad Gateway, when the Nextcloud instance is unreachable +- 503 Service Unavailable, when the migration command cannot be published to RabbitMQ. The tracking document is marked `failed` before returning diff --git a/docs/rabbitmq.md b/docs/rabbitmq.md index 1b5298bd330..82fe21cbb0e 100644 --- a/docs/rabbitmq.md +++ b/docs/rabbitmq.md @@ -134,6 +134,46 @@ rabbitmq: - If queue-level `dlx_name`/`dlq_name` are not specified, exchange-level defaults are used. - Messages that exceed the `delivery_limit` or are rejected will be sent to the DLX and routed to the DLQ. +### Publishers + +The Stack also publishes messages to RabbitMQ. Publishers do not declare any +queue or exchange on the Stack side: the exchange must already exist on the +broker, and a queue must be bound by the consuming service. Publishes use the +AMQP `mandatory` flag, so a publish to an exchange with no matching binding +fails with `PublishReturnedError` and the caller is expected to surface the +failure to the user. + +#### `auth` exchange + +Routing key: `user.deletion.requested`. Published from +`POST /settings/instance/deletion/force` when a user requests permanent +deletion of their account. The payload is the `UserDeletionRequestedMessage` +struct in `pkg/rabbitmq/contracts.go`. + +#### `migration` exchange + +Routing key: `nextcloud.migration.requested`. Published from +`POST /remote/nextcloud/migration` when a user starts a Nextcloud-to-Cozy +bulk migration. The payload is the `NextcloudMigrationRequestedMessage` +struct in `pkg/rabbitmq/contracts.go`: + +```json +{ + "migrationId": "d4e5f6a7b8c94d0ea1b2c3d4e5f6a7b8", + "workplaceFqdn": "alice.cozy.example.com", + "accountId": "a1b2c3d4e5f6", + "sourcePath": "/", + "timestamp": 1712563200 +} +``` + +Credentials are never in the payload: they live in the `io.cozy.accounts` +document referenced by `accountId`. The Stack populates `MessageID` with the +migration ID for cross-system tracing. The consuming service is responsible +for declaring its queue, binding it to this exchange, and processing the +messages; if no queue is bound when the Stack publishes, the user receives +`503` and the tracking document is marked `failed`. + ### Handlers Handlers implement a simple interface: diff --git a/model/nextcloud/migration.go b/model/nextcloud/migration.go new file mode 100644 index 00000000000..e8f00a3e437 --- /dev/null +++ b/model/nextcloud/migration.go @@ -0,0 +1,151 @@ +package nextcloud + +import ( + "errors" + "time" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/couchdb/mango" + "github.com/cozy/cozy-stack/pkg/jsonapi" +) + +const ( + MigrationStatusPending = "pending" + MigrationStatusRunning = "running" + MigrationStatusCompleted = "completed" + MigrationStatusFailed = "failed" +) + +const DefaultMigrationTargetDir = "/Nextcloud" + +var ErrMigrationConflict = errors.New("a nextcloud migration is already in progress") + +// Migration is the io.cozy.nextcloud.migrations tracking document. +// +// The schema (especially the nested Progress object) is the contract with +// twake-migration-nextcloud. Flat counters would crash the service's progress +// reducer because it spreads doc.progress and adds to its fields. +type Migration struct { + DocID string `json:"_id,omitempty"` + DocRev string `json:"_rev,omitempty"` + Status string `json:"status"` + TargetDir string `json:"target_dir"` + Progress MigrationProgress `json:"progress"` + Errors []MigrationError `json:"errors"` + Skipped []SkippedFile `json:"skipped"` + StartedAt *time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at"` +} + +type MigrationProgress struct { + FilesImported int64 `json:"files_imported"` + FilesTotal int64 `json:"files_total"` + BytesImported int64 `json:"bytes_imported"` + BytesTotal int64 `json:"bytes_total"` +} + +type MigrationError struct { + Path string `json:"path"` + Message string `json:"message"` + At time.Time `json:"at"` +} + +type SkippedFile struct { + Path string `json:"path"` + Reason string `json:"reason"` + Size int64 `json:"size"` +} + +func (m *Migration) ID() string { return m.DocID } +func (m *Migration) Rev() string { return m.DocRev } +func (m *Migration) DocType() string { return consts.NextcloudMigrations } +func (m *Migration) SetID(id string) { m.DocID = id } +func (m *Migration) SetRev(rev string) { m.DocRev = rev } + +func (m *Migration) Clone() couchdb.Doc { + cloned := *m + + if m.Errors != nil { + cloned.Errors = make([]MigrationError, len(m.Errors)) + copy(cloned.Errors, m.Errors) + } + if m.Skipped != nil { + cloned.Skipped = make([]SkippedFile, len(m.Skipped)) + copy(cloned.Skipped, m.Skipped) + } + if m.StartedAt != nil { + t := *m.StartedAt + cloned.StartedAt = &t + } + if m.FinishedAt != nil { + t := *m.FinishedAt + cloned.FinishedAt = &t + } + return &cloned +} + +func (m *Migration) Links() *jsonapi.LinksList { return nil } +func (m *Migration) Relationships() jsonapi.RelationshipMap { return nil } +func (m *Migration) Included() []jsonapi.Object { return nil } + +var ( + _ couchdb.Doc = (*Migration)(nil) + _ jsonapi.Object = (*Migration)(nil) +) + +// NewPendingMigration returns a fresh Migration document in the pending state. +// Errors and Skipped are explicit empty slices so the JSON serialization +// produces "[]" rather than "null" — the migration service consumes them as +// arrays and would crash on null. +func NewPendingMigration(targetDir string) *Migration { + if targetDir == "" { + targetDir = DefaultMigrationTargetDir + } + return &Migration{ + Status: MigrationStatusPending, + TargetDir: targetDir, + Errors: []MigrationError{}, + Skipped: []SkippedFile{}, + } +} + +func (m *Migration) MarkFailed(inst *instance.Instance, cause error) error { + now := time.Now().UTC() + m.Status = MigrationStatusFailed + if m.FinishedAt == nil { + m.FinishedAt = &now + } + m.Errors = append(m.Errors, MigrationError{ + Message: cause.Error(), + At: now, + }) + return couchdb.UpdateDoc(inst, m) +} + +// FindActiveMigration returns the first pending or running migration, or +// (nil, nil) if none. A missing doctype database or index is treated as "no +// active migration" so the first call on a fresh instance succeeds. +func FindActiveMigration(inst *instance.Instance) (*Migration, error) { + var docs []*Migration + req := &couchdb.FindRequest{ + UseIndex: "by-status", + Selector: mango.In("status", []interface{}{ + MigrationStatusPending, + MigrationStatusRunning, + }), + Limit: 1, + } + err := couchdb.FindDocs(inst, consts.NextcloudMigrations, req, &docs) + if err != nil { + if couchdb.IsNoDatabaseError(err) || couchdb.IsNoUsableIndexError(err) { + return nil, nil + } + return nil, err + } + if len(docs) == 0 { + return nil, nil + } + return docs[0], nil +} diff --git a/model/nextcloud/nextcloud.go b/model/nextcloud/nextcloud.go index 5c4bef0a2aa..e7a981ae193 100644 --- a/model/nextcloud/nextcloud.go +++ b/model/nextcloud/nextcloud.go @@ -3,6 +3,7 @@ package nextcloud import ( + "context" "encoding/json" "io" "net/http" @@ -20,6 +21,7 @@ import ( "github.com/cozy/cozy-stack/pkg/consts" "github.com/cozy/cozy-stack/pkg/couchdb" "github.com/cozy/cozy-stack/pkg/jsonapi" + "github.com/cozy/cozy-stack/pkg/logger" "github.com/cozy/cozy-stack/pkg/safehttp" "github.com/cozy/cozy-stack/pkg/webdav" "github.com/labstack/echo/v4" @@ -360,16 +362,36 @@ func (nc *NextCloud) buildTrashedURL(item webdav.Item, path string) string { return u.String() } +// FetchUserIDWithCredentials probes the OCS user_status endpoint and returns +// the user ID, or webdav.ErrInvalidAuth if the credentials are rejected. +// // https://docs.nextcloud.com/server/latest/developer_manual/client_apis/OCS/ocs-status-api.html#fetch-your-own-status -func (nc *NextCloud) fetchUserID() (string, error) { - logger := nc.webdav.Logger +func FetchUserIDWithCredentials(nextcloudURL, username, password string, logger *logger.Entry) (string, error) { + u, err := url.Parse(nextcloudURL) + if err != nil { + return "", err + } + if u.Scheme == "" || u.Host == "" { + return "", ErrInvalidAccount + } + return fetchUserIDFromHost(u.Scheme, u.Host, username, password, logger) +} + +const userStatusProbeTimeout = 30 * time.Second + +func fetchUserIDFromHost(scheme, host, username, password string, logger *logger.Entry) (string, error) { u := url.URL{ - Scheme: nc.webdav.Scheme, - Host: nc.webdav.Host, - User: url.UserPassword(nc.webdav.Username, nc.webdav.Password), + Scheme: scheme, + Host: host, + User: url.UserPassword(username, password), Path: "/ocs/v2.php/apps/user_status/api/v1/user_status", } - req, err := http.NewRequest(http.MethodGet, u.String(), nil) + // Cap the probe so a hung Nextcloud server can't pin a request goroutine — + // safehttp.ClientWithKeepAlive has handshake timeouts but no overall + // request deadline. + ctx, cancel := context.WithTimeout(context.Background(), userStatusProbeTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return "", err } @@ -396,6 +418,10 @@ func (nc *NextCloud) fetchUserID() (string, error) { return payload.OCS.Data.UserID, nil } +func (nc *NextCloud) fetchUserID() (string, error) { + return fetchUserIDFromHost(nc.webdav.Scheme, nc.webdav.Host, nc.webdav.Username, nc.webdav.Password, nc.webdav.Logger) +} + type OCSPayload struct { OCS struct { Data struct { diff --git a/pkg/consts/doctype.go b/pkg/consts/doctype.go index 9f73ce2bc3a..a9665ae5246 100644 --- a/pkg/consts/doctype.go +++ b/pkg/consts/doctype.go @@ -142,6 +142,9 @@ const ( // NextCloudFiles doc type is used when listing files from a NextCloud via // WebDAV. NextCloudFiles = "io.cozy.remote.nextcloud.files" + // NextcloudMigrations doc type is used to track bulk Nextcloud to Cozy + // migrations orchestrated by the external migration service. + NextcloudMigrations = "io.cozy.nextcloud.migrations" // ChatAssistants doc type for AI chat assistants. ChatAssistants = "io.cozy.ai.chat.assistants" // ChatConversations doc type is used for a chat between the user and a chatbot. diff --git a/pkg/couchdb/index.go b/pkg/couchdb/index.go index a3e73f9324c..9a1cfd292bf 100644 --- a/pkg/couchdb/index.go +++ b/pkg/couchdb/index.go @@ -14,7 +14,7 @@ import ( // IndexViewsVersion is the version of current definition of views & indexes. // This number should be incremented when this file changes. -const IndexViewsVersion int = 37 +const IndexViewsVersion int = 38 // Indexes is the index list required by an instance to run properly. var Indexes = []*mango.Index{ @@ -72,6 +72,10 @@ var Indexes = []*mango.Index{ // Used to find the active sharings mango.MakeIndex(consts.Sharings, "active", mango.IndexDef{Fields: []string{"active"}}), + + // Used to detect an already in-flight Nextcloud migration when a user + // tries to start a new one. + mango.MakeIndex(consts.NextcloudMigrations, "by-status", mango.IndexDef{Fields: []string{"status"}}), } // DiskUsageView is the view used for computing the disk usage for files diff --git a/pkg/rabbitmq/contracts.go b/pkg/rabbitmq/contracts.go index aec2a173c7f..11519b5c348 100644 --- a/pkg/rabbitmq/contracts.go +++ b/pkg/rabbitmq/contracts.go @@ -1,7 +1,8 @@ package rabbitmq const ( - ExchangeAuth = "auth" + ExchangeAuth = "auth" + ExchangeMigration = "migration" ) const ( @@ -15,8 +16,9 @@ const ( ) const ( - RoutingKeyUserPasswordUpdated = "user.password.updated" - RoutingKeyUserDeletionRequested = "user.deletion.requested" + RoutingKeyUserPasswordUpdated = "user.password.updated" + RoutingKeyUserDeletionRequested = "user.deletion.requested" + RoutingKeyNextcloudMigrationRequested = "nextcloud.migration.requested" ) // UserDeletionRequestedMessage is published when a user asks Twake to delete the account linked to the current cozy instance. @@ -26,3 +28,19 @@ type UserDeletionRequestedMessage struct { RequestedBy string `json:"requestedBy"` RequestedAt int64 `json:"requestedAt"` } + +// NextcloudMigrationRequestedMessage is published when a user starts a +// Nextcloud to Cozy migration from the Settings UI. The external migration +// service consumes it, fetches an app audience token from the Cloudery, and +// orchestrates the transfer through the Stack's Nextcloud routes. +// +// Credentials for the Nextcloud account are stored in the io.cozy.accounts +// document referenced by AccountID. They MUST NOT be included in this +// message: the broker is not a trust boundary for secrets. +type NextcloudMigrationRequestedMessage struct { + MigrationID string `json:"migrationId"` + WorkplaceFqdn string `json:"workplaceFqdn"` + AccountID string `json:"accountId"` + SourcePath string `json:"sourcePath,omitempty"` + Timestamp int64 `json:"timestamp"` +} diff --git a/web/remote/app_token_verification_test.go b/web/remote/app_token_verification_test.go new file mode 100644 index 00000000000..d67184c6fb3 --- /dev/null +++ b/web/remote/app_token_verification_test.go @@ -0,0 +1,114 @@ +package remote + +import ( + "encoding/xml" + "net/http" + "net/http/httptest" + "testing" + + "github.com/cozy/cozy-stack/model/account" + "github.com/cozy/cozy-stack/model/permission" + build "github.com/cozy/cozy-stack/pkg/config" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/tests/testutils" + weberrors "github.com/cozy/cozy-stack/web/errors" + "github.com/gavv/httpexpect/v2" + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" +) + +// TestAppAudienceTokenPassesNextcloudPermissionCheck guards that an app +// audience token whose webapp permission doc holds io.cozy.files can reach +// GET /remote/nextcloud/:account/ without being rejected by the permission +// middleware. +func TestAppAudienceTokenPassesNextcloudPermissionCheck(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + testutils.NeedCouchdb(t) + + oldBuildMode := build.BuildMode + build.BuildMode = build.ModeDev + t.Cleanup(func() { build.BuildMode = oldBuildMode }) + + setup := testutils.NewSetup(t, t.Name()) + testInstance := setup.GetTestInstance() + + rules := permission.Set{ + permission.Rule{Type: consts.Files, Verbs: permission.ALL}, + permission.Rule{Type: consts.NextcloudMigrations, Verbs: permission.ALL}, + } + permReq := permission.Permission{ + Permissions: rules, + Type: permission.TypeWebapp, + SourceID: consts.Apps + "/migrator", + } + require.NoError(t, couchdb.CreateDoc(testInstance, &permReq)) + manifest := &couchdb.JSONDoc{ + Type: consts.Apps, + M: map[string]interface{}{ + "_id": consts.Apps + "/migrator", + "slug": "migrator", + "permissions": rules, + }, + } + require.NoError(t, couchdb.CreateNamedDocWithDB(testInstance, manifest)) + token := testInstance.BuildAppToken("migrator", "") + require.NotEmpty(t, token) + + mockWebDAV := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ocs/v2.php/apps/user_status/api/v1/user_status" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ocs":{"data":{"userId":"migrator"}}}`)) + return + } + if r.Method == "PROPFIND" { + body, _ := xml.Marshal(struct { + XMLName xml.Name `xml:"d:multistatus"` + Xmlns string `xml:"xmlns:d,attr"` + }{Xmlns: "DAV:"}) + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + w.WriteHeader(http.StatusMultiStatus) + _, _ = w.Write(body) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(mockWebDAV.Close) + + accountDoc := &couchdb.JSONDoc{ + Type: consts.Accounts, + M: map[string]interface{}{ + "account_type": "nextcloud", + "name": "Migration Source", + "auth": map[string]interface{}{ + "login": "migrator", + "password": "secret", + "url": mockWebDAV.URL + "/", + }, + "webdav_user_id": "migrator", + }, + } + account.Encrypt(*accountDoc) + require.NoError(t, couchdb.CreateDoc(testInstance, accountDoc)) + accountID := accountDoc.ID() + + ts := setup.GetTestServer("/remote", Routes) + ts.Config.Handler.(*echo.Echo).HTTPErrorHandler = weberrors.ErrorHandler + t.Cleanup(ts.Close) + + e := testutils.CreateTestClient(t, ts.URL) + + e.GET("/remote/nextcloud/"+accountID+"/"). + WithHeader("Authorization", "Bearer "+token). + WithHost(testInstance.Domain). + Expect().Status(http.StatusOK). + JSON(httpexpect.ContentOpts{MediaType: "application/vnd.api+json"}). + Object(). + Value("data").Array() +} diff --git a/web/remote/migration.go b/web/remote/migration.go new file mode 100644 index 00000000000..0ef4d6b8061 --- /dev/null +++ b/web/remote/migration.go @@ -0,0 +1,237 @@ +package remote + +import ( + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/cozy/cozy-stack/model/account" + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/nextcloud" + "github.com/cozy/cozy-stack/model/permission" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/jsonapi" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/rabbitmq" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/cozy/cozy-stack/pkg/webdav" + "github.com/cozy/cozy-stack/web/middlewares" + "github.com/labstack/echo/v4" +) + +const nextcloudMigrationLogNamespace = "nextcloud-migration" + +// HTTPHandler owns the remote routes that need stack services. The +// package-level [Routes] function is kept for handlers that don't depend on +// any service; the migration endpoint lives on this struct because it +// publishes commands to the migration broker. +type HTTPHandler struct { + rmq rabbitmq.Service +} + +func NewHTTPHandler(rmq rabbitmq.Service) *HTTPHandler { + return &HTTPHandler{rmq: rmq} +} + +func (h *HTTPHandler) Register(router *echo.Group) { + Routes(router) + router.POST("/nextcloud/migration", h.postNextcloudMigration) +} + +type NextcloudMigrationRequest struct { + NextcloudURL string `json:"nextcloud_url"` + NextcloudLogin string `json:"nextcloud_login"` + NextcloudAppPassword string `json:"nextcloud_app_password"` + SourcePath string `json:"source_path,omitempty"` +} + +func (r *NextcloudMigrationRequest) normalize() error { + r.NextcloudURL = strings.TrimSpace(r.NextcloudURL) + r.NextcloudLogin = strings.TrimSpace(r.NextcloudLogin) + r.NextcloudAppPassword = strings.TrimSpace(r.NextcloudAppPassword) + r.SourcePath = strings.TrimSpace(r.SourcePath) + + required := []struct { + name string + value string + }{ + {"nextcloud_url", r.NextcloudURL}, + {"nextcloud_login", r.NextcloudLogin}, + {"nextcloud_app_password", r.NextcloudAppPassword}, + } + for _, f := range required { + if f.value == "" { + return fmt.Errorf("%s is required", f.name) + } + } + r.NextcloudURL = utils.EnsureHasSuffix(r.NextcloudURL, "/") + if r.SourcePath == "" { + r.SourcePath = "/" + } + return nil +} + +func (r *NextcloudMigrationRequest) authMap() map[string]interface{} { + return map[string]interface{}{ + "url": r.NextcloudURL, + "login": r.NextcloudLogin, + "password": r.NextcloudAppPassword, + } +} + +func (h *HTTPHandler) postNextcloudMigration(c echo.Context) error { + if err := middlewares.AllowWholeType(c, permission.POST, consts.NextcloudMigrations); err != nil { + return err + } + inst := middlewares.GetInstance(c) + baseLogger := inst.Logger().WithNamespace(nextcloudMigrationLogNamespace) + + var body NextcloudMigrationRequest + if err := c.Bind(&body); err != nil { + return jsonapi.BadRequest(errors.New("invalid JSON body")) + } + if err := body.normalize(); err != nil { + return jsonapi.BadRequest(err) + } + reqLogger := baseLogger.WithFields(logger.Fields{ + "nextcloud_host": utils.ExtractInstanceHost(body.NextcloudURL), + "nextcloud_login": body.NextcloudLogin, + }) + + active, err := nextcloud.FindActiveMigration(inst) + if err != nil { + reqLogger.Errorf("Failed to query active migrations: %s", err) + return jsonapi.InternalServerError(fmt.Errorf("find active migration: %w", err)) + } + if active != nil { + reqLogger.WithField("active_migration_id", active.ID()). + Infof("Rejecting new Nextcloud migration: one is already in flight") + return jsonapi.Conflict(nextcloud.ErrMigrationConflict) + } + + userID, err := nextcloud.FetchUserIDWithCredentials(body.NextcloudURL, body.NextcloudLogin, body.NextcloudAppPassword, baseLogger) + if err != nil { + if errors.Is(err, webdav.ErrInvalidAuth) { + reqLogger.Infof("Nextcloud credentials probe rejected by remote host") + return jsonapi.Unauthorized(errors.New("nextcloud credentials are invalid")) + } + reqLogger.Warnf("Nextcloud credentials probe failed: %s", err) + return jsonapi.BadGateway(fmt.Errorf("nextcloud unreachable: %w", err)) + } + + accountID, err := ensureNextcloudAccount(inst, &body, userID) + if err != nil { + reqLogger.Errorf("Failed to ensure nextcloud account: %s", err) + return jsonapi.InternalServerError(fmt.Errorf("ensure nextcloud account: %w", err)) + } + + doc := nextcloud.NewPendingMigration("") + if err := couchdb.CreateDoc(inst, doc); err != nil { + reqLogger.WithField("account_id", accountID). + Errorf("Failed to create migration tracking doc: %s", err) + return jsonapi.InternalServerError(fmt.Errorf("create migration tracking doc: %w", err)) + } + migrationLogger := reqLogger.WithFields(logger.Fields{ + "migration_id": doc.DocID, + "account_id": accountID, + }) + + msg := rabbitmq.NextcloudMigrationRequestedMessage{ + MigrationID: doc.DocID, + WorkplaceFqdn: inst.Domain, + AccountID: accountID, + SourcePath: body.SourcePath, + Timestamp: time.Now().Unix(), + } + publishErr := h.rmq.Publish(c.Request().Context(), rabbitmq.PublishRequest{ + ContextName: inst.ContextName, + Exchange: rabbitmq.ExchangeMigration, + RoutingKey: rabbitmq.RoutingKeyNextcloudMigrationRequested, + Payload: msg, + MessageID: doc.DocID, + }) + if publishErr != nil { + migrationLogger.Errorf("Failed to publish migration command: %s", publishErr) + if markErr := doc.MarkFailed(inst, fmt.Errorf("publish migration command: %w", publishErr)); markErr != nil { + migrationLogger.Errorf("Failed to mark migration as failed after publish error: %s", markErr) + } + return jsonapi.NewError(http.StatusServiceUnavailable, "migration service is unavailable, please retry later") + } + + migrationLogger.WithField("source_path", body.SourcePath). + Infof("Nextcloud migration triggered") + return jsonapi.Data(c, http.StatusCreated, doc, nil) +} + +func ensureNextcloudAccount(inst *instance.Instance, body *NextcloudMigrationRequest, userID string) (string, error) { + existing, err := findNextcloudAccountDoc(inst, body.NextcloudURL, body.NextcloudLogin) + if err != nil { + return "", err + } + if existing != nil { + existing.Type = consts.Accounts + existing.M["webdav_user_id"] = userID + existing.M["auth"] = body.authMap() + account.Encrypt(*existing) + if err := couchdb.UpdateDoc(inst, existing); err != nil { + return "", err + } + return existing.ID(), nil + } + + doc := &couchdb.JSONDoc{ + Type: consts.Accounts, + M: map[string]interface{}{ + "account_type": "nextcloud", + "webdav_user_id": userID, + "auth": body.authMap(), + }, + } + account.Encrypt(*doc) + account.ComputeName(*doc) + + if err := couchdb.CreateDoc(inst, doc); err != nil { + return "", err + } + return doc.ID(), nil +} + +// findNextcloudAccountDoc walks io.cozy.accounts and returns the first doc +// whose account_type is "nextcloud" and whose auth.url + auth.login match. +// Mirrors web/accounts/oauth.go:findAccountWithSameConnectionID — a Mango +// selector + composite index would be faster but the per-user account count +// is small enough that the scan is cheaper than the index maintenance. +func findNextcloudAccountDoc(inst *instance.Instance, ncURL, login string) (*couchdb.JSONDoc, error) { + var accounts []*couchdb.JSONDoc + req := &couchdb.AllDocsRequest{Limit: 1000} + err := couchdb.GetAllDocs(inst, consts.Accounts, req, &accounts) + if err != nil { + if couchdb.IsNoDatabaseError(err) { + return nil, nil + } + return nil, err + } + for _, doc := range accounts { + if doc == nil || doc.M == nil { + continue + } + if accType, _ := doc.M["account_type"].(string); accType != "nextcloud" { + continue + } + auth, ok := doc.M["auth"].(map[string]interface{}) + if !ok { + continue + } + if storedURL, _ := auth["url"].(string); storedURL != ncURL { + continue + } + if storedLogin, _ := auth["login"].(string); storedLogin != login { + continue + } + return doc, nil + } + return nil, nil +} diff --git a/web/remote/migration_test.go b/web/remote/migration_test.go new file mode 100644 index 00000000000..b6449c55edb --- /dev/null +++ b/web/remote/migration_test.go @@ -0,0 +1,471 @@ +package remote_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/nextcloud" + "github.com/cozy/cozy-stack/model/permission" + build "github.com/cozy/cozy-stack/pkg/config" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/rabbitmq" + "github.com/cozy/cozy-stack/tests/testutils" + weberrors "github.com/cozy/cozy-stack/web/errors" + "github.com/cozy/cozy-stack/web/remote" + "github.com/gavv/httpexpect/v2" + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type spyRabbitMQ struct { + mu sync.Mutex + messages []rabbitmq.PublishRequest + err error +} + +func (s *spyRabbitMQ) StartManagers() ([]*rabbitmq.RabbitMQManager, error) { + return nil, nil +} + +func (s *spyRabbitMQ) Publish(_ context.Context, req rabbitmq.PublishRequest) error { + if s.err != nil { + return s.err + } + if req.Payload != nil { + payload, err := json.Marshal(req.Payload) + if err != nil { + return err + } + req.Payload = payload + } + s.mu.Lock() + defer s.mu.Unlock() + s.messages = append(s.messages, req) + return nil +} + +func (s *spyRabbitMQ) last() rabbitmq.PublishRequest { + s.mu.Lock() + defer s.mu.Unlock() + return s.messages[len(s.messages)-1] +} + +func (s *spyRabbitMQ) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.messages) +} + +type nextcloudMockOptions struct { + authStatus int + userID string +} + +func startMockNextcloud(t *testing.T, opts nextcloudMockOptions) (url string, calls *int32) { + t.Helper() + if opts.authStatus == 0 { + opts.authStatus = http.StatusOK + } + if opts.userID == "" { + opts.userID = "alice" + } + var counter int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ocs/v2.php/apps/user_status/api/v1/user_status" { + atomic.AddInt32(&counter, 1) + if opts.authStatus != http.StatusOK { + w.WriteHeader(opts.authStatus) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprintf(w, `{"ocs":{"data":{"userId":%q}}}`, opts.userID) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + return srv.URL + "/", &counter +} + +func migrationPermission() *permission.Permission { + return &permission.Permission{ + Type: permission.TypeWebapp, + SourceID: consts.Apps + "/" + consts.SettingsSlug, + Permissions: permission.Set{ + permission.Rule{ + Type: consts.NextcloudMigrations, + Verbs: permission.Verbs(permission.POST), + }, + }, + } +} + +func setupMigrationRouter(t *testing.T, inst *instance.Instance, pdoc *permission.Permission, rmq rabbitmq.Service) *httptest.Server { + t.Helper() + + handler := echo.New() + handler.HTTPErrorHandler = weberrors.ErrorHandler + group := handler.Group("/remote", func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set("instance", inst) + if pdoc != nil { + c.Set("permissions_doc", pdoc) + } + return next(c) + } + }) + + remote.NewHTTPHandler(rmq).Register(group) + + ts := httptest.NewServer(handler) + t.Cleanup(ts.Close) + return ts +} + +func migrationRequestBody(url string, extra map[string]interface{}) map[string]interface{} { + body := map[string]interface{}{ + "nextcloud_url": url, + "nextcloud_login": "alice", + "nextcloud_app_password": "app-password-xxx", + "source_path": "/", + } + for k, v := range extra { + body[k] = v + } + return body +} + +func TestPostNextcloudMigration(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + testutils.NeedCouchdb(t) + + // safehttp refuses loopback hosts outside dev mode; flip the flag so + // httptest.NewServer URLs are reachable. + oldBuildMode := build.BuildMode + build.BuildMode = build.ModeDev + t.Cleanup(func() { build.BuildMode = oldBuildMode }) + + t.Run("HappyPath", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-happy") + inst := setup.GetTestInstance() + + ncURL, probeCalls := startMockNextcloud(t, nextcloudMockOptions{userID: "alice-webdav"}) + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + obj := e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, nil)). + Expect().Status(http.StatusCreated). + JSON(httpexpect.ContentOpts{MediaType: "application/vnd.api+json"}). + Object() + + data := obj.Value("data").Object() + data.Value("type").String().IsEqual(consts.NextcloudMigrations) + migrationID := data.Value("id").String().NotEmpty().Raw() + attrs := data.Value("attributes").Object() + attrs.Value("status").String().IsEqual(nextcloud.MigrationStatusPending) + attrs.Value("target_dir").String().IsEqual(nextcloud.DefaultMigrationTargetDir) + attrs.Value("errors").Array().IsEmpty() + attrs.Value("skipped").Array().IsEmpty() + progress := attrs.Value("progress").Object() + progress.Value("files_imported").Number().IsEqual(0) + progress.Value("files_total").Number().IsEqual(0) + progress.Value("bytes_imported").Number().IsEqual(0) + progress.Value("bytes_total").Number().IsEqual(0) + + require.Equal(t, int32(1), atomic.LoadInt32(probeCalls), "probe should hit the OCS endpoint once") + require.Equal(t, 1, spy.count(), "expected one RabbitMQ publish") + pub := spy.last() + assert.Equal(t, rabbitmq.ExchangeMigration, pub.Exchange) + assert.Equal(t, rabbitmq.RoutingKeyNextcloudMigrationRequested, pub.RoutingKey) + assert.Equal(t, migrationID, pub.MessageID) + + var payload rabbitmq.NextcloudMigrationRequestedMessage + require.NoError(t, json.Unmarshal(pub.Payload.([]byte), &payload)) + assert.Equal(t, migrationID, payload.MigrationID) + assert.Equal(t, inst.Domain, payload.WorkplaceFqdn) + assert.NotEmpty(t, payload.AccountID) + assert.Equal(t, "/", payload.SourcePath) + assert.NotZero(t, payload.Timestamp) + + var stored nextcloud.Migration + require.NoError(t, couchdb.GetDoc(inst, consts.NextcloudMigrations, migrationID, &stored)) + assert.Equal(t, nextcloud.MigrationStatusPending, stored.Status) + + var accDoc couchdb.JSONDoc + require.NoError(t, couchdb.GetDoc(inst, consts.Accounts, payload.AccountID, &accDoc)) + assert.Equal(t, "nextcloud", accDoc.M["account_type"]) + assert.Equal(t, "alice-webdav", accDoc.M["webdav_user_id"], + "probe-resolved userID must be cached on the account") + auth, ok := accDoc.M["auth"].(map[string]interface{}) + require.True(t, ok, "auth should be a map") + assert.Equal(t, "alice", auth["login"]) + assert.Nil(t, auth["password"], "plaintext password must not be persisted") + assert.NotEmpty(t, auth["credentials_encrypted"], "credentials should be encrypted at rest") + }) + + t.Run("WrongCredentialsReturn401", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-wrong-creds") + inst := setup.GetTestInstance() + + ncURL, probeCalls := startMockNextcloud(t, nextcloudMockOptions{authStatus: http.StatusUnauthorized}) + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, nil)). + Expect().Status(http.StatusUnauthorized) + + assert.Equal(t, int32(1), atomic.LoadInt32(probeCalls)) + assert.Equal(t, 0, spy.count(), "no publish when credentials are invalid") + + var docs []*nextcloud.Migration + req := &couchdb.AllDocsRequest{Limit: 10} + err := couchdb.GetAllDocs(inst, consts.NextcloudMigrations, req, &docs) + if err != nil && !couchdb.IsNoDatabaseError(err) { + t.Fatalf("unexpected error listing migrations: %s", err) + } + assert.Empty(t, docs, "no tracking doc should be created on auth failure") + + var accounts []*couchdb.JSONDoc + accReq := &couchdb.AllDocsRequest{Limit: 10} + err = couchdb.GetAllDocs(inst, consts.Accounts, accReq, &accounts) + if err != nil && !couchdb.IsNoDatabaseError(err) { + t.Fatalf("unexpected error listing accounts: %s", err) + } + for _, a := range accounts { + if a.M["account_type"] == "nextcloud" { + t.Fatalf("no nextcloud account should be created on auth failure, got %s", a.ID()) + } + } + }) + + t.Run("NextcloudUnreachableReturns502", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-unreachable") + inst := setup.GetTestInstance() + + // Close the server immediately so the URL points at a dead listener. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + deadURL := srv.URL + "/" + srv.Close() + + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(deadURL, nil)). + Expect().Status(http.StatusBadGateway) + + assert.Equal(t, 0, spy.count()) + }) + + t.Run("ConflictWhenMigrationAlreadyRunning", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-conflict") + inst := setup.GetTestInstance() + + existing := nextcloud.NewPendingMigration("") + existing.Status = nextcloud.MigrationStatusRunning + require.NoError(t, couchdb.CreateDoc(inst, existing)) + + ncURL, _ := startMockNextcloud(t, nextcloudMockOptions{}) + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, nil)). + Expect().Status(http.StatusConflict) + + assert.Equal(t, 0, spy.count(), "no message should be published on conflict") + }) + + t.Run("PublishFailureMarksMigrationFailed", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-publish-fail") + inst := setup.GetTestInstance() + + ncURL, _ := startMockNextcloud(t, nextcloudMockOptions{}) + spy := &spyRabbitMQ{err: fmt.Errorf("broker unreachable")} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, nil)). + Expect().Status(http.StatusServiceUnavailable) + + active, err := nextcloud.FindActiveMigration(inst) + require.NoError(t, err) + assert.Nil(t, active, "failed migrations must not block new ones") + + var docs []*nextcloud.Migration + req := &couchdb.AllDocsRequest{Limit: 10} + require.NoError(t, couchdb.GetAllDocs(inst, consts.NextcloudMigrations, req, &docs)) + require.Len(t, docs, 1) + assert.Equal(t, nextcloud.MigrationStatusFailed, docs[0].Status) + require.NotEmpty(t, docs[0].Errors) + assert.Contains(t, docs[0].Errors[0].Message, "broker unreachable") + }) + + t.Run("AccountIsReusedOnSecondMigration", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-account-reuse") + inst := setup.GetTestInstance() + + ncURL, _ := startMockNextcloud(t, nextcloudMockOptions{}) + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + first := e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, nil)). + Expect().Status(http.StatusCreated). + JSON(httpexpect.ContentOpts{MediaType: "application/vnd.api+json"}). + Object() + firstID := first.Value("data").Object().Value("id").String().Raw() + require.Equal(t, 1, spy.count()) + firstAccountID := decodeAccountID(t, spy.last().Payload.([]byte)) + + // Flip the first migration to completed so the conflict check lets + // the second one through. + var doc nextcloud.Migration + require.NoError(t, couchdb.GetDoc(inst, consts.NextcloudMigrations, firstID, &doc)) + doc.Status = nextcloud.MigrationStatusCompleted + require.NoError(t, couchdb.UpdateDoc(inst, &doc)) + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, nil)). + Expect().Status(http.StatusCreated) + require.Equal(t, 2, spy.count()) + secondAccountID := decodeAccountID(t, spy.last().Payload.([]byte)) + + assert.Equal(t, firstAccountID, secondAccountID, "account should be reused across migrations") + + var accounts []*couchdb.JSONDoc + req := &couchdb.AllDocsRequest{Limit: 10} + require.NoError(t, couchdb.GetAllDocs(inst, consts.Accounts, req, &accounts)) + var nextcloudAccountIDs []string + for _, a := range accounts { + if a.M["account_type"] == "nextcloud" { + nextcloudAccountIDs = append(nextcloudAccountIDs, a.ID()) + } + } + assert.Len(t, nextcloudAccountIDs, 1) + }) + + t.Run("AccountReuseRefreshesStoredPassword", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-refresh-password") + inst := setup.GetTestInstance() + + ncURL, _ := startMockNextcloud(t, nextcloudMockOptions{}) + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + stale := &couchdb.JSONDoc{ + Type: consts.Accounts, + M: map[string]interface{}{ + "account_type": "nextcloud", + "webdav_user_id": "stale-userid", + "auth": map[string]interface{}{ + "url": ncURL, + "login": "alice", + "password": "old-wrong-pass", + }, + }, + } + require.NoError(t, couchdb.CreateDoc(inst, stale)) + staleID := stale.ID() + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, map[string]interface{}{ + "nextcloud_app_password": "fresh-correct-pass", + })). + Expect().Status(http.StatusCreated) + + require.Equal(t, 1, spy.count()) + reusedID := decodeAccountID(t, spy.last().Payload.([]byte)) + assert.Equal(t, staleID, reusedID, "existing account should be reused, not duplicated") + + var refreshed couchdb.JSONDoc + require.NoError(t, couchdb.GetDoc(inst, consts.Accounts, staleID, &refreshed)) + assert.Equal(t, "alice", refreshed.M["webdav_user_id"], + "webdav_user_id should be refreshed from the probe") + auth, ok := refreshed.M["auth"].(map[string]interface{}) + require.True(t, ok) + assert.NotEmpty(t, auth["credentials_encrypted"]) + assert.Nil(t, auth["password"]) + }) + + t.Run("RejectsMissingCredentials", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-reject-missing") + inst := setup.GetTestInstance() + + ncURL, _ := startMockNextcloud(t, nextcloudMockOptions{}) + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, migrationPermission(), spy) + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, map[string]interface{}{ + "nextcloud_app_password": "", + })). + Expect().Status(http.StatusBadRequest) + + assert.Equal(t, 0, spy.count()) + }) + + t.Run("RejectsWithoutPermission", func(t *testing.T) { + setup := testutils.NewSetup(t, "ncmigration-reject-nomore") + inst := setup.GetTestInstance() + + ncURL, _ := startMockNextcloud(t, nextcloudMockOptions{}) + pdoc := &permission.Permission{ + Type: permission.TypeWebapp, + SourceID: consts.Apps + "/" + consts.SettingsSlug, + } + spy := &spyRabbitMQ{} + ts := setupMigrationRouter(t, inst, pdoc, spy) + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/remote/nextcloud/migration"). + WithHeader("Accept", "application/vnd.api+json"). + WithJSON(migrationRequestBody(ncURL, nil)). + Expect().Status(http.StatusForbidden) + + assert.Equal(t, 0, spy.count()) + }) +} + +func decodeAccountID(t *testing.T, payload []byte) string { + t.Helper() + var msg rabbitmq.NextcloudMigrationRequestedMessage + require.NoError(t, json.Unmarshal(payload, &msg)) + return msg.AccountID +} diff --git a/web/routing.go b/web/routing.go index 99ee3b06782..25048c4f531 100644 --- a/web/routing.go +++ b/web/routing.go @@ -234,7 +234,7 @@ func SetupRoutes(router *echo.Echo, services *stack.Services) error { realtime.Routes(router.Group("/realtime", mws...)) notes.Routes(router.Group("/notes", mws...)) office.Routes(router.Group("/office", mws...)) - remote.Routes(router.Group("/remote", mws...)) + remote.NewHTTPHandler(services.RabbitMQ).Register(router.Group("/remote", mws...)) sharings.Routes(router.Group("/sharings", mws...)) bitwarden.Routes(router.Group("/bitwarden", mws...)) shortcuts.Routes(router.Group("/shortcuts", mws...)) From f6cfb350a71b9393a3f948dd109b0dcd374fcace Mon Sep 17 00:00:00 2001 From: Khaled FERJANI Date: Tue, 14 Apr 2026 17:03:05 +0200 Subject: [PATCH 2/2] fix(nextcloud): probe OCS core instead of optional user_status app Managed Nextcloud providers (e.g. thegood.cloud) strip the optional user_status app, so the probe returned 404 on otherwise-valid instances. Combined with the coarse "non-200 = invalid auth" classifier, every migration request against these hosts surfaced as a misleading 401 "credentials invalid". Probe /ocs/v2.php/cloud/user (OCS Core, cannot be disabled) and narrow auth-failure classification to 401/403 only. --- model/nextcloud/nextcloud.go | 28 ++++++--- model/nextcloud/nextcloud_test.go | 74 +++++++++++++++++++++++ web/remote/app_token_verification_test.go | 4 +- web/remote/migration_test.go | 4 +- web/remote/remote_test.go | 6 +- 5 files changed, 100 insertions(+), 16 deletions(-) create mode 100644 model/nextcloud/nextcloud_test.go diff --git a/model/nextcloud/nextcloud.go b/model/nextcloud/nextcloud.go index e7a981ae193..43ddabcaf4b 100644 --- a/model/nextcloud/nextcloud.go +++ b/model/nextcloud/nextcloud.go @@ -5,6 +5,7 @@ package nextcloud import ( "context" "encoding/json" + "fmt" "io" "net/http" "net/url" @@ -362,10 +363,10 @@ func (nc *NextCloud) buildTrashedURL(item webdav.Item, path string) string { return u.String() } -// FetchUserIDWithCredentials probes the OCS user_status endpoint and returns +// FetchUserIDWithCredentials probes the OCS cloud/user endpoint and returns // the user ID, or webdav.ErrInvalidAuth if the credentials are rejected. // -// https://docs.nextcloud.com/server/latest/developer_manual/client_apis/OCS/ocs-status-api.html#fetch-your-own-status +// https://docs.nextcloud.com/server/latest/developer_manual/client_apis/OCS/ocs-api-overview.html func FetchUserIDWithCredentials(nextcloudURL, username, password string, logger *logger.Entry) (string, error) { u, err := url.Parse(nextcloudURL) if err != nil { @@ -377,19 +378,24 @@ func FetchUserIDWithCredentials(nextcloudURL, username, password string, logger return fetchUserIDFromHost(u.Scheme, u.Host, username, password, logger) } -const userStatusProbeTimeout = 30 * time.Second +const probeTimeout = 30 * time.Second + +// cloudUserProbePath is OCS Core and cannot be disabled by an admin, unlike +// apps/user_status which some managed Nextcloud providers strip. Probing Core +// avoids misclassifying a stripped-optional-app install as an auth failure. +const cloudUserProbePath = "/ocs/v2.php/cloud/user" func fetchUserIDFromHost(scheme, host, username, password string, logger *logger.Entry) (string, error) { u := url.URL{ Scheme: scheme, Host: host, User: url.UserPassword(username, password), - Path: "/ocs/v2.php/apps/user_status/api/v1/user_status", + Path: cloudUserProbePath, } // Cap the probe so a hung Nextcloud server can't pin a request goroutine — // safehttp.ClientWithKeepAlive has handshake timeouts but no overall // request deadline. - ctx, cancel := context.WithTimeout(context.Background(), userStatusProbeTimeout) + ctx, cancel := context.WithTimeout(context.Background(), probeTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { @@ -402,13 +408,17 @@ func fetchUserIDFromHost(scheme, host, username, password string, logger *logger res, err := safehttp.ClientWithKeepAlive.Do(req) elapsed := time.Since(start) if err != nil { - logger.Warnf("user_status %s: %s (%s)", u.Host, err, elapsed) + logger.Warnf("cloud/user %s: %s (%s)", u.Host, err, elapsed) return "", err } defer res.Body.Close() - logger.Infof("user_status %s: %d (%s)", u.Host, res.StatusCode, elapsed) - if res.StatusCode != 200 { + logger.Infof("cloud/user %s: %d (%s)", u.Host, res.StatusCode, elapsed) + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized, http.StatusForbidden: return "", webdav.ErrInvalidAuth + default: + return "", fmt.Errorf("unexpected status %d from nextcloud cloud/user probe", res.StatusCode) } var payload OCSPayload if err := json.NewDecoder(res.Body).Decode(&payload); err != nil { @@ -425,7 +435,7 @@ func (nc *NextCloud) fetchUserID() (string, error) { type OCSPayload struct { OCS struct { Data struct { - UserID string `json:"userId"` + UserID string `json:"id"` } `json:"data"` } `json:"ocs"` } diff --git a/model/nextcloud/nextcloud_test.go b/model/nextcloud/nextcloud_test.go new file mode 100644 index 00000000000..cbe5f92e03c --- /dev/null +++ b/model/nextcloud/nextcloud_test.go @@ -0,0 +1,74 @@ +package nextcloud + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + build "github.com/cozy/cozy-stack/pkg/config" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/webdav" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchUserIDWithCredentials(t *testing.T) { + // safehttp refuses loopback hosts outside dev mode. + oldBuildMode := build.BuildMode + build.BuildMode = build.ModeDev + t.Cleanup(func() { build.BuildMode = oldBuildMode }) + + log := logger.WithNamespace("nextcloud-test") + + t.Run("resolves user ID against OCS cloud/user on a Nextcloud without user_status", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ocs/v2.php/cloud/user" { + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprint(w, `{"ocs":{"meta":{"status":"ok","statuscode":200},"data":{"id":"alice-webdav"}}}`) + return + } + // Any other path (including user_status) returns 404, matching + // managed Nextcloud hosts that strip optional OCS apps. + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + userID, err := FetchUserIDWithCredentials(srv.URL+"/", "alice", "app-password", log) + require.NoError(t, err) + assert.Equal(t, "alice-webdav", userID) + }) + + t.Run("returns ErrInvalidAuth on 401", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + t.Cleanup(srv.Close) + + _, err := FetchUserIDWithCredentials(srv.URL+"/", "alice", "bad", log) + assert.ErrorIs(t, err, webdav.ErrInvalidAuth) + }) + + t.Run("returns ErrInvalidAuth on 403", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + _, err := FetchUserIDWithCredentials(srv.URL+"/", "alice", "bad", log) + assert.ErrorIs(t, err, webdav.ErrInvalidAuth) + }) + + t.Run("does not conflate a 500 with an auth failure", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(srv.Close) + + _, err := FetchUserIDWithCredentials(srv.URL+"/", "alice", "app-password", log) + require.Error(t, err) + assert.False(t, errors.Is(err, webdav.ErrInvalidAuth), + "a 500 from Nextcloud must not be reported as invalid credentials") + }) +} diff --git a/web/remote/app_token_verification_test.go b/web/remote/app_token_verification_test.go index d67184c6fb3..777a4c5c2ed 100644 --- a/web/remote/app_token_verification_test.go +++ b/web/remote/app_token_verification_test.go @@ -61,10 +61,10 @@ func TestAppAudienceTokenPassesNextcloudPermissionCheck(t *testing.T) { require.NotEmpty(t, token) mockWebDAV := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/ocs/v2.php/apps/user_status/api/v1/user_status" { + if r.URL.Path == "/ocs/v2.php/cloud/user" { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"ocs":{"data":{"userId":"migrator"}}}`)) + _, _ = w.Write([]byte(`{"ocs":{"data":{"id":"migrator"}}}`)) return } if r.Method == "PROPFIND" { diff --git a/web/remote/migration_test.go b/web/remote/migration_test.go index b6449c55edb..20e2aa21b9f 100644 --- a/web/remote/migration_test.go +++ b/web/remote/migration_test.go @@ -81,7 +81,7 @@ func startMockNextcloud(t *testing.T, opts nextcloudMockOptions) (url string, ca } var counter int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/ocs/v2.php/apps/user_status/api/v1/user_status" { + if r.URL.Path == "/ocs/v2.php/cloud/user" { atomic.AddInt32(&counter, 1) if opts.authStatus != http.StatusOK { w.WriteHeader(opts.authStatus) @@ -89,7 +89,7 @@ func startMockNextcloud(t *testing.T, opts nextcloudMockOptions) (url string, ca } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - _, _ = fmt.Fprintf(w, `{"ocs":{"data":{"userId":%q}}}`, opts.userID) + _, _ = fmt.Fprintf(w, `{"ocs":{"data":{"id":%q}}}`, opts.userID) return } w.WriteHeader(http.StatusNotFound) diff --git a/web/remote/remote_test.go b/web/remote/remote_test.go index 211994f5138..49285978ccc 100644 --- a/web/remote/remote_test.go +++ b/web/remote/remote_test.go @@ -142,11 +142,11 @@ func TestNextcloudDownstreamFailOnConflict(t *testing.T) { w.WriteHeader(http.StatusNoContent) return } - // Handle user_status endpoint (needed to get webdav_user_id) - if r.URL.Path == "/ocs/v2.php/apps/user_status/api/v1/user_status" { + // Handle OCS cloud/user endpoint (needed to get webdav_user_id) + if r.URL.Path == "/ocs/v2.php/cloud/user" { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"ocs":{"data":{"userId":"testuser"}}}`)) + w.Write([]byte(`{"ocs":{"data":{"id":"testuser"}}}`)) return } w.WriteHeader(http.StatusNotFound)