diff --git a/.gitignore b/.gitignore index 33e55761570..c092051d6c6 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ protogen/buf.sha1.lock /third-party-licenses # misc +.agents/ /tmp go.work go.work.sum diff --git a/.make/go.mk b/.make/go.mk index 7000c56e383..a16d58c4f21 100644 --- a/.make/go.mk +++ b/.make/go.mk @@ -118,7 +118,6 @@ debug-linux-docker-amd64: release-dirs -gcflags="all=-N -l" \ -tags 'netgo $(TAGS)' \ -buildmode=exe \ - -trimpath \ -ldflags '-extldflags "-static" $(DEBUG_LDFLAGS) $(DOCKER_LDFLAGS)' \ -o '$(DIST)/binaries/$(EXECUTABLE)-linux-amd64' \ ./cmd/$(NAME) @@ -130,7 +129,6 @@ debug-linux-docker-arm64: release-dirs -gcflags="all=-N -l" \ -tags 'netgo $(TAGS)' \ -buildmode=exe \ - -trimpath \ -ldflags '-extldflags "-static" $(DEBUG_LDFLAGS) $(DOCKER_LDFLAGS)' \ -o '$(DIST)/binaries/$(EXECUTABLE)-linux-arm64' \ ./cmd/$(NAME) diff --git a/deployments/examples/ocis_full/.env b/deployments/examples/ocis_full/.env index cd8bccf424f..b87b403717b 100644 --- a/deployments/examples/ocis_full/.env +++ b/deployments/examples/ocis_full/.env @@ -186,6 +186,11 @@ KEYCLOAK_TRACING= # Note: the leading colon is required to enable the service. #KEYCLOAK=:keycloak.yml +### oCIS Vault Storage Settings ### +# Enable the oCIS vault storage +# Note: the leading colon is required to enable the service. +#VAULT_STORAGE=:vault-storage.yml + ## Default Enabled Services ## @@ -297,4 +302,4 @@ MAIL_SERVER_DOCKER_TAG=v1.29.3 # This MUST be the last line as it assembles the supplemental compose files to be used. # ALL supplemental configs must be added here, whether commented or not. # Each var must either be empty or contain :path/file.yml -COMPOSE_FILE=docker-compose.yml${OCIS:-}${TIKA:-}${S3NG:-}${S3NG_MINIO:-}${COLLABORA:-}${IMPORTER:-}${CLAMAV:-}${ONLYOFFICE:-}${EXTENSIONS:-}${UNZIP:-}${DRAWIO:-}${JSONVIEWER:-}${PROGRESSBARS:-}${EXTERNALSITES:-}${PHOTOADDON:-}${ADVANCEDSEARCH:-}${MAIL_SERVER:-}${MONITORING:-}${KEYCLOAK:-} +COMPOSE_FILE=docker-compose.yml${OCIS:-}${TIKA:-}${S3NG:-}${S3NG_MINIO:-}${COLLABORA:-}${IMPORTER:-}${CLAMAV:-}${ONLYOFFICE:-}${EXTENSIONS:-}${UNZIP:-}${DRAWIO:-}${JSONVIEWER:-}${PROGRESSBARS:-}${EXTERNALSITES:-}${PHOTOADDON:-}${ADVANCEDSEARCH:-}${MAIL_SERVER:-}${MONITORING:-}${KEYCLOAK:-}${VAULT_STORAGE:-} diff --git a/deployments/examples/ocis_full/vault-storage.yml b/deployments/examples/ocis_full/vault-storage.yml new file mode 100644 index 00000000000..d5ad6d3c093 --- /dev/null +++ b/deployments/examples/ocis_full/vault-storage.yml @@ -0,0 +1,37 @@ +services: + ocis: + environment: + OCIS_MFA_ENABLED: true + NATS_NATS_HOST: 0.0.0.0 + SETTINGS_GRPC_ADDR: ocis:9191 + PROXY_CREATE_VAULT_HOME: true + GRAPH_ENABLE_VAULT_MODE: true + + storage-users-vault: + image: ${OCIS_DOCKER_IMAGE:-owncloud/ocis}:${OCIS_DOCKER_TAG:-latest} + networks: + ocis-net: + depends_on: + ocis: + condition: service_started + command: ["storage-users", "server"] + environment: + OCIS_LOG_LEVEL: debug + OCIS_GATEWAY_GRPC_ADDR: ocis:9142 + STORAGE_USERS_ENABLE_VAULT_MODE: true + STORAGE_USERS_SERVICE_NAME: storage-users-vault + STORAGE_USERS_GRPC_ADDR: storage-users-vault:9170 + STORAGE_USERS_HTTP_ADDR: storage-users-vault:9168 + STORAGE_USERS_DATA_SERVER_URL: http://storage-users-vault:9168/data + STORAGE_USERS_DEBUG_ADDR: storage-users-vault:9169 + STORAGE_USERS_OCIS_ROOT: /var/lib/ocis/storage/users-vault + STORAGE_USERS_EVENTS_CONSUMER_GROUP: vault-dcfs + MICRO_REGISTRY_ADDRESS: ocis:9233 + OCIS_EVENTS_ENDPOINT: ocis:9233 + OCIS_CACHE_STORE_NODES: ocis:9233 + volumes: + - ocis-data:/var/lib/ocis + - ocis-config:/etc/ocis + logging: + driver: ${LOG_DRIVER:-local} + restart: always diff --git a/go.mod b/go.mod index c1b7de742bb..9ae3a981912 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( github.com/open-policy-agent/opa v1.12.3 github.com/orcaman/concurrent-map v1.0.0 github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245 - github.com/owncloud/reva/v2 v2.0.0-20260324082555-823c2f1c2593 + github.com/owncloud/reva/v2 v2.0.0-20260324173335-cc6175484320 github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.12 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index 7fd0d5eb664..0b6ae425c8e 100644 --- a/go.sum +++ b/go.sum @@ -742,8 +742,8 @@ github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HD github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245 h1:JRidLTAKhnvyLMRtVtSF4lhBa0NSAOs6fof+d6JnKII= github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245/go.mod h1:z61VMGAJRtR1nbgXWiNoCkxUXP1B3Je9rMuJbnGd+Og= -github.com/owncloud/reva/v2 v2.0.0-20260324082555-823c2f1c2593 h1:RNHAod2gNBEac0KQJfJ6+PCX1t7g9hFmONTGrXFvFII= -github.com/owncloud/reva/v2 v2.0.0-20260324082555-823c2f1c2593/go.mod h1:+rCy6oGYb2/qs5gmQa8y/pHARw634vB73MZGDY2SBIQ= +github.com/owncloud/reva/v2 v2.0.0-20260324173335-cc6175484320 h1:UPOCuW88zJx9UYevS3j3dqI9ncjBihcBP7o0Igvl0ZI= +github.com/owncloud/reva/v2 v2.0.0-20260324173335-cc6175484320/go.mod h1:+rCy6oGYb2/qs5gmQa8y/pHARw634vB73MZGDY2SBIQ= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pablodz/inotifywaitgo v0.0.9 h1:njquRbBU7fuwIe5rEvtaniVBjwWzcpdUVptSgzFqZsw= diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 53699fccce0..92f44c19f4e 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -85,18 +85,11 @@ type StorageRegistry struct { // Cache holds cache config type Cache struct { - ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"pre5.0"` - ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"pre5.0"` - ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"5.0"` - ProviderCacheAuthUsername string `yaml:"provider_cache_auth_username" env:"OCIS_CACHE_AUTH_USERNAME;GATEWAY_PROVIDER_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` - ProviderCacheAuthPassword string `yaml:"provider_cache_auth_password" env:"OCIS_CACHE_AUTH_PASSWORD;GATEWAY_PROVIDER_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` - CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"pre5.0"` - CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"pre5.0"` - CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - CreateHomeCacheDisablePersistence bool `yaml:"create_home_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_CREATE_HOME_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the create home cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"5.0"` - CreateHomeCacheAuthUsername string `yaml:"create_home_cache_auth_username" env:"OCIS_CACHE_AUTH_USERNAME;GATEWAY_CREATE_HOME_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` - CreateHomeCacheAuthPassword string `yaml:"create_home_cache_auth_password" env:"OCIS_CACHE_AUTH_PASSWORD;GATEWAY_CREATE_HOME_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` + ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"pre5.0"` + ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` + ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"pre5.0"` + ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` + ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"5.0"` + ProviderCacheAuthUsername string `yaml:"provider_cache_auth_username" env:"OCIS_CACHE_AUTH_USERNAME;GATEWAY_PROVIDER_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` + ProviderCacheAuthPassword string `yaml:"provider_cache_auth_password" env:"OCIS_CACHE_AUTH_PASSWORD;GATEWAY_PROVIDER_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` } diff --git a/services/gateway/pkg/config/defaults/defaultconfig.go b/services/gateway/pkg/config/defaults/defaultconfig.go index 5d003bb3790..4bab0339312 100644 --- a/services/gateway/pkg/config/defaults/defaultconfig.go +++ b/services/gateway/pkg/config/defaults/defaultconfig.go @@ -39,14 +39,10 @@ func DefaultConfig() *config.Config { DisableHomeCreationOnLogin: true, TransferExpires: 24 * 60 * 60, Cache: config.Cache{ - ProviderCacheStore: "noop", - ProviderCacheNodes: []string{"127.0.0.1:9233"}, - ProviderCacheDatabase: "cache-providers", - ProviderCacheTTL: 300 * time.Second, - CreateHomeCacheStore: "memory", - CreateHomeCacheNodes: []string{"127.0.0.1:9233"}, - CreateHomeCacheDatabase: "cache-createhome", - CreateHomeCacheTTL: 300 * time.Second, + ProviderCacheStore: "noop", + ProviderCacheNodes: []string{"127.0.0.1:9233"}, + ProviderCacheDatabase: "cache-providers", + ProviderCacheTTL: 300 * time.Second, }, FrontendPublicURL: "https://localhost:9200", diff --git a/services/gateway/pkg/revaconfig/config.go b/services/gateway/pkg/revaconfig/config.go index d4589e5d7fb..a24a9a0e371 100644 --- a/services/gateway/pkg/revaconfig/config.go +++ b/services/gateway/pkg/revaconfig/config.go @@ -75,16 +75,6 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]i "cache_auth_username": cfg.Cache.ProviderCacheAuthUsername, "cache_auth_password": cfg.Cache.ProviderCacheAuthPassword, }, - "create_personal_space_cache_config": map[string]interface{}{ - "cache_store": cfg.Cache.CreateHomeCacheStore, - "cache_nodes": cfg.Cache.CreateHomeCacheNodes, - "cache_database": cfg.Cache.CreateHomeCacheDatabase, - "cache_table": "create_personal_space", - "cache_ttl": cfg.Cache.CreateHomeCacheTTL, - "cache_disable_persistence": cfg.Cache.CreateHomeCacheDisablePersistence, - "cache_auth_username": cfg.Cache.CreateHomeCacheAuthUsername, - "cache_auth_password": cfg.Cache.CreateHomeCacheAuthPassword, - }, }, "authregistry": map[string]interface{}{ "driver": "static", @@ -162,6 +152,22 @@ func spacesProviders(cfg *config.Config, logger log.Logger) map[string]map[strin }, }, }, + "com.owncloud.api.storage-users-vault": { + // Use the dedicated storage provider for vault + "providerid": utils.VaultStorageProviderID, + "spaces": map[string]interface{}{ + "personal": map[string]interface{}{ + // The mount point must have the "vault/" prefix to be picked up by the vault storage provider + "mount_point": "/vault/users", + "path_template": "/vault/users/{{.Space.Owner.Id.OpaqueId}}", + }, + "project": map[string]interface{}{ + // The mount point must have the "vault/" prefix to be picked up by the vault storage provider + "mount_point": "/vault/projects", + "path_template": "/vault/projects/{{.Space.Name}}", + }, + }, + }, cfg.StorageSharesEndpoint: { "providerid": utils.ShareStorageProviderID, "spaces": map[string]interface{}{ diff --git a/services/graph/pkg/config/config.go b/services/graph/pkg/config/config.go index ab0072af375..813d24382b4 100644 --- a/services/graph/pkg/config/config.go +++ b/services/graph/pkg/config/config.go @@ -39,6 +39,8 @@ type Config struct { Validation Validation `yaml:"validation"` + EnableVaultMode bool `yaml:"enable_vault_mode" env:"GRAPH_ENABLE_VAULT_MODE" desc:"Enable vault mode for the graph service runned in addition to the regular graph service. Required the running the storage-users-vault additional service." introductionVersion:"daledda"` + Context context.Context `yaml:"-"` } diff --git a/services/graph/pkg/config/service.go b/services/graph/pkg/config/service.go index d1eac383f0b..f7edce2b7dd 100644 --- a/services/graph/pkg/config/service.go +++ b/services/graph/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string `yaml:"-"` + Name string `yaml:"name" env:"GRAPH_SERVICE_NAME" desc:"The name of the service." introductionVersion:"daledda"` } diff --git a/services/graph/pkg/middleware/mfa.go b/services/graph/pkg/middleware/mfa.go new file mode 100644 index 00000000000..33c1206e09b --- /dev/null +++ b/services/graph/pkg/middleware/mfa.go @@ -0,0 +1,23 @@ +package middleware + +import ( + "net/http" + + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/ocis-pkg/mfa" +) + +// RequireMFA middleware is used to require the user in context to have MFA satisfied +func RequireMFA(logger log.Logger) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !mfa.Has(r.Context()) { + l := logger.SubloggerWithRequestID(r.Context()) + l.Error().Str("path", r.URL.Path).Msg("MFA required but not satisfied") + mfa.SetRequiredStatus(w) + return + } + next.ServeHTTP(w, r) + }) + } +} diff --git a/services/graph/pkg/middleware/vault.go b/services/graph/pkg/middleware/vault.go new file mode 100644 index 00000000000..5dfd3450f8f --- /dev/null +++ b/services/graph/pkg/middleware/vault.go @@ -0,0 +1,27 @@ +package middleware + +import ( + "context" + "net/http" +) + +type key int + +const vaultModeKey key = iota + +func SetVaultMode(ctx context.Context, enabled bool) context.Context { + return context.WithValue(ctx, vaultModeKey, enabled) +} + +func IsVaultMode(ctx context.Context) bool { + val, ok := ctx.Value(vaultModeKey).(bool) + return val && ok +} + +func VaultModeMiddleware() func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r.WithContext(SetVaultMode(r.Context(), true))) + }) + } +} diff --git a/services/graph/pkg/service/v0/driveitems.go b/services/graph/pkg/service/v0/driveitems.go index 2a134500cab..0edcbf50ea7 100644 --- a/services/graph/pkg/service/v0/driveitems.go +++ b/services/graph/pkg/service/v0/driveitems.go @@ -30,6 +30,7 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/graph/pkg/errorcode" + "github.com/owncloud/ocis/v2/services/graph/pkg/middleware" ) // CreateUploadSession create an upload session to allow your app to upload files up to the maximum file size. @@ -158,6 +159,11 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) { filters = append(filters, listStorageSpacesUserFilter(currentUser.GetId().GetOpaqueId())) filters = append(filters, listStorageSpacesTypeFilter("personal")) + // force vault storage space if vault mode is enabled + if middleware.IsVaultMode(ctx) { + filters = append(filters, listStorageSpacesIDFilter(storagespace.FormatStorageID(utils.VaultStorageProviderID, currentUser.GetId().GetOpaqueId()))) + } + res, err := gatewayClient.ListStorageSpaces(ctx, &storageprovider.ListStorageSpacesRequest{ Filters: filters, }) diff --git a/services/graph/pkg/service/v0/drives.go b/services/graph/pkg/service/v0/drives.go index c4657b32054..bdb0e943414 100644 --- a/services/graph/pkg/service/v0/drives.go +++ b/services/graph/pkg/service/v0/drives.go @@ -29,10 +29,10 @@ import ( "google.golang.org/protobuf/proto" "github.com/owncloud/ocis/v2/ocis-pkg/l10n" - "github.com/owncloud/ocis/v2/ocis-pkg/mfa" v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0" settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0" "github.com/owncloud/ocis/v2/services/graph/pkg/errorcode" + "github.com/owncloud/ocis/v2/services/graph/pkg/middleware" settingsServiceExt "github.com/owncloud/ocis/v2/services/settings/pkg/store/defaults" ) @@ -133,13 +133,6 @@ func (g Graph) GetAllDrives(version APIVersion) http.HandlerFunc { // GetAllDrivesV1 attempts to retrieve the current users drives; // it includes another user's drives, if the current user has the permission. func (g Graph) GetAllDrivesV1(w http.ResponseWriter, r *http.Request) { - if !mfa.Has(r.Context()) { - logger := g.logger.SubloggerWithRequestID(r.Context()) - logger.Error().Str("path", r.URL.Path).Msg("MFA required but not satisfied") - mfa.SetRequiredStatus(w) - return - } - spaces, errCode := g.getDrives(r, true, APIVersion_1) if errCode != nil { errorcode.RenderError(w, r, errCode) @@ -160,13 +153,6 @@ func (g Graph) GetAllDrivesV1(w http.ResponseWriter, r *http.Request) { // it includes the grantedtoV2 property // it uses unified roles instead of the cs3 representations func (g Graph) GetAllDrivesV1Beta1(w http.ResponseWriter, r *http.Request) { - if !mfa.Has(r.Context()) { - logger := g.logger.SubloggerWithRequestID(r.Context()) - logger.Error().Str("path", r.URL.Path).Msg("MFA required but not satisfied") - mfa.SetRequiredStatus(w) - return - } - drives, errCode := g.getDrives(r, true, APIVersion_1_Beta_1) if errCode != nil { errorcode.RenderError(w, r, errCode) @@ -437,6 +423,11 @@ func (g Graph) createDrive(w http.ResponseWriter, r *http.Request, apiVersion AP csr.Owner = us } + // force vault storage space if vault mode is enabled + if middleware.IsVaultMode(ctx) { + csr.Opaque = utils.AppendPlainToOpaque(csr.Opaque, "storage_id", utils.VaultStorageProviderID) + } + resp, err := gatewayClient.CreateStorageSpace(ctx, &csr) if err != nil { logger.Error().Err(err).Msg("could not create drive: transport error") @@ -762,6 +753,7 @@ func (g Graph) ListStorageSpacesWithFilters(ctx context.Context, filters []*stor if err != nil { return nil, err } + lReq := &storageprovider.ListStorageSpacesRequest{ Opaque: &types.Opaque{Map: map[string]*types.OpaqueEntry{ "permissions": { @@ -776,6 +768,11 @@ func (g Graph) ListStorageSpacesWithFilters(ctx context.Context, filters []*stor Filters: filters, } + // force vault storage space if vault mode is enabled + if middleware.IsVaultMode(ctx) { + utils.AppendPlainToOpaque(lReq.Opaque, "storage_id", utils.VaultStorageProviderID) + } + gatewayClient, err := g.gatewaySelector.Next() if err != nil { return nil, err diff --git a/services/graph/pkg/service/v0/graph_test.go b/services/graph/pkg/service/v0/graph_test.go index 1112dacb2bd..f8c58e4b6cc 100644 --- a/services/graph/pkg/service/v0/graph_test.go +++ b/services/graph/pkg/service/v0/graph_test.go @@ -113,7 +113,7 @@ var _ = Describe("Graph", func() { r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/me/drives", nil) r = r.WithContext(ctx) rr := httptest.NewRecorder() - svc.GetDrivesV1(rr, r) + svc.ServeHTTP(rr, r) Expect(rr.Code).To(Equal(http.StatusOK)) }) @@ -126,7 +126,7 @@ var _ = Describe("Graph", func() { r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/drives", nil) r = r.WithContext(mfa.Set(ctx, true)) rr := httptest.NewRecorder() - svc.GetAllDrivesV1(rr, r) + svc.ServeHTTP(rr, r) Expect(rr.Code).To(Equal(http.StatusOK)) }) @@ -138,7 +138,7 @@ var _ = Describe("Graph", func() { r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/drives", nil) rr := httptest.NewRecorder() - svc.GetAllDrivesV1(rr, r) + svc.ServeHTTP(rr, r) Expect(rr.Code).To(Equal(http.StatusForbidden)) Expect(rr.Header().Get("X-Ocis-Mfa-Required")).To(Equal("true")) }) diff --git a/services/graph/pkg/service/v0/service.go b/services/graph/pkg/service/v0/service.go index a24029616f2..8969c960eaf 100644 --- a/services/graph/pkg/service/v0/service.go +++ b/services/graph/pkg/service/v0/service.go @@ -203,6 +203,8 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx requireAdmin = options.RequireAdminMiddleware } + requireMFA := graphm.RequireMFA(options.Logger) + drivesDriveItemService, err := NewDrivesDriveItemService(options.Logger, options.GatewaySelector) if err != nil { return svc, err @@ -223,9 +225,8 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx return svc, err } - m.Route(options.Config.HTTP.Root, func(r chi.Router) { + graphRoutes := func(r chi.Router) { r.Use(middleware.StripSlashes) - r.Route("/v1beta1", func(r chi.Router) { r.Route("/me", func(r chi.Router) { r.Get("/drives", svc.GetDrives(APIVersion_1_Beta_1)) @@ -235,7 +236,7 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx }) }) r.Route("/drives", func(r chi.Router) { - r.Get("/", svc.GetAllDrives(APIVersion_1_Beta_1)) + r.With(requireMFA).Get("/", svc.GetAllDrives(APIVersion_1_Beta_1)) r.Post("/", svc.CreateDriveV1Beta1) r.Route("/{driveID}", func(r chi.Router) { r.Get("/", svc.GetSingleDriveV1Beta1) @@ -331,7 +332,7 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx }) }) r.Route("/drives", func(r chi.Router) { - r.Get("/", svc.GetAllDrives(APIVersion_1)) + r.With(requireMFA).Get("/", svc.GetAllDrives(APIVersion_1)) r.Post("/", svc.CreateDrive) r.Route("/{driveID}", func(r chi.Router) { r.Patch("/", svc.UpdateDrive) @@ -394,7 +395,18 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx }) }) }) - }) + } + + m.Route(options.Config.HTTP.Root, graphRoutes) + + // Ini the Vault routes + if options.Config.EnableVaultMode { + m.Route("/vault/graph", func(r chi.Router) { + r.Use(requireMFA) + r.Use(graphm.VaultModeMiddleware()) + graphRoutes(r) + }) + } _ = chi.Walk(m, func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error { options.Logger.Debug().Str("method", method).Str("route", route).Int("middlewares", len(middlewares)).Msg("serving endpoint") diff --git a/services/graph/pkg/service/v0/sharedbyme.go b/services/graph/pkg/service/v0/sharedbyme.go index 0daf1fa2e6d..3666696ca8e 100644 --- a/services/graph/pkg/service/v0/sharedbyme.go +++ b/services/graph/pkg/service/v0/sharedbyme.go @@ -5,8 +5,11 @@ import ( "github.com/go-chi/render" libregraph "github.com/owncloud/libre-graph-api-go" + "github.com/owncloud/reva/v2/pkg/storagespace" + "github.com/owncloud/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/services/graph/pkg/errorcode" + "github.com/owncloud/ocis/v2/services/graph/pkg/middleware" ) type driveItemsByResourceID map[string]libregraph.DriveItem @@ -39,8 +42,15 @@ func (g Graph) GetSharedByMe(w http.ResponseWriter, r *http.Request) { } res := make([]libregraph.DriveItem, 0, len(driveItems)) + isVault := middleware.IsVaultMode(ctx) for _, v := range driveItems { - res = append(res, v) + storageID, _ := storagespace.SplitStorageID(v.GetId()) + // filters out shares that are not relevant to the current mode (vault or regular). + if isVault && storageID == utils.VaultStorageProviderID { + res = append(res, v) + } else if !isVault && storageID != utils.VaultStorageProviderID { + res = append(res, v) + } } render.Status(r, http.StatusOK) diff --git a/services/graph/pkg/service/v0/sharedwithme.go b/services/graph/pkg/service/v0/sharedwithme.go index 0a51d8b0880..a25ffadd702 100644 --- a/services/graph/pkg/service/v0/sharedwithme.go +++ b/services/graph/pkg/service/v0/sharedwithme.go @@ -8,8 +8,10 @@ import ( ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" "github.com/go-chi/render" libregraph "github.com/owncloud/libre-graph-api-go" + "github.com/owncloud/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/services/graph/pkg/errorcode" + "github.com/owncloud/ocis/v2/services/graph/pkg/middleware" "github.com/owncloud/ocis/v2/services/graph/pkg/unifiedrole" ) @@ -40,6 +42,9 @@ func (g Graph) listSharedWithMe(ctx context.Context) ([]libregraph.DriveItem, er g.logger.Error().Err(err).Msg("listing shares failed") return nil, err } + + listReceivedSharesResponse.Shares = filterVaultShares(ctx, listReceivedSharesResponse.GetShares()) + availableRoles := unifiedrole.GetRoles(unifiedrole.RoleFilterIDs(g.config.UnifiedRoles.AvailableRoles...)) driveItems, err := cs3ReceivedSharesToDriveItems(ctx, g.logger, gatewayClient, g.identityCache, listReceivedSharesResponse.GetShares(), availableRoles) if err != nil { @@ -63,3 +68,17 @@ func (g Graph) listSharedWithMe(ctx context.Context) ([]libregraph.DriveItem, er return driveItems, err } + +// filterVaultShares filters out shares that are not relevant to the current mode (vault or regular). +func filterVaultShares(ctx context.Context, shares []*collaboration.ReceivedShare) []*collaboration.ReceivedShare { + result := make([]*collaboration.ReceivedShare, 0, len(shares)) + isVault := middleware.IsVaultMode(ctx) + for _, share := range shares { + if isVault && share.GetShare().GetResourceId().StorageId == utils.VaultStorageProviderID { + result = append(result, share) + } else if !isVault && share.GetShare().GetResourceId().StorageId != utils.VaultStorageProviderID { + result = append(result, share) + } + } + return result +} diff --git a/services/graph/pkg/service/v0/spacetemplates.go b/services/graph/pkg/service/v0/spacetemplates.go index 05b7ad02461..0f5e9798fba 100644 --- a/services/graph/pkg/service/v0/spacetemplates.go +++ b/services/graph/pkg/service/v0/spacetemplates.go @@ -13,6 +13,7 @@ import ( v1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/owncloud/ocis/v2/ocis-pkg/l10n" l10n_pkg "github.com/owncloud/ocis/v2/services/graph/pkg/l10n" + "github.com/owncloud/ocis/v2/services/graph/pkg/middleware" "github.com/owncloud/reva/v2/pkg/storage/utils/metadata" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" @@ -53,7 +54,11 @@ func (g Graph) applySpaceTemplate(ctx context.Context, gwc gateway.GatewayAPICli } func (g Graph) applyDefaultTemplate(ctx context.Context, gwc gateway.GatewayAPIClient, root *storageprovider.ResourceId, locale string) error { - mdc := metadata.NewCS3(g.config.Reva.Address, g.config.Spaces.StorageUsersAddress) + storageUsersAddress := g.config.Spaces.StorageUsersAddress + if middleware.IsVaultMode(ctx) { + storageUsersAddress = storageUsersAddress + "-vault" + } + mdc := metadata.NewCS3(g.config.Reva.Address, storageUsersAddress) mdc.SpaceRoot = root var opaque *v1beta1.Opaque diff --git a/services/policies/pkg/service/event/service.go b/services/policies/pkg/service/event/service.go index 69f035eebd2..defbd60fa14 100644 --- a/services/policies/pkg/service/event/service.go +++ b/services/policies/pkg/service/event/service.go @@ -125,6 +125,7 @@ func (s Service) processEvent(e events.Event) error { if err := events.Publish(ctx, s.stream, events.PostprocessingStepFinished{ Outcome: outcome, UploadID: ev.UploadID, + ResourceID: ev.ResourceID, ExecutingUser: ev.ExecutingUser, Filename: ev.Filename, FinishedStep: ev.StepToStart, diff --git a/services/postprocessing/pkg/postprocessing/postprocessing.go b/services/postprocessing/pkg/postprocessing/postprocessing.go index aca4ea3e86d..d067dcbe34b 100644 --- a/services/postprocessing/pkg/postprocessing/postprocessing.go +++ b/services/postprocessing/pkg/postprocessing/postprocessing.go @@ -119,6 +119,7 @@ func (pp *Postprocessing) finished(outcome events.PostprocessingOutcome) events. UploadID: pp.ID, ExecutingUser: pp.User, Filename: pp.Filename, + ResourceID: pp.ResourceID, Outcome: outcome, ImpersonatingUser: pp.ImpersonatingUser, } diff --git a/services/proxy/pkg/command/server.go b/services/proxy/pkg/command/server.go index c0a08116358..348f10a87f8 100644 --- a/services/proxy/pkg/command/server.go +++ b/services/proxy/pkg/command/server.go @@ -373,6 +373,7 @@ func loadMiddlewares(logger log.Logger, cfg *config.Config, middleware.Logger(logger), middleware.WithRevaGatewaySelector(gatewaySelector), middleware.RoleQuotas(cfg.RoleQuotas), + middleware.CreateVaultHome(cfg.CreateVaultHome), ), // trigger space assignment when a user logs in middleware.SpaceManager( diff --git a/services/proxy/pkg/config/config.go b/services/proxy/pkg/config/config.go index 9ce6faf3f1d..7a2684c9136 100644 --- a/services/proxy/pkg/config/config.go +++ b/services/proxy/pkg/config/config.go @@ -48,6 +48,7 @@ type Config struct { ClaimSpaceManagement ClaimSpaceManagement `yaml:"claim_space_management"` MultiFactorAuthentication MFAConfig `yaml:"mfa"` MultiInstance MultiInstanceConfig `yaml:"multi_instance"` + CreateVaultHome bool `yaml:"create_vault_home" env:"PROXY_CREATE_VAULT_HOME" desc:"Set this to true to automatically create a new vault home for the user if it does not exist." introductionVersion:"daledda"` Context context.Context `json:"-" yaml:"-"` } diff --git a/services/proxy/pkg/config/defaults/defaultconfig.go b/services/proxy/pkg/config/defaults/defaultconfig.go index 820590f6f29..a1a97b454f1 100644 --- a/services/proxy/pkg/config/defaults/defaultconfig.go +++ b/services/proxy/pkg/config/defaults/defaultconfig.go @@ -273,6 +273,10 @@ func DefaultPolicies() []config.Policy { Endpoint: "/graph/", Service: "com.owncloud.web.graph", }, + { + Endpoint: "/vault/graph/", + Service: "com.owncloud.web.graph", + }, { Endpoint: "/api/v0/settings", Service: "com.owncloud.web.settings", diff --git a/services/proxy/pkg/middleware/create_home.go b/services/proxy/pkg/middleware/create_home.go index a71f3825354..cc61e57013c 100644 --- a/services/proxy/pkg/middleware/create_home.go +++ b/services/proxy/pkg/middleware/create_home.go @@ -3,16 +3,18 @@ package middleware import ( "net/http" "strconv" + "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/jellydator/ttlcache/v3" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/graph/pkg/errorcode" revactx "github.com/owncloud/reva/v2/pkg/ctx" - "github.com/owncloud/reva/v2/pkg/rgrpc/status" "github.com/owncloud/reva/v2/pkg/rgrpc/todo/pool" + "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" "google.golang.org/grpc/metadata" ) @@ -22,12 +24,20 @@ func CreateHome(optionSetters ...Option) func(next http.Handler) http.Handler { options := newOptions(optionSetters...) logger := options.Logger + cache := ttlcache.New( + ttlcache.WithTTL[string, struct{}](60*time.Second), + ttlcache.WithDisableTouchOnHit[string, struct{}](), + ) + go cache.Start() + return func(next http.Handler) http.Handler { return &createHome{ next: next, logger: logger, revaGatewaySelector: options.RevaGatewaySelector, roleQuotas: options.RoleQuotas, + createVaultHome: options.CreateVaultHome, + cache: cache, } } } @@ -37,6 +47,8 @@ type createHome struct { logger log.Logger revaGatewaySelector pool.Selectable[gateway.GatewayAPIClient] roleQuotas map[string]uint64 + createVaultHome bool + cache *ttlcache.Cache[string, struct{}] } func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -48,34 +60,65 @@ func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) { token := req.Header.Get("x-access-token") // we need to pass the token to authenticate the CreateHome request. - //ctx := tokenpkg.ContextSetToken(r.Context(), token) ctx := metadata.AppendToOutgoingContext(req.Context(), revactx.TokenHeader, token) createHomeReq := &provider.CreateHomeRequest{} u, ok := revactx.ContextGetUser(ctx) - if ok { - roleIDs, err := m.getUserRoles(u) - if err != nil { - m.logger.Error().Err(err).Str("userid", u.Id.OpaqueId).Msg("failed to get roles for user") - errorcode.GeneralException.Render(w, req, http.StatusInternalServerError, "Unauthorized") - return - } - if limit, hasLimit := m.checkRoleQuotaLimit(roleIDs); hasLimit { - createHomeReq.Opaque = utils.AppendPlainToOpaque(nil, "quota", strconv.FormatUint(limit, 10)) - } + if !ok || u == nil { + m.logger.Error().Msg("no user in context") + m.next.ServeHTTP(w, req) + return + } + roleIDs, err := m.getUserRoles(u) + if err != nil { + m.logger.Error().Err(err).Str("userid", u.Id.OpaqueId).Msg("failed to get roles for user") + errorcode.GeneralException.Render(w, req, http.StatusInternalServerError, "Unauthorized") + return + } + if limit, hasLimit := m.checkRoleQuotaLimit(roleIDs); hasLimit { + createHomeReq.Opaque = utils.AppendPlainToOpaque(nil, "quota", strconv.FormatUint(limit, 10)) } client, err := m.revaGatewaySelector.Next() if err != nil { m.logger.Err(err).Msg("error selecting next gateway client") } else { - createHomeRes, err := client.CreateHome(ctx, createHomeReq) - if err != nil { - m.logger.Err(err).Msg("error calling CreateHome") - } else if createHomeRes.Status.Code != rpc.Code_CODE_OK { - err := status.NewErrorFromCode(createHomeRes.Status.Code, "gateway") - if createHomeRes.Status.Code != rpc.Code_CODE_ALREADY_EXISTS { - m.logger.Err(err).Msg("error when calling Createhome") + key := u.GetId().GetOpaqueId() + if !m.cache.Has(key) { + createHomeRes, err := client.CreateHome(ctx, createHomeReq) + switch { + case err != nil: + m.logger.Err(err).Msg("error calling CreateHome") + case createHomeRes.GetStatus().GetCode() == rpc.Code_CODE_OK: + m.logger.Debug().Interface("userID", u.GetId().GetOpaqueId()).Msg("personal space created") + m.cache.Set(key, struct{}{}, 0) + case createHomeRes.GetStatus().GetCode() == rpc.Code_CODE_ALREADY_EXISTS: + m.logger.Info().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", createHomeRes.GetStatus()).Msg("personal space already exists") + m.cache.Set(key, struct{}{}, 0) + default: + m.logger.Error().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", createHomeRes.GetStatus()).Msg("personal space creation failed") + } + } + + if m.createVaultHome { + vaultKey := storagespace.FormatStorageID(utils.VaultStorageProviderID, u.GetId().GetOpaqueId()) + if !m.cache.Has(vaultKey) { + // Create vault personal space + // Inject storage_id into opaque for vault personal space + createHomeReq.Opaque = utils.AppendPlainToOpaque(createHomeReq.Opaque, "storage_id", utils.VaultStorageProviderID) + cpsRes, err := client.CreateHome(ctx, createHomeReq) + switch { + case err != nil: + m.logger.Err(err).Msg("error calling CreateHome for vault personal") + case cpsRes.GetStatus().GetCode() == rpc.Code_CODE_OK: + m.logger.Debug().Interface("userID", u.GetId().GetOpaqueId()).Msg("vault personal space created") + m.cache.Set(vaultKey, struct{}{}, 0) + case cpsRes.GetStatus().GetCode() == rpc.Code_CODE_ALREADY_EXISTS: + m.logger.Info().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", cpsRes.GetStatus()).Msg("vault personal space already exists") + m.cache.Set(vaultKey, struct{}{}, 0) + default: + m.logger.Error().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", cpsRes.GetStatus()).Msg("vault personal space creation failed") + } } } } diff --git a/services/proxy/pkg/middleware/options.go b/services/proxy/pkg/middleware/options.go index 503273a564e..243d69114c7 100644 --- a/services/proxy/pkg/middleware/options.go +++ b/services/proxy/pkg/middleware/options.go @@ -69,6 +69,8 @@ type Options struct { // RoleQuotas hold userid:quota mappings. These will be used when provisioning new users. // The users will get as much quota as is set for their role. RoleQuotas map[string]uint64 + // CreateVaultHome creates a new vault home for the user if it does not exist. + CreateVaultHome bool // TraceProvider sets the tracing provider. TraceProvider trace.TracerProvider // SkipUserInfo prevents the oidc middleware from querying the userinfo endpoint and read any claims directly from the access token instead @@ -243,6 +245,13 @@ func RoleQuotas(roleQuotas map[string]uint64) Option { } } +// CreateVaultHome sets the create vault home flag +func CreateVaultHome(createVaultHome bool) Option { + return func(o *Options) { + o.CreateVaultHome = createVaultHome + } +} + // TraceProvider sets the tracing provider. func TraceProvider(tp trace.TracerProvider) Option { return func(o *Options) { diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index c3bfa18c90e..09526b90b4b 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -45,6 +45,8 @@ type Config struct { MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_USERS_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services." introductionVersion:"5.0"` CliMaxAttemptsRenameFile int `yaml:"max_attempts_rename_file" env:"STORAGE_USERS_CLI_MAX_ATTEMPTS_RENAME_FILE" desc:"The maximum number of attempts to rename a file when a user restores a file to an existing destination with the same name. The minimum value is 100." introductionVersion:"5.0"` + EnableVaultMode bool `yaml:"enable_vault_mode" env:"STORAGE_USERS_ENABLE_VAULT_MODE" desc:"Enable vault mode for the storage-users service runned in addition to the regular storage-users service by owerrwiting the MountID to VaultStorageProviderID. Required the running the storage-users-vault additional service." introductionVersion:"daledda"` + Context context.Context `yaml:"-"` } @@ -215,6 +217,7 @@ type Events struct { TLSRootCaCertPath string `yaml:"tls_root_ca_cert_path" env:"OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE;STORAGE_USERS_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided STORAGE_USERS_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"pre5.0"` EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;STORAGE_USERS_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"pre5.0"` NumConsumers int `yaml:"num_consumers" env:"STORAGE_USERS_EVENTS_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for post-processing files. Multiple consumers increase parallelisation, but will also increase CPU and memory demands. The setting has no effect when the OCIS_ASYNC_UPLOADS is set to false. The default and minimum value is 1." introductionVersion:"pre5.0"` + ConsumerGroup string `yaml:"consumer_group" env:"STORAGE_USERS_EVENTS_CONSUMER_GROUP" desc:"The consumer group name to use for the event consumers. The consumer group name is used to identify the consumers." introductionVersion:"daledda"` AuthUsername string `yaml:"username" env:"OCIS_EVENTS_AUTH_USERNAME;STORAGE_USERS_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"5.0"` AuthPassword string `yaml:"password" env:"OCIS_EVENTS_AUTH_PASSWORD;STORAGE_USERS_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"5.0"` } diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index d6de665deea..f77ef6372f5 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -8,6 +8,7 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/shared" "github.com/owncloud/ocis/v2/ocis-pkg/structs" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" + "github.com/owncloud/reva/v2/pkg/utils" ) // FullDefaultConfig returns a fully initialized default configuration @@ -226,6 +227,11 @@ func EnsureDefaults(cfg *config.Config) { cfg.HTTP.CORS.AllowedOrigins[0] == "https://localhost:9200") { cfg.HTTP.CORS.AllowedOrigins = []string{cfg.Commons.OcisURL} } + + // set mount id to vault storage provider id + if cfg.EnableVaultMode { + cfg.MountID = utils.VaultStorageProviderID + } } // Sanitize sanitized the configuration diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 311e40c7591..9d6da1c1191 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -198,7 +198,8 @@ func Ocis(cfg *config.Config) map[string]interface{} { "cache_auth_password": cfg.IDCache.AuthPassword, }, "events": map[string]interface{}{ - "numconsumers": cfg.Events.NumConsumers, + "numconsumers": cfg.Events.NumConsumers, + "consumer_group": cfg.Events.ConsumerGroup, }, "tokens": map[string]interface{}{ "transfer_shared_secret": cfg.Commons.TransferSecret, @@ -321,7 +322,8 @@ func S3NG(cfg *config.Config) map[string]interface{} { "cache_auth_password": cfg.IDCache.AuthPassword, }, "events": map[string]interface{}{ - "numconsumers": cfg.Events.NumConsumers, + "numconsumers": cfg.Events.NumConsumers, + "consumer_group": cfg.Events.ConsumerGroup, }, "tokens": map[string]interface{}{ "transfer_shared_secret": cfg.Commons.TransferSecret, diff --git a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go index 4a8d15df4f4..c45f26e3061 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go +++ b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go @@ -143,6 +143,12 @@ func (s *svc) CreateHome(ctx context.Context, req *provider.CreateHomeRequest) ( }, } } + + // pass storage_id to the storage provider to handle vault storage id + if storageId := utils.ReadPlainFromOpaque(req.GetOpaque(), "storage_id"); storageId != "" { + createReq.Opaque = utils.AppendPlainToOpaque(createReq.Opaque, "storage_id", storageId) + } + res, err := s.CreateStorageSpace(ctx, createReq) if err != nil { return &provider.CreateHomeResponse{ @@ -170,6 +176,11 @@ func (s *svc) CreateStorageSpace(ctx context.Context, req *provider.CreateStorag } } + if storageId := utils.ReadPlainFromOpaque(req.GetOpaque(), "storage_id"); storageId != "" { + space.Root = &provider.ResourceId{StorageId: storageId} + req.Opaque = utils.AppendPlainToOpaque(req.Opaque, "storage_id", storageId) + } + srClient, err := s.getStorageRegistryClient(ctx, s.c.StorageRegistryEndpoint) if err != nil { return &provider.CreateStorageSpaceResponse{ @@ -247,6 +258,7 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp filters["path"] = path } + hasFileIdFilter := false for _, f := range req.Filters { switch f.Type { case provider.ListStorageSpacesRequest_Filter_TYPE_ID: @@ -255,6 +267,7 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp continue } filters["storage_id"], filters["space_id"], filters["opaque_id"] = sid, spid, oid + hasFileIdFilter = true case provider.ListStorageSpacesRequest_Filter_TYPE_OWNER: filters["owner_idp"] = f.GetOwner().GetIdp() filters["owner_id"] = f.GetOwner().GetOpaqueId() @@ -270,6 +283,10 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp } } + if !hasFileIdFilter && utils.ReadPlainFromOpaque(req.Opaque, "storage_id") != "" { + filters["storage_id"] = utils.ReadPlainFromOpaque(req.Opaque, "storage_id") + } + c, err := s.getStorageRegistryClient(ctx, s.c.StorageRegistryEndpoint) if err != nil { return &provider.ListStorageSpacesResponse{ diff --git a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go index ba76690e235..855a7832cfc 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go +++ b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go @@ -24,8 +24,8 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" - sdk "github.com/owncloud/reva/v2/pkg/sdk/common" "github.com/owncloud/reva/v2/pkg/storage/cache" + "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" "github.com/pkg/errors" "google.golang.org/grpc" @@ -41,15 +41,22 @@ type cachedRegistryClient struct { } func (c *cachedRegistryClient) ListStorageProviders(ctx context.Context, in *registry.ListStorageProvidersRequest, opts ...grpc.CallOption) (*registry.ListStorageProvidersResponse, error) { - - spaceID := sdk.DecodeOpaqueMap(in.Opaque)["space_id"] + spaceID := utils.ReadPlainFromOpaque(in.GetOpaque(), "space_id") + resourceID := spaceID + if storageID := utils.ReadPlainFromOpaque(in.GetOpaque(), "storage_id"); storageID != "" { + if spaceID != "" { + resourceID = storagespace.FormatStorageID(storageID, spaceID) + } else { + resourceID = storageID + } + } u, ok := ctxpkg.ContextGetUser(ctx) if !ok { return nil, errors.New("user not found in context") } - key := c.cache.GetKey(u.GetId(), spaceID) + key := c.cache.GetKey(u.GetId(), resourceID) if key != "" { s := ®istry.ListStorageProvidersResponse{} if err := c.cache.PullFromCache(key, s); err == nil { diff --git a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go index d790bf2c1d5..2bf7025e9b8 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go +++ b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go @@ -33,6 +33,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/mitchellh/mapstructure" "github.com/owncloud/reva/v2/pkg/appctx" "github.com/owncloud/reva/v2/pkg/conversions" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" @@ -47,7 +48,6 @@ import ( "github.com/owncloud/reva/v2/pkg/storage/fs/registry" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/rs/zerolog" "go.opentelemetry.io/otel/attribute" @@ -787,6 +787,7 @@ func (s *Service) Stat(ctx context.Context, req *provider.StatRequest) (*provide s.addMissingStorageProviderID(md.GetId(), nil) s.addMissingStorageProviderID(md.GetParentId(), nil) s.addMissingStorageProviderID(md.GetSpace().GetRoot(), nil) + s.addMissingStorageProviderID(md.GetSpace().GetRootInfo().GetId(), nil) return &provider.StatResponse{ Status: status.NewOK(ctx), diff --git a/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go b/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go index 41c92b936c6..12093c61230 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go +++ b/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go @@ -137,8 +137,7 @@ func (h *Handler) addSpaceMember(w http.ResponseWriter, r *http.Request, info *p response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "error getting storage provider", err) return } - - providerClient, err := h.getStorageProviderClient(p) + providerClient, err := pool.GetStorageProviderServiceClient(p.Address) if err != nil { response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "error getting storage provider client", err) return @@ -244,8 +243,7 @@ func (h *Handler) removeSpaceMember(w http.ResponseWriter, r *http.Request, spac if ref.ResourceId.OpaqueId == "" { ref.ResourceId.OpaqueId = ref.ResourceId.SpaceId } - - providerClient, err := h.getStorageProviderClient(prov) + providerClient, err := pool.GetStorageProviderServiceClient(prov.Address) if err != nil { response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "error getting storage provider client", err) return @@ -290,16 +288,6 @@ func (h *Handler) removeSpaceMember(w http.ResponseWriter, r *http.Request, spac response.WriteOCSSuccess(w, r, nil) } -func (h *Handler) getStorageProviderClient(p *registry.ProviderInfo) (provider.ProviderAPIClient, error) { - c, err := pool.GetStorageProviderServiceClient(p.Address) - if err != nil { - err = errors.Wrap(err, "shares spaces: error getting a storage provider client") - return nil, err - } - - return c, nil -} - func (h *Handler) findProvider(ctx context.Context, ref *provider.Reference) (*registry.ProviderInfo, error) { c, err := pool.GetStorageRegistryClient(h.storageRegistryAddr) if err != nil { diff --git a/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go b/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go index f4268920a3d..64318cb9487 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go @@ -103,6 +103,7 @@ type PostprocessingStepFinished struct { UploadID string ExecutingUser *user.User Filename string + ResourceID *provider.ResourceId FinishedStep Postprocessingstep // name of the step Result interface{} // result information see VirusscanResult for example @@ -145,6 +146,7 @@ type VirusscanResult struct { type PostprocessingFinished struct { UploadID string Filename string + ResourceID *provider.ResourceId SpaceOwner *user.UserId ExecutingUser *user.User Result map[Postprocessingstep]interface{} // it is a map[step]Event diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go index ac586e96d39..d2b0cbc7fb7 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go @@ -34,6 +34,7 @@ import ( providerpb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/mitchellh/mapstructure" "github.com/owncloud/reva/v2/pkg/appctx" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/errtypes" @@ -44,7 +45,6 @@ import ( pkgregistry "github.com/owncloud/reva/v2/pkg/storage/registry/registry" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" - "github.com/mitchellh/mapstructure" "google.golang.org/grpc" ) @@ -195,6 +195,18 @@ func (r *registry) GetProvider(ctx context.Context, space *providerpb.StorageSpa if space.SpaceType != "" && spaceType != space.SpaceType { continue } + + // Filter out vault spaces if no storageId is provided + if space.GetRoot().GetStorageId() != "" { + if space.GetRoot().GetStorageId() != provider.ProviderID { + continue + } + } else { + if strings.HasPrefix(sc.MountPoint, "/vault/") { + continue + } + } + if space.Owner != nil { user := ctxpkg.ContextMustGetUser(ctx) spacePath, err = sc.SpacePath(user, space) @@ -289,7 +301,7 @@ func (r *registry) ListProviders(ctx context.Context, filters map[string]string) // return all providers return r.findAllProviders(ctx, mask), nil default: - return r.findProvidersForFilter(ctx, r.buildFilters(filters), unrestricted, mask), nil + return r.findProvidersForFilter(ctx, r.buildFilters(filters), filters["storage_id"], unrestricted, mask), nil } } @@ -340,7 +352,7 @@ func (r *registry) buildFilters(filterMap map[string]string) []*providerpb.ListS return filters } -func (r *registry) findProvidersForFilter(ctx context.Context, filters []*providerpb.ListStorageSpacesRequest_Filter, unrestricted bool, _ string) []*registrypb.ProviderInfo { +func (r *registry) findProvidersForFilter(ctx context.Context, filters []*providerpb.ListStorageSpacesRequest_Filter, storageId string, unrestricted bool, _ string) []*registrypb.ProviderInfo { var requestedSpaceType string for _, f := range filters { @@ -352,7 +364,10 @@ func (r *registry) findProvidersForFilter(ctx context.Context, filters []*provid currentUser := ctxpkg.ContextMustGetUser(ctx) providerInfos := []*registrypb.ProviderInfo{} for address, provider := range r.c.Providers { - + // skip mismatching storageproviders + if storageId != "" && storageId != provider.ProviderID { + continue + } // when a specific space type is requested we may skip this provider altogether if it is not configured for that type // we have to ignore a space type filter with +grant or +mountpoint type because they can live on any provider if requestedSpaceType != "" && !strings.HasPrefix(requestedSpaceType, "+") { @@ -385,6 +400,10 @@ func (r *registry) findProvidersForFilter(ctx context.Context, filters []*provid if sc, ok = provider.Spaces[space.SpaceType]; !ok { continue } + // Filter out vault spaces if no storageId is provided + if storageId == "" && strings.HasPrefix(sc.MountPoint, "/vault/") { + continue + } spacePath, err = sc.SpacePath(currentUser, space) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("provider", provider).Interface("space", space).Msg("failed to execute template, continuing") diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index 79dcc454a76..c4c4fd1e08f 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -258,7 +258,7 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor return nil, errors.New("need nats for async file processing") } - ch, err := events.Consume(fs.stream, "dcfs", _registeredEvents...) + ch, err := events.Consume(fs.stream, o.Events.ConsumerGroup, _registeredEvents...) if err != nil { return nil, err } @@ -285,6 +285,10 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { switch ev := event.Event.(type) { case events.PostprocessingFinished: sublog := log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { + sublog.Debug().Msg("ignoring event for different storage") + continue + } session, err := fs.sessionStore.Get(ctx, ev.UploadID) if err != nil { sublog.Error().Err(err).Msg("Failed to get upload") @@ -450,6 +454,10 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) case events.RevertRevision: sublog := log.With().Str("event", "RevertRevision").Interface("nodeid", ev.ResourceID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { + sublog.Debug().Msg("ignoring event for different storage") + continue + } n, err := fs.lu.NodeFromID(ctx, ev.ResourceID) if err != nil { sublog.Error().Err(err).Msg("Failed to get node") @@ -462,6 +470,10 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } case events.PostprocessingStepFinished: sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { + sublog.Debug().Msg("ignoring event for different storage") + continue + } if ev.FinishedStep != events.PPStepAntivirus { // atm we are only interested in antivirus results continue diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go index 5c76a383eac..210f2068130 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go @@ -23,10 +23,10 @@ import ( "strings" "time" + "github.com/mitchellh/mapstructure" "github.com/owncloud/reva/v2/pkg/rgrpc/todo/pool" "github.com/owncloud/reva/v2/pkg/sharedconf" "github.com/owncloud/reva/v2/pkg/storage/cache" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -103,7 +103,8 @@ type AsyncPropagatorOptions struct { // EventOptions are the configurable options for events type EventOptions struct { - NumConsumers int `mapstructure:"numconsumers"` + NumConsumers int `mapstructure:"numconsumers"` + ConsumerGroup string `mapstructure:"consumer_group"` } // TokenOptions are the configurable option for tokens @@ -172,5 +173,9 @@ func New(m map[string]interface{}) (*Options, error) { o.UploadDirectory = filepath.Join(o.Root, "uploads") } + if o.Events.ConsumerGroup == "" { + o.Events.ConsumerGroup = "dcfs" + } + return o, nil } diff --git a/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go b/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go index c1031368743..c562636e8b0 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go @@ -64,6 +64,9 @@ var ( // OCMStorageSpaceID is the space id used by the ocmreceived storageprovider OCMStorageSpaceID = "89f37a33-858b-45fa-8890-a1f2b27d90e1" + // VaultStorageProviderID is the storage id used by the vault storageprovider + VaultStorageProviderID = "1a01c2c4-4309-4483-a845-842fd56d8622" + // SpaceGrant is used to signal the storageprovider that the grant is on a space SpaceGrant struct{} ) diff --git a/vendor/modules.txt b/vendor/modules.txt index b9913c1fe98..2ad55e4ef15 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1316,7 +1316,7 @@ github.com/orcaman/concurrent-map # github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245 ## explicit; go 1.18 github.com/owncloud/libre-graph-api-go -# github.com/owncloud/reva/v2 v2.0.0-20260324082555-823c2f1c2593 +# github.com/owncloud/reva/v2 v2.0.0-20260324173335-cc6175484320 ## explicit; go 1.24.0 github.com/owncloud/reva/v2/cmd/revad/internal/grace github.com/owncloud/reva/v2/cmd/revad/runtime