diff --git a/.gitignore b/.gitignore index f9988f0c791..387fa1b0157 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ protogen/buf.sha1.lock /third-party-licenses # misc +.agents/ /tmp go.work go.work.sum diff --git a/deployments/examples/ocis_keycloak/.env b/deployments/examples/ocis_keycloak/.env index 147d87389d1..7daaabc2201 100644 --- a/deployments/examples/ocis_keycloak/.env +++ b/deployments/examples/ocis_keycloak/.env @@ -41,3 +41,6 @@ KEYCLOAK_TRACING= # you need uncomment following line. Please see documentation at # https://owncloud.dev/ocis/deployment/monitoring-tracing/ #COMPOSE_FILE=docker-compose.yml:monitoring_tracing/docker-compose-additions.yml + +# To add vault support to this stack, uncomment the following line: +#COMPOSE_FILE=docker-compose.yml:vault.yml diff --git a/deployments/examples/ocis_keycloak/README.md b/deployments/examples/ocis_keycloak/README.md index 0837071b7c9..1d792db46de 100644 --- a/deployments/examples/ocis_keycloak/README.md +++ b/deployments/examples/ocis_keycloak/README.md @@ -4,3 +4,16 @@ document this deployment example in: docs/ocis/deployment/ocis_keycloak.md Please refer to [our documentation](https://owncloud.dev/ocis/deployment/ocis_keycloak/) for instructions on how to deploy this scenario. + + +## Vault + +Adds vault sidecar services (`graph-vault`, `storage-users-vault`) to `ocis_keycloak`. + +### Running + +Uncomment in `.env` or run directly: + +```bash +docker compose -f docker-compose.yml -f vault.yml up -d +``` diff --git a/deployments/examples/ocis_keycloak/config/keycloak/ocis-realm.dist.json b/deployments/examples/ocis_keycloak/config/keycloak/ocis-realm.dist.json index 63200a3ac07..0bae69707d9 100644 --- a/deployments/examples/ocis_keycloak/config/keycloak/ocis-realm.dist.json +++ b/deployments/examples/ocis_keycloak/config/keycloak/ocis-realm.dist.json @@ -2277,7 +2277,7 @@ "requirement": "REQUIRED", "priority": 1, "autheticatorFlow": false, - "userSetupAllowed": false + "userSetupAllowed": true } ] }, diff --git a/deployments/examples/ocis_keycloak/docker-compose.yml b/deployments/examples/ocis_keycloak/docker-compose.yml index f9881577a3c..b8af057da3d 100644 --- a/deployments/examples/ocis_keycloak/docker-compose.yml +++ b/deployments/examples/ocis_keycloak/docker-compose.yml @@ -83,11 +83,13 @@ services: KEYCLOAK_DOMAIN: ${KEYCLOAK_DOMAIN:-keycloak.owncloud.test} OCIS_MFA_ENABLED: ${OCIS_MFA_ENABLED:-false} WEB_OIDC_SCOPE: "openid profile email acr" + # WEB_ASSET_CORE_PATH: /web/dist # local web dist volumes: - ./config/ocis/banned-password-list.txt:/etc/ocis/banned-password-list.txt - ./config/ocis/csp.yaml:/etc/ocis/csp.yaml - ocis-config:/etc/ocis - ocis-data:/var/lib/ocis + # - /Users/mk/dev/kiteworks/web/dist:/web/dist # local web dist labels: - "traefik.enable=true" - "traefik.http.routers.ocis.entrypoints=https" @@ -109,6 +111,11 @@ services: POSTGRES_DB: keycloak POSTGRES_USER: keycloak POSTGRES_PASSWORD: keycloak + healthcheck: + test: ["CMD-SHELL", "pg_isready -U keycloak"] + interval: 5s + timeout: 5s + retries: 10 logging: driver: ${LOG_DRIVER:-local} restart: always @@ -138,6 +145,13 @@ services: # tracing KC_TRACING_ENABLED: ${KEYCLOAK_TRACING:-false} KC_TRACING_ENDPOINT: http://jaeger:4317 + KC_HEALTH_ENABLED: "true" + healthcheck: + test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/9000 && echo -e 'GET /health/ready HTTP/1.0\\r\\n\\r\\n' >&3 && grep -q '\"status\": \"UP\"' <&3"] + interval: 10s + timeout: 5s + retries: 18 + start_period: 30s labels: - "traefik.enable=true" - "traefik.http.routers.keycloak.entrypoints=https" @@ -146,7 +160,8 @@ services: - "traefik.http.routers.keycloak.service=keycloak" - "traefik.http.services.keycloak.loadbalancer.server.port=8080" depends_on: - - postgres + postgres: + condition: service_healthy logging: driver: ${LOG_DRIVER:-local} restart: always diff --git a/deployments/examples/ocis_keycloak/vault.yml b/deployments/examples/ocis_keycloak/vault.yml new file mode 100644 index 00000000000..1d7ef0e5436 --- /dev/null +++ b/deployments/examples/ocis_keycloak/vault.yml @@ -0,0 +1,48 @@ +--- +version: "3.7" + +services: + ocis: + environment: + # Vault: expose internal services so sidecar containers can reach them + NATS_NATS_HOST: 0.0.0.0 + OCIS_GATEWAY_GRPC_ADDR: 0.0.0.0:9142 + # Vault: auto-create vault home space on first user login + PROXY_CREATE_VAULT_HOME: "true" + GRAPH_ENABLE_VAULT_MODE: "true" + # MFA: guard Admin Settings with step-up auth (LOA2) + # OCIS_MFA_ENABLED: "true" + + storage-users-vault: + image: ${OCIS_DOCKER_IMAGE:-owncloud/ocis}:${OCIS_DOCKER_TAG:-latest} + networks: + ocis-net: + depends_on: + ocis: + condition: service_started + command: ["storage-users", "server"] + environment: + OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-info} + OCIS_LOG_COLOR: "${OCIS_LOG_COLOR:-false}" + OCIS_GATEWAY_GRPC_ADDR: ocis:9142 + STORAGE_USERS_ENABLE_VAULT_MODE: "true" + STORAGE_USERS_SERVICE_NAME: storage-users-vault + STORAGE_USERS_GRPC_ADDR: 0.0.0.0:9170 + STORAGE_USERS_HTTP_ADDR: 0.0.0.0:9168 + STORAGE_USERS_DATA_SERVER_URL: http://storage-users-vault:9168/data + STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9169 + STORAGE_USERS_OCIS_ROOT: /var/lib/ocis/storage/users-vault + STORAGE_USERS_EVENTS_CONSUMER_GROUP: vault-dcfs + MICRO_REGISTRY_ADDRESS: ocis:9233 + OCIS_EVENTS_ENDPOINT: ocis:9233 + OCIS_CACHE_STORE_NODES: ocis:9233 + OCIS_INSECURE: "${INSECURE:-false}" + volumes: + - ocis-data:/var/lib/ocis + - ocis-config:/etc/ocis + logging: + driver: ${LOG_DRIVER:-local} + restart: always + +networks: + ocis-net: diff --git a/go.mod b/go.mod index 50e092b2fdd..d90ad2392b2 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( github.com/open-policy-agent/opa v1.12.3 github.com/orcaman/concurrent-map v1.0.0 github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245 - github.com/owncloud/reva/v2 v2.0.0-20260304131727-ce3149532498 + github.com/owncloud/reva/v2 v2.0.0-20260316140824-a145e1807968 github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.12 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index 796f00f0a12..de0ac894349 100644 --- a/go.sum +++ b/go.sum @@ -742,8 +742,8 @@ github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HD github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245 h1:JRidLTAKhnvyLMRtVtSF4lhBa0NSAOs6fof+d6JnKII= github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245/go.mod h1:z61VMGAJRtR1nbgXWiNoCkxUXP1B3Je9rMuJbnGd+Og= -github.com/owncloud/reva/v2 v2.0.0-20260304131727-ce3149532498 h1:ozyudPff1XLWSeTDW4ajT1tCoSQf18cFQRxtLWFCNbk= -github.com/owncloud/reva/v2 v2.0.0-20260304131727-ce3149532498/go.mod h1:+rCy6oGYb2/qs5gmQa8y/pHARw634vB73MZGDY2SBIQ= +github.com/owncloud/reva/v2 v2.0.0-20260316140824-a145e1807968 h1:p5NMF+1kuaZCxMJCf00akd6w+gXAVmnIz0yUzWDmWK8= +github.com/owncloud/reva/v2 v2.0.0-20260316140824-a145e1807968/go.mod h1:+rCy6oGYb2/qs5gmQa8y/pHARw634vB73MZGDY2SBIQ= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pablodz/inotifywaitgo v0.0.9 h1:njquRbBU7fuwIe5rEvtaniVBjwWzcpdUVptSgzFqZsw= diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 53699fccce0..92f44c19f4e 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -85,18 +85,11 @@ type StorageRegistry struct { // Cache holds cache config type Cache struct { - ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"pre5.0"` - ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"pre5.0"` - ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"5.0"` - ProviderCacheAuthUsername string `yaml:"provider_cache_auth_username" env:"OCIS_CACHE_AUTH_USERNAME;GATEWAY_PROVIDER_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` - ProviderCacheAuthPassword string `yaml:"provider_cache_auth_password" env:"OCIS_CACHE_AUTH_PASSWORD;GATEWAY_PROVIDER_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` - CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"pre5.0"` - CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"pre5.0"` - CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` - CreateHomeCacheDisablePersistence bool `yaml:"create_home_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_CREATE_HOME_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the create home cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"5.0"` - CreateHomeCacheAuthUsername string `yaml:"create_home_cache_auth_username" env:"OCIS_CACHE_AUTH_USERNAME;GATEWAY_CREATE_HOME_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` - CreateHomeCacheAuthPassword string `yaml:"create_home_cache_auth_password" env:"OCIS_CACHE_AUTH_PASSWORD;GATEWAY_CREATE_HOME_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` + ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"pre5.0"` + ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` + ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"pre5.0"` + ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` + ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"5.0"` + ProviderCacheAuthUsername string `yaml:"provider_cache_auth_username" env:"OCIS_CACHE_AUTH_USERNAME;GATEWAY_PROVIDER_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` + ProviderCacheAuthPassword string `yaml:"provider_cache_auth_password" env:"OCIS_CACHE_AUTH_PASSWORD;GATEWAY_PROVIDER_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"` } diff --git a/services/gateway/pkg/config/defaults/defaultconfig.go b/services/gateway/pkg/config/defaults/defaultconfig.go index 5d003bb3790..4bab0339312 100644 --- a/services/gateway/pkg/config/defaults/defaultconfig.go +++ b/services/gateway/pkg/config/defaults/defaultconfig.go @@ -39,14 +39,10 @@ func DefaultConfig() *config.Config { DisableHomeCreationOnLogin: true, TransferExpires: 24 * 60 * 60, Cache: config.Cache{ - ProviderCacheStore: "noop", - ProviderCacheNodes: []string{"127.0.0.1:9233"}, - ProviderCacheDatabase: "cache-providers", - ProviderCacheTTL: 300 * time.Second, - CreateHomeCacheStore: "memory", - CreateHomeCacheNodes: []string{"127.0.0.1:9233"}, - CreateHomeCacheDatabase: "cache-createhome", - CreateHomeCacheTTL: 300 * time.Second, + ProviderCacheStore: "noop", + ProviderCacheNodes: []string{"127.0.0.1:9233"}, + ProviderCacheDatabase: "cache-providers", + ProviderCacheTTL: 300 * time.Second, }, FrontendPublicURL: "https://localhost:9200", diff --git a/services/gateway/pkg/revaconfig/config.go b/services/gateway/pkg/revaconfig/config.go index d4589e5d7fb..a24a9a0e371 100644 --- a/services/gateway/pkg/revaconfig/config.go +++ b/services/gateway/pkg/revaconfig/config.go @@ -75,16 +75,6 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]i "cache_auth_username": cfg.Cache.ProviderCacheAuthUsername, "cache_auth_password": cfg.Cache.ProviderCacheAuthPassword, }, - "create_personal_space_cache_config": map[string]interface{}{ - "cache_store": cfg.Cache.CreateHomeCacheStore, - "cache_nodes": cfg.Cache.CreateHomeCacheNodes, - "cache_database": cfg.Cache.CreateHomeCacheDatabase, - "cache_table": "create_personal_space", - "cache_ttl": cfg.Cache.CreateHomeCacheTTL, - "cache_disable_persistence": cfg.Cache.CreateHomeCacheDisablePersistence, - "cache_auth_username": cfg.Cache.CreateHomeCacheAuthUsername, - "cache_auth_password": cfg.Cache.CreateHomeCacheAuthPassword, - }, }, "authregistry": map[string]interface{}{ "driver": "static", @@ -162,6 +152,22 @@ func spacesProviders(cfg *config.Config, logger log.Logger) map[string]map[strin }, }, }, + "com.owncloud.api.storage-users-vault": { + // Use the dedicated storage provider for vault + "providerid": utils.VaultStorageProviderID, + "spaces": map[string]interface{}{ + "personal": map[string]interface{}{ + // The mount point must have the "vault/" prefix to be picked up by the vault storage provider + "mount_point": "/vault/users", + "path_template": "/vault/users/{{.Space.Owner.Id.OpaqueId}}", + }, + "project": map[string]interface{}{ + // The mount point must have the "vault/" prefix to be picked up by the vault storage provider + "mount_point": "/vault/projects", + "path_template": "/vault/projects/{{.Space.Name}}", + }, + }, + }, cfg.StorageSharesEndpoint: { "providerid": utils.ShareStorageProviderID, "spaces": map[string]interface{}{ diff --git a/services/graph/pkg/config/config.go b/services/graph/pkg/config/config.go index ab0072af375..813d24382b4 100644 --- a/services/graph/pkg/config/config.go +++ b/services/graph/pkg/config/config.go @@ -39,6 +39,8 @@ type Config struct { Validation Validation `yaml:"validation"` + EnableVaultMode bool `yaml:"enable_vault_mode" env:"GRAPH_ENABLE_VAULT_MODE" desc:"Enable vault mode for the graph service runned in addition to the regular graph service. Required the running the storage-users-vault additional service." introductionVersion:"daledda"` + Context context.Context `yaml:"-"` } diff --git a/services/graph/pkg/config/service.go b/services/graph/pkg/config/service.go index d1eac383f0b..f7edce2b7dd 100644 --- a/services/graph/pkg/config/service.go +++ b/services/graph/pkg/config/service.go @@ -2,5 +2,5 @@ package config // Service defines the available service configuration. type Service struct { - Name string `yaml:"-"` + Name string `yaml:"name" env:"GRAPH_SERVICE_NAME" desc:"The name of the service." introductionVersion:"daledda"` } diff --git a/services/graph/pkg/middleware/mfa.go b/services/graph/pkg/middleware/mfa.go new file mode 100644 index 00000000000..33c1206e09b --- /dev/null +++ b/services/graph/pkg/middleware/mfa.go @@ -0,0 +1,23 @@ +package middleware + +import ( + "net/http" + + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/ocis-pkg/mfa" +) + +// RequireMFA middleware is used to require the user in context to have MFA satisfied +func RequireMFA(logger log.Logger) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !mfa.Has(r.Context()) { + l := logger.SubloggerWithRequestID(r.Context()) + l.Error().Str("path", r.URL.Path).Msg("MFA required but not satisfied") + mfa.SetRequiredStatus(w) + return + } + next.ServeHTTP(w, r) + }) + } +} diff --git a/services/graph/pkg/service/v0/driveitems.go b/services/graph/pkg/service/v0/driveitems.go index 2a134500cab..7d8124397d1 100644 --- a/services/graph/pkg/service/v0/driveitems.go +++ b/services/graph/pkg/service/v0/driveitems.go @@ -158,6 +158,11 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) { filters = append(filters, listStorageSpacesUserFilter(currentUser.GetId().GetOpaqueId())) filters = append(filters, listStorageSpacesTypeFilter("personal")) + // force vault storage space if vault mode is enabled + if g.config.EnableVaultMode { + filters = append(filters, listStorageSpacesIDFilter(utils.VaultStorageProviderID)) + } + res, err := gatewayClient.ListStorageSpaces(ctx, &storageprovider.ListStorageSpacesRequest{ Filters: filters, }) diff --git a/services/graph/pkg/service/v0/drives.go b/services/graph/pkg/service/v0/drives.go index c4657b32054..9cfc5dc364e 100644 --- a/services/graph/pkg/service/v0/drives.go +++ b/services/graph/pkg/service/v0/drives.go @@ -29,7 +29,6 @@ import ( "google.golang.org/protobuf/proto" "github.com/owncloud/ocis/v2/ocis-pkg/l10n" - "github.com/owncloud/ocis/v2/ocis-pkg/mfa" v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0" settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0" "github.com/owncloud/ocis/v2/services/graph/pkg/errorcode" @@ -133,13 +132,6 @@ func (g Graph) GetAllDrives(version APIVersion) http.HandlerFunc { // GetAllDrivesV1 attempts to retrieve the current users drives; // it includes another user's drives, if the current user has the permission. func (g Graph) GetAllDrivesV1(w http.ResponseWriter, r *http.Request) { - if !mfa.Has(r.Context()) { - logger := g.logger.SubloggerWithRequestID(r.Context()) - logger.Error().Str("path", r.URL.Path).Msg("MFA required but not satisfied") - mfa.SetRequiredStatus(w) - return - } - spaces, errCode := g.getDrives(r, true, APIVersion_1) if errCode != nil { errorcode.RenderError(w, r, errCode) @@ -160,13 +152,6 @@ func (g Graph) GetAllDrivesV1(w http.ResponseWriter, r *http.Request) { // it includes the grantedtoV2 property // it uses unified roles instead of the cs3 representations func (g Graph) GetAllDrivesV1Beta1(w http.ResponseWriter, r *http.Request) { - if !mfa.Has(r.Context()) { - logger := g.logger.SubloggerWithRequestID(r.Context()) - logger.Error().Str("path", r.URL.Path).Msg("MFA required but not satisfied") - mfa.SetRequiredStatus(w) - return - } - drives, errCode := g.getDrives(r, true, APIVersion_1_Beta_1) if errCode != nil { errorcode.RenderError(w, r, errCode) @@ -437,6 +422,11 @@ func (g Graph) createDrive(w http.ResponseWriter, r *http.Request, apiVersion AP csr.Owner = us } + // force vault storage space if vault mode is enabled + if g.config.EnableVaultMode { + csr.Opaque = utils.AppendPlainToOpaque(csr.Opaque, "storage_id", utils.VaultStorageProviderID) + } + resp, err := gatewayClient.CreateStorageSpace(ctx, &csr) if err != nil { logger.Error().Err(err).Msg("could not create drive: transport error") @@ -762,6 +752,7 @@ func (g Graph) ListStorageSpacesWithFilters(ctx context.Context, filters []*stor if err != nil { return nil, err } + lReq := &storageprovider.ListStorageSpacesRequest{ Opaque: &types.Opaque{Map: map[string]*types.OpaqueEntry{ "permissions": { @@ -776,6 +767,11 @@ func (g Graph) ListStorageSpacesWithFilters(ctx context.Context, filters []*stor Filters: filters, } + // force vault storage space if vault mode is enabled + if g.config.EnableVaultMode { + utils.AppendPlainToOpaque(lReq.Opaque, "storage_id", utils.VaultStorageProviderID) + } + gatewayClient, err := g.gatewaySelector.Next() if err != nil { return nil, err diff --git a/services/graph/pkg/service/v0/graph_test.go b/services/graph/pkg/service/v0/graph_test.go index 1112dacb2bd..f8c58e4b6cc 100644 --- a/services/graph/pkg/service/v0/graph_test.go +++ b/services/graph/pkg/service/v0/graph_test.go @@ -113,7 +113,7 @@ var _ = Describe("Graph", func() { r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/me/drives", nil) r = r.WithContext(ctx) rr := httptest.NewRecorder() - svc.GetDrivesV1(rr, r) + svc.ServeHTTP(rr, r) Expect(rr.Code).To(Equal(http.StatusOK)) }) @@ -126,7 +126,7 @@ var _ = Describe("Graph", func() { r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/drives", nil) r = r.WithContext(mfa.Set(ctx, true)) rr := httptest.NewRecorder() - svc.GetAllDrivesV1(rr, r) + svc.ServeHTTP(rr, r) Expect(rr.Code).To(Equal(http.StatusOK)) }) @@ -138,7 +138,7 @@ var _ = Describe("Graph", func() { r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/drives", nil) rr := httptest.NewRecorder() - svc.GetAllDrivesV1(rr, r) + svc.ServeHTTP(rr, r) Expect(rr.Code).To(Equal(http.StatusForbidden)) Expect(rr.Header().Get("X-Ocis-Mfa-Required")).To(Equal("true")) }) diff --git a/services/graph/pkg/service/v0/service.go b/services/graph/pkg/service/v0/service.go index a24029616f2..33645af271a 100644 --- a/services/graph/pkg/service/v0/service.go +++ b/services/graph/pkg/service/v0/service.go @@ -203,6 +203,8 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx requireAdmin = options.RequireAdminMiddleware } + requireMFA := graphm.RequireMFA(options.Logger) + drivesDriveItemService, err := NewDrivesDriveItemService(options.Logger, options.GatewaySelector) if err != nil { return svc, err @@ -225,6 +227,9 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx m.Route(options.Config.HTTP.Root, func(r chi.Router) { r.Use(middleware.StripSlashes) + if options.Config.EnableVaultMode { + r.Use(requireMFA) + } r.Route("/v1beta1", func(r chi.Router) { r.Route("/me", func(r chi.Router) { @@ -235,7 +240,7 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx }) }) r.Route("/drives", func(r chi.Router) { - r.Get("/", svc.GetAllDrives(APIVersion_1_Beta_1)) + r.With(requireMFA).Get("/", svc.GetAllDrives(APIVersion_1_Beta_1)) r.Post("/", svc.CreateDriveV1Beta1) r.Route("/{driveID}", func(r chi.Router) { r.Get("/", svc.GetSingleDriveV1Beta1) @@ -331,7 +336,7 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx }) }) r.Route("/drives", func(r chi.Router) { - r.Get("/", svc.GetAllDrives(APIVersion_1)) + r.With(requireMFA).Get("/", svc.GetAllDrives(APIVersion_1)) r.Post("/", svc.CreateDrive) r.Route("/{driveID}", func(r chi.Router) { r.Patch("/", svc.UpdateDrive) diff --git a/services/policies/pkg/service/event/service.go b/services/policies/pkg/service/event/service.go index 69f035eebd2..defbd60fa14 100644 --- a/services/policies/pkg/service/event/service.go +++ b/services/policies/pkg/service/event/service.go @@ -125,6 +125,7 @@ func (s Service) processEvent(e events.Event) error { if err := events.Publish(ctx, s.stream, events.PostprocessingStepFinished{ Outcome: outcome, UploadID: ev.UploadID, + ResourceID: ev.ResourceID, ExecutingUser: ev.ExecutingUser, Filename: ev.Filename, FinishedStep: ev.StepToStart, diff --git a/services/postprocessing/pkg/postprocessing/postprocessing.go b/services/postprocessing/pkg/postprocessing/postprocessing.go index aca4ea3e86d..d067dcbe34b 100644 --- a/services/postprocessing/pkg/postprocessing/postprocessing.go +++ b/services/postprocessing/pkg/postprocessing/postprocessing.go @@ -119,6 +119,7 @@ func (pp *Postprocessing) finished(outcome events.PostprocessingOutcome) events. UploadID: pp.ID, ExecutingUser: pp.User, Filename: pp.Filename, + ResourceID: pp.ResourceID, Outcome: outcome, ImpersonatingUser: pp.ImpersonatingUser, } diff --git a/services/proxy/pkg/command/server.go b/services/proxy/pkg/command/server.go index c0a08116358..348f10a87f8 100644 --- a/services/proxy/pkg/command/server.go +++ b/services/proxy/pkg/command/server.go @@ -373,6 +373,7 @@ func loadMiddlewares(logger log.Logger, cfg *config.Config, middleware.Logger(logger), middleware.WithRevaGatewaySelector(gatewaySelector), middleware.RoleQuotas(cfg.RoleQuotas), + middleware.CreateVaultHome(cfg.CreateVaultHome), ), // trigger space assignment when a user logs in middleware.SpaceManager( diff --git a/services/proxy/pkg/config/config.go b/services/proxy/pkg/config/config.go index 9ce6faf3f1d..7a2684c9136 100644 --- a/services/proxy/pkg/config/config.go +++ b/services/proxy/pkg/config/config.go @@ -48,6 +48,7 @@ type Config struct { ClaimSpaceManagement ClaimSpaceManagement `yaml:"claim_space_management"` MultiFactorAuthentication MFAConfig `yaml:"mfa"` MultiInstance MultiInstanceConfig `yaml:"multi_instance"` + CreateVaultHome bool `yaml:"create_vault_home" env:"PROXY_CREATE_VAULT_HOME" desc:"Set this to true to automatically create a new vault home for the user if it does not exist." introductionVersion:"daledda"` Context context.Context `json:"-" yaml:"-"` } diff --git a/services/proxy/pkg/config/defaults/defaultconfig.go b/services/proxy/pkg/config/defaults/defaultconfig.go index 820590f6f29..9e3d4eb8146 100644 --- a/services/proxy/pkg/config/defaults/defaultconfig.go +++ b/services/proxy/pkg/config/defaults/defaultconfig.go @@ -273,6 +273,10 @@ func DefaultPolicies() []config.Policy { Endpoint: "/graph/", Service: "com.owncloud.web.graph", }, + { + Endpoint: "/vault/graph/", + Service: "com.owncloud.web.graph-vault", + }, { Endpoint: "/api/v0/settings", Service: "com.owncloud.web.settings", diff --git a/services/proxy/pkg/middleware/create_home.go b/services/proxy/pkg/middleware/create_home.go index a71f3825354..c007e45c6e8 100644 --- a/services/proxy/pkg/middleware/create_home.go +++ b/services/proxy/pkg/middleware/create_home.go @@ -3,15 +3,16 @@ package middleware import ( "net/http" "strconv" + "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/jellydator/ttlcache/v3" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/graph/pkg/errorcode" revactx "github.com/owncloud/reva/v2/pkg/ctx" - "github.com/owncloud/reva/v2/pkg/rgrpc/status" "github.com/owncloud/reva/v2/pkg/rgrpc/todo/pool" "github.com/owncloud/reva/v2/pkg/utils" "google.golang.org/grpc/metadata" @@ -22,12 +23,20 @@ func CreateHome(optionSetters ...Option) func(next http.Handler) http.Handler { options := newOptions(optionSetters...) logger := options.Logger + cache := ttlcache.New( + ttlcache.WithTTL[string, string](60*time.Second), + ttlcache.WithDisableTouchOnHit[string, string](), + ) + go cache.Start() + return func(next http.Handler) http.Handler { return &createHome{ next: next, logger: logger, revaGatewaySelector: options.RevaGatewaySelector, roleQuotas: options.RoleQuotas, + createVaultHome: options.CreateVaultHome, + cache: cache, } } } @@ -37,6 +46,8 @@ type createHome struct { logger log.Logger revaGatewaySelector pool.Selectable[gateway.GatewayAPIClient] roleQuotas map[string]uint64 + createVaultHome bool + cache *ttlcache.Cache[string, string] } func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -48,7 +59,6 @@ func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) { token := req.Header.Get("x-access-token") // we need to pass the token to authenticate the CreateHome request. - //ctx := tokenpkg.ContextSetToken(r.Context(), token) ctx := metadata.AppendToOutgoingContext(req.Context(), revactx.TokenHeader, token) createHomeReq := &provider.CreateHomeRequest{} @@ -69,13 +79,42 @@ func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) { if err != nil { m.logger.Err(err).Msg("error selecting next gateway client") } else { - createHomeRes, err := client.CreateHome(ctx, createHomeReq) - if err != nil { - m.logger.Err(err).Msg("error calling CreateHome") - } else if createHomeRes.Status.Code != rpc.Code_CODE_OK { - err := status.NewErrorFromCode(createHomeRes.Status.Code, "gateway") - if createHomeRes.Status.Code != rpc.Code_CODE_ALREADY_EXISTS { - m.logger.Err(err).Msg("error when calling Createhome") + key := "home" + u.GetId().GetOpaqueId() + if !m.cache.Has(key) { + createHomeRes, err := client.CreateHome(ctx, createHomeReq) + switch { + case err != nil: + m.logger.Err(err).Msg("error calling CreateHome") + case createHomeRes.GetStatus().GetCode() == rpc.Code_CODE_OK: + m.logger.Debug().Interface("userID", u.GetId().GetOpaqueId()).Msg("personal space created") + m.cache.Set(key, "ok", 0) + case createHomeRes.GetStatus().GetCode() == rpc.Code_CODE_ALREADY_EXISTS: + m.logger.Info().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", createHomeRes.GetStatus()).Msg("personal space already exists") + m.cache.Set(key, "ok", 0) + default: + m.logger.Error().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", createHomeRes.GetStatus()).Msg("personal space creation failed") + } + } + + if m.createVaultHome { + vaultKey := "vault" + u.GetId().GetOpaqueId() + if !m.cache.Has(vaultKey) { + // Create vault personal space + // Inject storage_id into opaque for vault personal space + createHomeReq.Opaque = utils.AppendPlainToOpaque(createHomeReq.Opaque, "storage_id", utils.VaultStorageProviderID) + cpsRes, err := client.CreateHome(ctx, createHomeReq) + switch { + case err != nil: + m.logger.Err(err).Msg("error calling CreateHome for vault personal") + case cpsRes.GetStatus().GetCode() == rpc.Code_CODE_OK: + m.logger.Debug().Interface("userID", u.GetId().GetOpaqueId()).Msg("vault personal space created") + m.cache.Set(vaultKey, "ok", 0) + case cpsRes.GetStatus().GetCode() == rpc.Code_CODE_ALREADY_EXISTS: + m.logger.Info().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", cpsRes.GetStatus()).Msg("vault personal space already exists") + m.cache.Set(vaultKey, "ok", 0) + default: + m.logger.Error().Interface("userID", u.GetId().GetOpaqueId()).Interface("status", cpsRes.GetStatus()).Msg("vault personal space creation failed") + } } } } diff --git a/services/proxy/pkg/middleware/options.go b/services/proxy/pkg/middleware/options.go index 503273a564e..243d69114c7 100644 --- a/services/proxy/pkg/middleware/options.go +++ b/services/proxy/pkg/middleware/options.go @@ -69,6 +69,8 @@ type Options struct { // RoleQuotas hold userid:quota mappings. These will be used when provisioning new users. // The users will get as much quota as is set for their role. RoleQuotas map[string]uint64 + // CreateVaultHome creates a new vault home for the user if it does not exist. + CreateVaultHome bool // TraceProvider sets the tracing provider. TraceProvider trace.TracerProvider // SkipUserInfo prevents the oidc middleware from querying the userinfo endpoint and read any claims directly from the access token instead @@ -243,6 +245,13 @@ func RoleQuotas(roleQuotas map[string]uint64) Option { } } +// CreateVaultHome sets the create vault home flag +func CreateVaultHome(createVaultHome bool) Option { + return func(o *Options) { + o.CreateVaultHome = createVaultHome + } +} + // TraceProvider sets the tracing provider. func TraceProvider(tp trace.TracerProvider) Option { return func(o *Options) { diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index c3bfa18c90e..09526b90b4b 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -45,6 +45,8 @@ type Config struct { MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_USERS_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services." introductionVersion:"5.0"` CliMaxAttemptsRenameFile int `yaml:"max_attempts_rename_file" env:"STORAGE_USERS_CLI_MAX_ATTEMPTS_RENAME_FILE" desc:"The maximum number of attempts to rename a file when a user restores a file to an existing destination with the same name. The minimum value is 100." introductionVersion:"5.0"` + EnableVaultMode bool `yaml:"enable_vault_mode" env:"STORAGE_USERS_ENABLE_VAULT_MODE" desc:"Enable vault mode for the storage-users service runned in addition to the regular storage-users service by owerrwiting the MountID to VaultStorageProviderID. Required the running the storage-users-vault additional service." introductionVersion:"daledda"` + Context context.Context `yaml:"-"` } @@ -215,6 +217,7 @@ type Events struct { TLSRootCaCertPath string `yaml:"tls_root_ca_cert_path" env:"OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE;STORAGE_USERS_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided STORAGE_USERS_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"pre5.0"` EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;STORAGE_USERS_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"pre5.0"` NumConsumers int `yaml:"num_consumers" env:"STORAGE_USERS_EVENTS_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for post-processing files. Multiple consumers increase parallelisation, but will also increase CPU and memory demands. The setting has no effect when the OCIS_ASYNC_UPLOADS is set to false. The default and minimum value is 1." introductionVersion:"pre5.0"` + ConsumerGroup string `yaml:"consumer_group" env:"STORAGE_USERS_EVENTS_CONSUMER_GROUP" desc:"The consumer group name to use for the event consumers. The consumer group name is used to identify the consumers." introductionVersion:"daledda"` AuthUsername string `yaml:"username" env:"OCIS_EVENTS_AUTH_USERNAME;STORAGE_USERS_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"5.0"` AuthPassword string `yaml:"password" env:"OCIS_EVENTS_AUTH_PASSWORD;STORAGE_USERS_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"5.0"` } diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index d6de665deea..f77ef6372f5 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -8,6 +8,7 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/shared" "github.com/owncloud/ocis/v2/ocis-pkg/structs" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" + "github.com/owncloud/reva/v2/pkg/utils" ) // FullDefaultConfig returns a fully initialized default configuration @@ -226,6 +227,11 @@ func EnsureDefaults(cfg *config.Config) { cfg.HTTP.CORS.AllowedOrigins[0] == "https://localhost:9200") { cfg.HTTP.CORS.AllowedOrigins = []string{cfg.Commons.OcisURL} } + + // set mount id to vault storage provider id + if cfg.EnableVaultMode { + cfg.MountID = utils.VaultStorageProviderID + } } // Sanitize sanitized the configuration diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 311e40c7591..9d6da1c1191 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -198,7 +198,8 @@ func Ocis(cfg *config.Config) map[string]interface{} { "cache_auth_password": cfg.IDCache.AuthPassword, }, "events": map[string]interface{}{ - "numconsumers": cfg.Events.NumConsumers, + "numconsumers": cfg.Events.NumConsumers, + "consumer_group": cfg.Events.ConsumerGroup, }, "tokens": map[string]interface{}{ "transfer_shared_secret": cfg.Commons.TransferSecret, @@ -321,7 +322,8 @@ func S3NG(cfg *config.Config) map[string]interface{} { "cache_auth_password": cfg.IDCache.AuthPassword, }, "events": map[string]interface{}{ - "numconsumers": cfg.Events.NumConsumers, + "numconsumers": cfg.Events.NumConsumers, + "consumer_group": cfg.Events.ConsumerGroup, }, "tokens": map[string]interface{}{ "transfer_shared_secret": cfg.Commons.TransferSecret, diff --git a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/gateway.go b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/gateway.go index a83b653a103..8b824df4bd8 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/gateway.go +++ b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/gateway.go @@ -24,13 +24,13 @@ import ( "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + "github.com/mitchellh/mapstructure" "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/rgrpc" "github.com/owncloud/reva/v2/pkg/sharedconf" "github.com/owncloud/reva/v2/pkg/storage/cache" "github.com/owncloud/reva/v2/pkg/token" "github.com/owncloud/reva/v2/pkg/token/manager/registry" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/rs/zerolog" "google.golang.org/grpc" @@ -70,13 +70,12 @@ type config struct { TokenManager string `mapstructure:"token_manager"` // ShareFolder is the location where to create shares in the recipient's storage provider. // FIXME get rid of ShareFolder, there are findByPath calls in the ocmshareporvider.go and usershareprovider.go - ShareFolder string `mapstructure:"share_folder"` - DataTransfersFolder string `mapstructure:"data_transfers_folder"` - TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"` - AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent - CreatePersonalSpaceCacheConfig cache.Config `mapstructure:"create_personal_space_cache_config"` - ProviderCacheConfig cache.Config `mapstructure:"provider_cache_config"` - UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"` + ShareFolder string `mapstructure:"share_folder"` + DataTransfersFolder string `mapstructure:"data_transfers_folder"` + TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"` + AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent + ProviderCacheConfig cache.Config `mapstructure:"provider_cache_config"` + UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"` } // sets defaults @@ -130,22 +129,13 @@ func (c *config) init() { if c.ProviderCacheConfig.Database == "" { c.ProviderCacheConfig.Database = "reva" } - - if c.CreatePersonalSpaceCacheConfig.Store == "" { - c.CreatePersonalSpaceCacheConfig.Store = "memory" - } - - if c.CreatePersonalSpaceCacheConfig.Database == "" { - c.CreatePersonalSpaceCacheConfig.Database = "reva" - } } type svc struct { - c *config - dataGatewayURL url.URL - tokenmgr token.Manager - providerCache cache.ProviderCache - createPersonalSpaceCache cache.CreatePersonalSpaceCache + c *config + dataGatewayURL url.URL + tokenmgr token.Manager + providerCache cache.ProviderCache } // New creates a new gateway svc that acts as a proxy for any grpc operation. @@ -171,11 +161,10 @@ func New(m map[string]interface{}, _ *grpc.Server, _ *zerolog.Logger) (rgrpc.Ser } s := &svc{ - c: c, - dataGatewayURL: *u, - tokenmgr: tokenManager, - providerCache: cache.GetProviderCache(c.ProviderCacheConfig), - createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CreatePersonalSpaceCacheConfig), + c: c, + dataGatewayURL: *u, + tokenmgr: tokenManager, + providerCache: cache.GetProviderCache(c.ProviderCacheConfig), } return s, nil @@ -187,7 +176,6 @@ func (s *svc) Register(ss *grpc.Server) { func (s *svc) Close() error { s.providerCache.Close() - s.createPersonalSpaceCache.Close() return nil } diff --git a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go index dd8e9041050..c3a8a3d825c 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go +++ b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovider.go @@ -38,6 +38,7 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "google.golang.org/grpc/codes" + "github.com/golang-jwt/jwt/v5" "github.com/owncloud/reva/v2/pkg/appctx" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/errtypes" @@ -48,7 +49,6 @@ import ( "github.com/owncloud/reva/v2/pkg/share" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" - "github.com/golang-jwt/jwt/v5" "github.com/pkg/errors" gstatus "google.golang.org/grpc/status" ) @@ -143,6 +143,12 @@ func (s *svc) CreateHome(ctx context.Context, req *provider.CreateHomeRequest) ( }, } } + + // pass storage_id to the storage provider to handle vault storage id + if storageId := utils.ReadPlainFromOpaque(req.GetOpaque(), "storage_id"); storageId != "" { + createReq.Opaque = utils.AppendPlainToOpaque(createReq.Opaque, "storage_id", storageId) + } + res, err := s.CreateStorageSpace(ctx, createReq) if err != nil { return &provider.CreateHomeResponse{ @@ -170,7 +176,12 @@ func (s *svc) CreateStorageSpace(ctx context.Context, req *provider.CreateStorag } } - srClient, err := s.getStorageRegistryClient(ctx, s.c.StorageRegistryEndpoint) + if storageId := utils.ReadPlainFromOpaque(req.GetOpaque(), "storage_id"); storageId != "" { + space.Root = &provider.ResourceId{StorageId: storageId} + req.Opaque = utils.AppendPlainToOpaque(req.Opaque, "storage_id", storageId) + } + + srClient, err := pool.GetStorageRegistryClient(s.c.StorageRegistryEndpoint) if err != nil { return &provider.CreateStorageSpaceResponse{ Status: status.NewStatusFromErrType(ctx, "gateway could get storage registry client", err), @@ -213,7 +224,7 @@ func (s *svc) CreateStorageSpace(ctx context.Context, req *provider.CreateStorag } // just pick the first provider, we expect only one - c, err := s.getSpacesProviderClient(ctx, res.Providers[0]) + c, err := pool.GetSpacesProviderServiceClient(res.Providers[0].Address) if err != nil { return &provider.CreateStorageSpaceResponse{ Status: status.NewStatusFromErrType(ctx, "gateway could not get storage provider client", err), @@ -247,6 +258,7 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp filters["path"] = path } + hasFileIdFilter := false for _, f := range req.Filters { switch f.Type { case provider.ListStorageSpacesRequest_Filter_TYPE_ID: @@ -255,6 +267,7 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp continue } filters["storage_id"], filters["space_id"], filters["opaque_id"] = sid, spid, oid + hasFileIdFilter = true case provider.ListStorageSpacesRequest_Filter_TYPE_OWNER: filters["owner_idp"] = f.GetOwner().GetIdp() filters["owner_id"] = f.GetOwner().GetOpaqueId() @@ -270,7 +283,11 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp } } - c, err := s.getStorageRegistryClient(ctx, s.c.StorageRegistryEndpoint) + if !hasFileIdFilter && utils.ReadPlainFromOpaque(req.Opaque, "storage_id") != "" { + filters["storage_id"] = utils.ReadPlainFromOpaque(req.Opaque, "storage_id") + } + + c, err := pool.GetStorageRegistryClient(s.c.StorageRegistryEndpoint) if err != nil { return &provider.ListStorageSpacesResponse{ Status: status.NewStatusFromErrType(ctx, "gateway could not get storage registry client", err), @@ -324,10 +341,6 @@ func (s *svc) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorag }, nil } - if res.Status.Code == rpc.Code_CODE_OK { - id := res.StorageSpace.Root - s.providerCache.RemoveListStorageProviders(id) - } return res, nil } @@ -362,7 +375,6 @@ func (s *svc) DeleteStorageSpace(ctx context.Context, req *provider.DeleteStorag } id := &provider.ResourceId{OpaqueId: req.GetId().GetOpaqueId()} - s.providerCache.RemoveListStorageProviders(id) if dsRes.Status.Code != rpc.Code_CODE_OK { return dsRes, nil @@ -433,7 +445,7 @@ func (s *svc) GetHome(ctx context.Context, _ *provider.GetHomeRequest) (*provide return nil, errors.New("user not found in context") } - srClient, err := s.getStorageRegistryClient(ctx, s.c.StorageRegistryEndpoint) + srClient, err := pool.GetStorageRegistryClient(s.c.StorageRegistryEndpoint) if err != nil { return &provider.GetHomeResponse{ Status: status.NewStatusFromErrType(ctx, "gateway could not get storage registry client", err), @@ -1027,7 +1039,7 @@ func (s *svc) find(ctx context.Context, ref *provider.Reference) (provider.Provi return nil, nil, err } - client, err := s.getStorageProviderClient(ctx, p[0]) + client, err := pool.GetStorageProviderServiceClient(p[0].Address) return client, p[0], err } @@ -1041,7 +1053,7 @@ func (s *svc) findSpacesProvider(ctx context.Context, ref *provider.Reference) ( return nil, nil, err } - client, err := s.getSpacesProviderClient(ctx, p[0]) + client, err := pool.GetSpacesProviderServiceClient(p[0].Address) return client, p[0], err } @@ -1066,41 +1078,6 @@ func (s *svc) findAndUnwrap(ctx context.Context, ref *provider.Reference) (provi return c, p, relativeReference, nil } -func (s *svc) getSpacesProviderClient(_ context.Context, p *registry.ProviderInfo) (provider.SpacesAPIClient, error) { - c, err := pool.GetSpacesProviderServiceClient(p.Address) - if err != nil { - return nil, err - } - - return &cachedSpacesAPIClient{ - c: c, - createPersonalSpaceCache: s.createPersonalSpaceCache, - }, nil -} - -func (s *svc) getStorageProviderClient(_ context.Context, p *registry.ProviderInfo) (provider.ProviderAPIClient, error) { - c, err := pool.GetStorageProviderServiceClient(p.Address) - if err != nil { - return nil, err - } - - return &cachedAPIClient{ - c: c, - createPersonalSpaceCache: s.createPersonalSpaceCache, - }, nil -} - -func (s *svc) getStorageRegistryClient(_ context.Context, address string) (registry.RegistryAPIClient, error) { - c, err := pool.GetStorageRegistryClient(address) - if err != nil { - return nil, err - } - return &cachedRegistryClient{ - c: c, - cache: s.providerCache, - }, nil -} - func (s *svc) findSingleSpace(ctx context.Context, ref *provider.Reference) ([]*registry.ProviderInfo, error) { switch { case ref == nil: diff --git a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go index c5fc1ba8c43..e58b4382640 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go +++ b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/gateway/storageprovidercache.go @@ -22,12 +22,9 @@ import ( "context" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" - sdk "github.com/owncloud/reva/v2/pkg/sdk/common" "github.com/owncloud/reva/v2/pkg/storage/cache" - "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" "github.com/pkg/errors" "google.golang.org/grpc" @@ -43,8 +40,10 @@ type cachedRegistryClient struct { } func (c *cachedRegistryClient) ListStorageProviders(ctx context.Context, in *registry.ListStorageProvidersRequest, opts ...grpc.CallOption) (*registry.ListStorageProvidersResponse, error) { - - spaceID := sdk.DecodeOpaqueMap(in.Opaque)["space_id"] + spaceID := utils.ReadPlainFromOpaque(in.GetOpaque(), "space_id") + if storageID := utils.ReadPlainFromOpaque(in.GetOpaque(), "storage_id"); storageID != "" { + spaceID = storageID + "$" + spaceID + } u, ok := ctxpkg.ContextGetUser(ctx) if !ok { @@ -83,195 +82,3 @@ func (c *cachedRegistryClient) GetStorageProviders(ctx context.Context, in *regi func (c *cachedRegistryClient) GetHome(ctx context.Context, in *registry.GetHomeRequest, opts ...grpc.CallOption) (*registry.GetHomeResponse, error) { return c.c.GetHome(ctx, in, opts...) } - -/* - Cached Spaces Provider -*/ - -type cachedSpacesAPIClient struct { - c provider.SpacesAPIClient - createPersonalSpaceCache cache.CreatePersonalSpaceCache -} - -// CreateStorageSpace creates a storage space -func (c *cachedSpacesAPIClient) CreateStorageSpace(ctx context.Context, in *provider.CreateStorageSpaceRequest, opts ...grpc.CallOption) (*provider.CreateStorageSpaceResponse, error) { - if in.Type == "personal" { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - return nil, errors.New("user not found in context") - } - - key := c.createPersonalSpaceCache.GetKey(u.GetId()) - if key != "" { - s := &provider.CreateStorageSpaceResponse{} - if err := c.createPersonalSpaceCache.PullFromCache(key, s); err == nil { - return s, nil - } - } - resp, err := c.c.CreateStorageSpace(ctx, in, opts...) - switch { - case err != nil: - return nil, err - case resp.Status.Code != rpc.Code_CODE_OK && resp.Status.Code != rpc.Code_CODE_ALREADY_EXISTS: - return resp, nil - case key == "": - return resp, nil - default: - return resp, c.createPersonalSpaceCache.PushToCache(key, resp) - } - } - return c.c.CreateStorageSpace(ctx, in, opts...) -} - -func (c *cachedSpacesAPIClient) ListStorageSpaces(ctx context.Context, in *provider.ListStorageSpacesRequest, opts ...grpc.CallOption) (*provider.ListStorageSpacesResponse, error) { - return c.c.ListStorageSpaces(ctx, in, opts...) -} -func (c *cachedSpacesAPIClient) UpdateStorageSpace(ctx context.Context, in *provider.UpdateStorageSpaceRequest, opts ...grpc.CallOption) (*provider.UpdateStorageSpaceResponse, error) { - return c.c.UpdateStorageSpace(ctx, in, opts...) -} -func (c *cachedSpacesAPIClient) DeleteStorageSpace(ctx context.Context, in *provider.DeleteStorageSpaceRequest, opts ...grpc.CallOption) (*provider.DeleteStorageSpaceResponse, error) { - resp, err := c.c.DeleteStorageSpace(ctx, in, opts...) - switch { - case err != nil: - return nil, err - case resp.Status.Code != rpc.Code_CODE_OK: - return resp, nil - default: - _, spaceid, _, _ := storagespace.SplitID(in.GetId().GetOpaqueId()) - _ = c.createPersonalSpaceCache.Delete(spaceid) - return resp, nil - } -} - -/* - Cached Storage Provider -*/ - -type cachedAPIClient struct { - c provider.ProviderAPIClient - createPersonalSpaceCache cache.CreatePersonalSpaceCache -} - -// CreateHome caches calls to CreateHome locally - anyways they only need to be called once per user -func (c *cachedAPIClient) CreateHome(ctx context.Context, in *provider.CreateHomeRequest, opts ...grpc.CallOption) (*provider.CreateHomeResponse, error) { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - return nil, errors.New("user not found in context") - } - - key := c.createPersonalSpaceCache.GetKey(u.GetId()) - if key != "" { - s := &provider.CreateHomeResponse{} - if err := c.createPersonalSpaceCache.PullFromCache(key, s); err == nil { - return s, nil - } - } - resp, err := c.c.CreateHome(ctx, in, opts...) - switch { - case err != nil: - return nil, err - case resp.Status.Code != rpc.Code_CODE_OK && resp.Status.Code != rpc.Code_CODE_ALREADY_EXISTS: - return resp, nil - case key == "": - return resp, nil - default: - return resp, c.createPersonalSpaceCache.PushToCache(key, resp) - } -} - -// methods below here are not cached, they just call the client directly - -// Stat returns the Resoure info for a given resource -func (c *cachedAPIClient) Stat(ctx context.Context, in *provider.StatRequest, opts ...grpc.CallOption) (*provider.StatResponse, error) { - return c.c.Stat(ctx, in, opts...) -} -func (c *cachedAPIClient) AddGrant(ctx context.Context, in *provider.AddGrantRequest, opts ...grpc.CallOption) (*provider.AddGrantResponse, error) { - return c.c.AddGrant(ctx, in, opts...) -} -func (c *cachedAPIClient) CreateContainer(ctx context.Context, in *provider.CreateContainerRequest, opts ...grpc.CallOption) (*provider.CreateContainerResponse, error) { - return c.c.CreateContainer(ctx, in, opts...) -} -func (c *cachedAPIClient) Delete(ctx context.Context, in *provider.DeleteRequest, opts ...grpc.CallOption) (*provider.DeleteResponse, error) { - return c.c.Delete(ctx, in, opts...) -} -func (c *cachedAPIClient) DenyGrant(ctx context.Context, in *provider.DenyGrantRequest, opts ...grpc.CallOption) (*provider.DenyGrantResponse, error) { - return c.c.DenyGrant(ctx, in, opts...) -} -func (c *cachedAPIClient) GetPath(ctx context.Context, in *provider.GetPathRequest, opts ...grpc.CallOption) (*provider.GetPathResponse, error) { - return c.c.GetPath(ctx, in, opts...) -} -func (c *cachedAPIClient) GetQuota(ctx context.Context, in *provider.GetQuotaRequest, opts ...grpc.CallOption) (*provider.GetQuotaResponse, error) { - return c.c.GetQuota(ctx, in, opts...) -} -func (c *cachedAPIClient) InitiateFileDownload(ctx context.Context, in *provider.InitiateFileDownloadRequest, opts ...grpc.CallOption) (*provider.InitiateFileDownloadResponse, error) { - return c.c.InitiateFileDownload(ctx, in, opts...) -} -func (c *cachedAPIClient) InitiateFileUpload(ctx context.Context, in *provider.InitiateFileUploadRequest, opts ...grpc.CallOption) (*provider.InitiateFileUploadResponse, error) { - return c.c.InitiateFileUpload(ctx, in, opts...) -} -func (c *cachedAPIClient) ListGrants(ctx context.Context, in *provider.ListGrantsRequest, opts ...grpc.CallOption) (*provider.ListGrantsResponse, error) { - return c.c.ListGrants(ctx, in, opts...) -} -func (c *cachedAPIClient) ListContainerStream(ctx context.Context, in *provider.ListContainerStreamRequest, opts ...grpc.CallOption) (provider.ProviderAPI_ListContainerStreamClient, error) { - return c.c.ListContainerStream(ctx, in, opts...) -} -func (c *cachedAPIClient) ListContainer(ctx context.Context, in *provider.ListContainerRequest, opts ...grpc.CallOption) (*provider.ListContainerResponse, error) { - return c.c.ListContainer(ctx, in, opts...) -} -func (c *cachedAPIClient) ListFileVersions(ctx context.Context, in *provider.ListFileVersionsRequest, opts ...grpc.CallOption) (*provider.ListFileVersionsResponse, error) { - return c.c.ListFileVersions(ctx, in, opts...) -} -func (c *cachedAPIClient) ListRecycleStream(ctx context.Context, in *provider.ListRecycleStreamRequest, opts ...grpc.CallOption) (provider.ProviderAPI_ListRecycleStreamClient, error) { - return c.c.ListRecycleStream(ctx, in, opts...) -} -func (c *cachedAPIClient) ListRecycle(ctx context.Context, in *provider.ListRecycleRequest, opts ...grpc.CallOption) (*provider.ListRecycleResponse, error) { - return c.c.ListRecycle(ctx, in, opts...) -} -func (c *cachedAPIClient) Move(ctx context.Context, in *provider.MoveRequest, opts ...grpc.CallOption) (*provider.MoveResponse, error) { - return c.c.Move(ctx, in, opts...) -} -func (c *cachedAPIClient) RemoveGrant(ctx context.Context, in *provider.RemoveGrantRequest, opts ...grpc.CallOption) (*provider.RemoveGrantResponse, error) { - return c.c.RemoveGrant(ctx, in, opts...) -} -func (c *cachedAPIClient) PurgeRecycle(ctx context.Context, in *provider.PurgeRecycleRequest, opts ...grpc.CallOption) (*provider.PurgeRecycleResponse, error) { - return c.c.PurgeRecycle(ctx, in, opts...) -} -func (c *cachedAPIClient) RestoreFileVersion(ctx context.Context, in *provider.RestoreFileVersionRequest, opts ...grpc.CallOption) (*provider.RestoreFileVersionResponse, error) { - return c.c.RestoreFileVersion(ctx, in, opts...) -} -func (c *cachedAPIClient) RestoreRecycleItem(ctx context.Context, in *provider.RestoreRecycleItemRequest, opts ...grpc.CallOption) (*provider.RestoreRecycleItemResponse, error) { - return c.c.RestoreRecycleItem(ctx, in, opts...) -} -func (c *cachedAPIClient) UpdateGrant(ctx context.Context, in *provider.UpdateGrantRequest, opts ...grpc.CallOption) (*provider.UpdateGrantResponse, error) { - return c.c.UpdateGrant(ctx, in, opts...) -} -func (c *cachedAPIClient) CreateSymlink(ctx context.Context, in *provider.CreateSymlinkRequest, opts ...grpc.CallOption) (*provider.CreateSymlinkResponse, error) { - return c.c.CreateSymlink(ctx, in, opts...) -} -func (c *cachedAPIClient) CreateReference(ctx context.Context, in *provider.CreateReferenceRequest, opts ...grpc.CallOption) (*provider.CreateReferenceResponse, error) { - return c.c.CreateReference(ctx, in, opts...) -} -func (c *cachedAPIClient) SetArbitraryMetadata(ctx context.Context, in *provider.SetArbitraryMetadataRequest, opts ...grpc.CallOption) (*provider.SetArbitraryMetadataResponse, error) { - return c.c.SetArbitraryMetadata(ctx, in, opts...) -} -func (c *cachedAPIClient) UnsetArbitraryMetadata(ctx context.Context, in *provider.UnsetArbitraryMetadataRequest, opts ...grpc.CallOption) (*provider.UnsetArbitraryMetadataResponse, error) { - return c.c.UnsetArbitraryMetadata(ctx, in, opts...) -} -func (c *cachedAPIClient) SetLock(ctx context.Context, in *provider.SetLockRequest, opts ...grpc.CallOption) (*provider.SetLockResponse, error) { - return c.c.SetLock(ctx, in, opts...) -} -func (c *cachedAPIClient) GetLock(ctx context.Context, in *provider.GetLockRequest, opts ...grpc.CallOption) (*provider.GetLockResponse, error) { - return c.c.GetLock(ctx, in, opts...) -} -func (c *cachedAPIClient) RefreshLock(ctx context.Context, in *provider.RefreshLockRequest, opts ...grpc.CallOption) (*provider.RefreshLockResponse, error) { - return c.c.RefreshLock(ctx, in, opts...) -} -func (c *cachedAPIClient) Unlock(ctx context.Context, in *provider.UnlockRequest, opts ...grpc.CallOption) (*provider.UnlockResponse, error) { - return c.c.Unlock(ctx, in, opts...) -} -func (c *cachedAPIClient) GetHome(ctx context.Context, in *provider.GetHomeRequest, opts ...grpc.CallOption) (*provider.GetHomeResponse, error) { - return c.c.GetHome(ctx, in, opts...) -} -func (c *cachedAPIClient) TouchFile(ctx context.Context, in *provider.TouchFileRequest, opts ...grpc.CallOption) (*provider.TouchFileResponse, error) { - return c.c.TouchFile(ctx, in, opts...) -} diff --git a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go index d790bf2c1d5..2bf7025e9b8 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go +++ b/vendor/github.com/owncloud/reva/v2/internal/grpc/services/storageprovider/storageprovider.go @@ -33,6 +33,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/mitchellh/mapstructure" "github.com/owncloud/reva/v2/pkg/appctx" "github.com/owncloud/reva/v2/pkg/conversions" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" @@ -47,7 +48,6 @@ import ( "github.com/owncloud/reva/v2/pkg/storage/fs/registry" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/rs/zerolog" "go.opentelemetry.io/otel/attribute" @@ -787,6 +787,7 @@ func (s *Service) Stat(ctx context.Context, req *provider.StatRequest) (*provide s.addMissingStorageProviderID(md.GetId(), nil) s.addMissingStorageProviderID(md.GetParentId(), nil) s.addMissingStorageProviderID(md.GetSpace().GetRoot(), nil) + s.addMissingStorageProviderID(md.GetSpace().GetRootInfo().GetId(), nil) return &provider.StatResponse{ Status: status.NewOK(ctx), diff --git a/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go b/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go index 41c92b936c6..12093c61230 100644 --- a/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go +++ b/vendor/github.com/owncloud/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/spaces.go @@ -137,8 +137,7 @@ func (h *Handler) addSpaceMember(w http.ResponseWriter, r *http.Request, info *p response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "error getting storage provider", err) return } - - providerClient, err := h.getStorageProviderClient(p) + providerClient, err := pool.GetStorageProviderServiceClient(p.Address) if err != nil { response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "error getting storage provider client", err) return @@ -244,8 +243,7 @@ func (h *Handler) removeSpaceMember(w http.ResponseWriter, r *http.Request, spac if ref.ResourceId.OpaqueId == "" { ref.ResourceId.OpaqueId = ref.ResourceId.SpaceId } - - providerClient, err := h.getStorageProviderClient(prov) + providerClient, err := pool.GetStorageProviderServiceClient(prov.Address) if err != nil { response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "error getting storage provider client", err) return @@ -290,16 +288,6 @@ func (h *Handler) removeSpaceMember(w http.ResponseWriter, r *http.Request, spac response.WriteOCSSuccess(w, r, nil) } -func (h *Handler) getStorageProviderClient(p *registry.ProviderInfo) (provider.ProviderAPIClient, error) { - c, err := pool.GetStorageProviderServiceClient(p.Address) - if err != nil { - err = errors.Wrap(err, "shares spaces: error getting a storage provider client") - return nil, err - } - - return c, nil -} - func (h *Handler) findProvider(ctx context.Context, ref *provider.Reference) (*registry.ProviderInfo, error) { c, err := pool.GetStorageRegistryClient(h.storageRegistryAddr) if err != nil { diff --git a/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go b/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go index f4268920a3d..64318cb9487 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/events/postprocessing.go @@ -103,6 +103,7 @@ type PostprocessingStepFinished struct { UploadID string ExecutingUser *user.User Filename string + ResourceID *provider.ResourceId FinishedStep Postprocessingstep // name of the step Result interface{} // result information see VirusscanResult for example @@ -145,6 +146,7 @@ type VirusscanResult struct { type PostprocessingFinished struct { UploadID string Filename string + ResourceID *provider.ResourceId SpaceOwner *user.UserId ExecutingUser *user.User Result map[Postprocessingstep]interface{} // it is a map[step]Event diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/cache.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/cache.go index d91c8576ea8..1829ede4c01 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/cache.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/cache.go @@ -34,12 +34,10 @@ import ( var ( // DefaultStatCache is the memory store. - statCaches = make(map[string]StatCache) - providerCaches = make(map[string]ProviderCache) - createHomeCaches = make(map[string]CreateHomeCache) - createPersonalSpaceCaches = make(map[string]CreatePersonalSpaceCache) - fileMetadataCaches = make(map[string]FileMetadataCache) - mutex sync.Mutex + statCaches = make(map[string]StatCache) + providerCaches = make(map[string]ProviderCache) + fileMetadataCaches = make(map[string]FileMetadataCache) + mutex sync.Mutex ) // Config contains the configuring for a cache @@ -82,19 +80,6 @@ type ProviderCache interface { GetKey(userID *userpb.UserId, spaceID string) string } -// CreateHomeCache handles removing keys from a create home cache -type CreateHomeCache interface { - Cache - RemoveCreateHome(res *provider.ResourceId) - GetKey(userID *userpb.UserId) string -} - -// CreatePersonalSpaceCache handles removing keys from a create home cache -type CreatePersonalSpaceCache interface { - Cache - GetKey(userID *userpb.UserId) string -} - // FileMetadataCache handles file metadata type FileMetadataCache interface { Cache @@ -127,32 +112,6 @@ func GetProviderCache(cfg Config) ProviderCache { return providerCaches[key] } -// GetCreateHomeCache will return an existing CreateHomeCache for the given store, nodes, database and table -// If it does not exist yet it will be created, different TTLs are ignored -func GetCreateHomeCache(cfg Config) CreateHomeCache { - mutex.Lock() - defer mutex.Unlock() - - key := strings.Join(append(append([]string{cfg.Store}, cfg.Nodes...), cfg.Database, cfg.Table), ":") - if createHomeCaches[key] == nil { - createHomeCaches[key] = NewCreateHomeCache(cfg) - } - return createHomeCaches[key] -} - -// GetCreatePersonalSpaceCache will return an existing CreatePersonalSpaceCache for the given store, nodes, database and table -// If it does not exist yet it will be created, different TTLs are ignored -func GetCreatePersonalSpaceCache(cfg Config) CreatePersonalSpaceCache { - mutex.Lock() - defer mutex.Unlock() - - key := strings.Join(append(append([]string{cfg.Store}, cfg.Nodes...), cfg.Database, cfg.Table), ":") - if createPersonalSpaceCaches[key] == nil { - createPersonalSpaceCaches[key] = NewCreatePersonalSpaceCache(cfg) - } - return createPersonalSpaceCaches[key] -} - // GetFileMetadataCache will return an existing GetFileMetadataCache for the given store, nodes, database and table // If it does not exist yet it will be created, different TTLs are ignored func GetFileMetadataCache(cfg Config) FileMetadataCache { diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/createhome.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/createhome.go deleted file mode 100644 index 3b3c67835dd..00000000000 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/createhome.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package cache - -import ( - "strings" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" -) - -// CreateHomeCache can invalidate all create home related cache entries -type createHomeCache struct { - cacheStore -} - -// NewCreateHomeCache creates a new CreateHomeCache -func NewCreateHomeCache(cfg Config) CreateHomeCache { - c := &createHomeCache{} - c.s = getStore(cfg) - c.database = cfg.Database - c.table = cfg.Table - c.ttl = cfg.TTL - - return c -} - -// RemoveCreateHome removes a reference from the listproviders cache -func (c createHomeCache) RemoveCreateHome(res *provider.ResourceId) { - if res == nil { - return - } - sid := res.SpaceId - - keys, err := c.List() - if err != nil { - // FIXME log error - return - } - // FIXME add context option to List, Read and Write to upstream - for _, key := range keys { - if strings.Contains(key, sid) { - _ = c.Delete(key) - continue - } - } -} - -func (c createHomeCache) GetKey(userID *userpb.UserId) string { - return userID.GetOpaqueId() -} diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/createpersonalspace.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/createpersonalspace.go deleted file mode 100644 index b7422b504e2..00000000000 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/cache/createpersonalspace.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package cache - -import ( - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" -) - -// CreatePersonalSpaceCache can invalidate all create home related cache entries -type createPersonalSpaceCache struct { - cacheStore -} - -// NewCreatePersonalSpaceCache creates a new CreatePersonalSpaceCache -func NewCreatePersonalSpaceCache(cfg Config) CreatePersonalSpaceCache { - c := &createPersonalSpaceCache{} - c.s = getStore(cfg) - c.database = cfg.Database - c.table = cfg.Table - c.ttl = cfg.TTL - - return c -} - -func (c createPersonalSpaceCache) GetKey(userID *userpb.UserId) string { - return userID.GetOpaqueId() -} diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go index ac586e96d39..3ba2ef6600c 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/storage/registry/spaces/spaces.go @@ -34,6 +34,7 @@ import ( providerpb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/mitchellh/mapstructure" "github.com/owncloud/reva/v2/pkg/appctx" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/errtypes" @@ -44,7 +45,6 @@ import ( pkgregistry "github.com/owncloud/reva/v2/pkg/storage/registry/registry" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/utils" - "github.com/mitchellh/mapstructure" "google.golang.org/grpc" ) @@ -195,6 +195,18 @@ func (r *registry) GetProvider(ctx context.Context, space *providerpb.StorageSpa if space.SpaceType != "" && spaceType != space.SpaceType { continue } + + // Filter out vault spaces if no storageId is provided + if space.GetRoot().GetStorageId() != "" { + if space.GetRoot().GetStorageId() != provider.ProviderID { + continue + } + } else { + if strings.HasPrefix(sc.MountPoint, "/vault/") { + continue + } + } + if space.Owner != nil { user := ctxpkg.ContextMustGetUser(ctx) spacePath, err = sc.SpacePath(user, space) @@ -289,7 +301,7 @@ func (r *registry) ListProviders(ctx context.Context, filters map[string]string) // return all providers return r.findAllProviders(ctx, mask), nil default: - return r.findProvidersForFilter(ctx, r.buildFilters(filters), unrestricted, mask), nil + return r.findProvidersForFilter(ctx, r.buildFilters(filters), filters["storage_id"], unrestricted, mask), nil } } @@ -340,7 +352,7 @@ func (r *registry) buildFilters(filterMap map[string]string) []*providerpb.ListS return filters } -func (r *registry) findProvidersForFilter(ctx context.Context, filters []*providerpb.ListStorageSpacesRequest_Filter, unrestricted bool, _ string) []*registrypb.ProviderInfo { +func (r *registry) findProvidersForFilter(ctx context.Context, filters []*providerpb.ListStorageSpacesRequest_Filter, storageId string, unrestricted bool, _ string) []*registrypb.ProviderInfo { var requestedSpaceType string for _, f := range filters { @@ -357,6 +369,10 @@ func (r *registry) findProvidersForFilter(ctx context.Context, filters []*provid // we have to ignore a space type filter with +grant or +mountpoint type because they can live on any provider if requestedSpaceType != "" && !strings.HasPrefix(requestedSpaceType, "+") { found := false + if storageId != "" && storageId != provider.ProviderID { + // skip mismatching storageproviders + continue + } for spaceType := range provider.Spaces { if spaceType == requestedSpaceType { found = true @@ -385,6 +401,10 @@ func (r *registry) findProvidersForFilter(ctx context.Context, filters []*provid if sc, ok = provider.Spaces[space.SpaceType]; !ok { continue } + // Filter out vault spaces if no storageId is provided + if storageId == "" && strings.HasPrefix(sc.MountPoint, "/vault/") { + continue + } spacePath, err = sc.SpacePath(currentUser, space) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("provider", provider).Interface("space", space).Msg("failed to execute template, continuing") diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index 79dcc454a76..c4c4fd1e08f 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -258,7 +258,7 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor return nil, errors.New("need nats for async file processing") } - ch, err := events.Consume(fs.stream, "dcfs", _registeredEvents...) + ch, err := events.Consume(fs.stream, o.Events.ConsumerGroup, _registeredEvents...) if err != nil { return nil, err } @@ -285,6 +285,10 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { switch ev := event.Event.(type) { case events.PostprocessingFinished: sublog := log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { + sublog.Debug().Msg("ignoring event for different storage") + continue + } session, err := fs.sessionStore.Get(ctx, ev.UploadID) if err != nil { sublog.Error().Err(err).Msg("Failed to get upload") @@ -450,6 +454,10 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) case events.RevertRevision: sublog := log.With().Str("event", "RevertRevision").Interface("nodeid", ev.ResourceID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { + sublog.Debug().Msg("ignoring event for different storage") + continue + } n, err := fs.lu.NodeFromID(ctx, ev.ResourceID) if err != nil { sublog.Error().Err(err).Msg("Failed to get node") @@ -462,6 +470,10 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } case events.PostprocessingStepFinished: sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { + sublog.Debug().Msg("ignoring event for different storage") + continue + } if ev.FinishedStep != events.PPStepAntivirus { // atm we are only interested in antivirus results continue diff --git a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go index 5c76a383eac..210f2068130 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options/options.go @@ -23,10 +23,10 @@ import ( "strings" "time" + "github.com/mitchellh/mapstructure" "github.com/owncloud/reva/v2/pkg/rgrpc/todo/pool" "github.com/owncloud/reva/v2/pkg/sharedconf" "github.com/owncloud/reva/v2/pkg/storage/cache" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -103,7 +103,8 @@ type AsyncPropagatorOptions struct { // EventOptions are the configurable options for events type EventOptions struct { - NumConsumers int `mapstructure:"numconsumers"` + NumConsumers int `mapstructure:"numconsumers"` + ConsumerGroup string `mapstructure:"consumer_group"` } // TokenOptions are the configurable option for tokens @@ -172,5 +173,9 @@ func New(m map[string]interface{}) (*Options, error) { o.UploadDirectory = filepath.Join(o.Root, "uploads") } + if o.Events.ConsumerGroup == "" { + o.Events.ConsumerGroup = "dcfs" + } + return o, nil } diff --git a/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go b/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go index c1031368743..c562636e8b0 100644 --- a/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go +++ b/vendor/github.com/owncloud/reva/v2/pkg/utils/utils.go @@ -64,6 +64,9 @@ var ( // OCMStorageSpaceID is the space id used by the ocmreceived storageprovider OCMStorageSpaceID = "89f37a33-858b-45fa-8890-a1f2b27d90e1" + // VaultStorageProviderID is the storage id used by the vault storageprovider + VaultStorageProviderID = "1a01c2c4-4309-4483-a845-842fd56d8622" + // SpaceGrant is used to signal the storageprovider that the grant is on a space SpaceGrant struct{} ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 16fade20f5b..7aaef51414a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1316,7 +1316,7 @@ github.com/orcaman/concurrent-map # github.com/owncloud/libre-graph-api-go v1.0.5-0.20260216101009-eeac018af245 ## explicit; go 1.18 github.com/owncloud/libre-graph-api-go -# github.com/owncloud/reva/v2 v2.0.0-20260304131727-ce3149532498 +# github.com/owncloud/reva/v2 v2.0.0-20260316140824-a145e1807968 ## explicit; go 1.24.0 github.com/owncloud/reva/v2/cmd/revad/internal/grace github.com/owncloud/reva/v2/cmd/revad/runtime