From 72a01658b59981d7af12ef17a36bb5230aac8bd2 Mon Sep 17 00:00:00 2001 From: Ayd Asraf Date: Wed, 18 Mar 2026 15:43:18 +0100 Subject: [PATCH 01/32] chore: bump version to v1.22.0 --- .gitignore | 15 +- .wiki/Configuration-HA.md | 156 + .wiki/Configuration-Health.md | 120 + .wiki/Configuration-Metadata.md | 52 + .wiki/Configuration-Metrics.md | 39 + .wiki/Home.md | 6 +- .wiki/Rest-api.md | 82 + .wiki/_Sidebar.md | 2 + README.md | 26 +- artipie-backfill/README.md | 239 + artipie-backfill/pom.xml | 143 + .../com/artipie/backfill/ArtifactRecord.java | 33 + .../artipie/backfill/ArtipieMetaSidecar.java | 137 + .../com/artipie/backfill/BackfillCli.java | 478 ++ .../com/artipie/backfill/BatchInserter.java | 304 ++ .../artipie/backfill/BulkBackfillRunner.java | 338 ++ .../com/artipie/backfill/ComposerScanner.java | 373 ++ .../com/artipie/backfill/DebianScanner.java | 213 + .../com/artipie/backfill/DockerScanner.java | 386 ++ .../com/artipie/backfill/FileScanner.java | 101 + .../java/com/artipie/backfill/GemScanner.java | 122 + .../java/com/artipie/backfill/GoScanner.java | 331 ++ .../com/artipie/backfill/HelmScanner.java | 194 + .../com/artipie/backfill/MavenScanner.java | 194 + .../java/com/artipie/backfill/NpmScanner.java | 517 ++ .../artipie/backfill/ProgressReporter.java | 114 + .../com/artipie/backfill/PypiScanner.java | 170 + .../com/artipie/backfill/RepoConfigYaml.java | 81 + .../java/com/artipie/backfill/RepoEntry.java | 15 + .../artipie/backfill/RepoTypeNormalizer.java | 39 + .../java/com/artipie/backfill/Scanner.java | 31 + .../com/artipie/backfill/ScannerFactory.java | 97 + .../org.apache.logging.log4j.spi.Provider | 1 + .../src/main/resources/log4j2.xml | 15 + .../com/artipie/backfill/BackfillCliTest.java | 273 + .../backfill/BackfillIntegrationTest.java | 533 ++ .../artipie/backfill/BatchInserterTest.java | 174 + .../backfill/BulkBackfillRunnerTest.java | 354 ++ .../artipie/backfill/ComposerScannerTest.java | 481 ++ .../artipie/backfill/DebianScannerTest.java | 313 ++ .../artipie/backfill/DockerScannerTest.java | 381 ++ .../com/artipie/backfill/GemScannerTest.java | 244 + .../com/artipie/backfill/GoScannerTest.java | 327 ++ .../com/artipie/backfill/HelmScannerTest.java | 219 + .../artipie/backfill/MavenScannerTest.java | 357 ++ .../com/artipie/backfill/NpmScannerTest.java | 475 ++ .../backfill/ProgressReporterTest.java | 95 + .../com/artipie/backfill/PypiScannerTest.java | 317 ++ .../artipie/backfill/RepoConfigYamlTest.java | 137 + .../backfill/RepoTypeNormalizerTest.java | 37 + .../artipie/backfill/ScannerFactoryTest.java | 99 + artipie-core/pom.xml | 13 +- .../asto/dedup/ContentAddressableStorage.java | 132 + .../cache/CacheInvalidationPubSub.java | 184 + .../artipie/cache/PublishingCleanable.java | 67 + .../java/com/artipie/cache/StoragesCache.java | 7 +- .../com/artipie/cache/ValkeyConnection.java | 161 +- .../com/artipie/cluster/ClusterEventBus.java | 237 + .../com/artipie/cooldown/CooldownCache.java | 37 +- .../artipie/cooldown/CooldownSettings.java | 30 +- .../metadata/CooldownMetadataServiceImpl.java | 28 +- .../com/artipie/http/ResponseBuilder.java | 6 + .../main/java/com/artipie/http/RsStatus.java | 4 + .../com/artipie/http/auth/AuthzSlice.java | 8 +- .../artipie/http/auth/CombinedAuthzSlice.java | 8 +- .../http/cache/BaseCachedProxySlice.java | 999 ++++ .../http/cache/ConditionalRequest.java | 77 + .../com/artipie/http/cache/DedupStrategy.java | 33 + .../artipie/http/cache/DigestComputer.java | 157 + .../com/artipie/http/cache/NegativeCache.java | 48 +- .../http/cache/NegativeCacheRegistry.java | 111 + .../artipie/http/cache/ProxyCacheConfig.java | 208 +- .../http/cache/RequestDeduplicator.java | 198 + .../com/artipie/http/cache/SidecarFile.java | 28 + .../java/com/artipie/http/headers/Login.java | 7 +- .../com/artipie/http/misc/ConfigDefaults.java | 63 + .../artipie/http/misc/DispatchedStorage.java | 151 + .../http/misc/RepoNameMeterFilter.java | 79 + .../artipie/http/misc/StorageExecutors.java | 146 + .../com/artipie/http/retry/RetrySlice.java | 167 + .../http/slice/CircuitBreakerSlice.java | 241 +- .../com/artipie/http/slice/LoggingSlice.java | 6 +- .../com/artipie/http/slice/TrimPathSlice.java | 7 +- .../http/timeout/AutoBlockRegistry.java | 110 + .../http/timeout/AutoBlockSettings.java | 23 + .../com/artipie/http/timeout/BlockState.java | 21 + .../artipie/http/timeout/TimeoutSettings.java | 93 + .../com/artipie/index/ArtifactDocument.java | 42 + .../java/com/artipie/index/ArtifactIndex.java | 139 + .../java/com/artipie/index/SearchResult.java | 46 + .../java/com/artipie/layout/GradleLayout.java | 71 - .../com/artipie/layout/LayoutFactory.java | 2 +- .../com/artipie/scheduling/ArtifactEvent.java | 50 +- .../artipie/scheduling/EventsProcessor.java | 88 +- .../artipie/scheduling/JobDataRegistry.java | 69 + .../com/artipie/scheduling/QuartzJob.java | 8 +- .../security/perms/PermissionConfig.java | 125 + .../artipie/cache/ValkeyConnectionTest.java | 113 + .../artipie/cluster/ClusterEventBusTest.java | 152 + ...seCachedProxySliceContentEncodingTest.java | 379 ++ .../http/cache/ConditionalRequestTest.java | 134 + .../artipie/http/cache/DedupStrategyTest.java | 32 + .../http/cache/DigestComputerTest.java | 192 + .../http/cache/NegativeCacheRegistryTest.java | 50 + .../http/cache/RequestDeduplicatorTest.java | 240 + .../artipie/http/cache/SidecarFileTest.java | 44 + .../artipie/http/misc/ConfigDefaultsTest.java | 50 + .../http/misc/DispatchedStorageTest.java | 299 ++ .../http/misc/RepoNameMeterFilterTest.java | 63 + .../http/misc/StorageExecutorsTest.java | 68 + .../artipie/http/retry/RetrySliceTest.java | 229 + .../http/slice/CircuitBreakerSliceTest.java | 78 + .../http/timeout/AutoBlockRegistryTest.java | 105 + .../http/timeout/TimeoutSettingsTest.java | 60 + .../com/artipie/layout/GradleLayoutTest.java | 119 - .../com/artipie/layout/LayoutFactoryTest.java | 2 +- artipie-main/Dockerfile | 2 +- artipie-main/docker-compose/.env.example | 2 +- .../artipie/repo/gradle_proxy.yaml | 1 + .../security/users/ayd.asraf@auto1.local.yml | 5 + .../docker-compose/docker-compose.yaml | 21 +- .../docker-compose/nginx/conf.d/default.conf | 18 + artipie-main/pom.xml | 124 +- .../java/com/artipie/RepositorySlices.java | 308 +- .../src/main/java/com/artipie/VertxMain.java | 269 +- .../artipie/adapters/docker/DockerProxy.java | 36 +- .../docker/DockerProxyCooldownSlice.java | 12 +- .../artipie/adapters/gradle/GradleProxy.java | 87 - .../artipie/adapters/npm/NpmProxyAdapter.java | 9 +- .../adapters/php/ComposerGroupSlice.java | 216 +- .../java/com/artipie/api/AuthTokenRest.java | 2 +- .../java/com/artipie/api/AuthzHandler.java | 9 +- .../main/java/com/artipie/api/BaseRest.java | 24 +- .../main/java/com/artipie/api/CacheRest.java | 367 -- .../java/com/artipie/api/RepositoryRest.java | 43 +- .../main/java/com/artipie/api/RestApi.java | 69 +- .../main/java/com/artipie/api/RolesRest.java | 10 +- .../main/java/com/artipie/api/SearchRest.java | 245 + .../java/com/artipie/api/SettingsRest.java | 46 +- .../com/artipie/api/StorageAliasesRest.java | 3 +- .../main/java/com/artipie/api/UsersRest.java | 68 +- .../main/java/com/artipie/api/Validator.java | 19 +- .../artipie/api/perms/ApiCachePermission.java | 137 - .../api/perms/ApiCooldownPermission.java | 139 + .../perms/ApiCooldownPermissionFactory.java | 29 + .../api/perms/ApiSearchPermission.java | 139 + .../api/perms/ApiSearchPermissionFactory.java | 29 + .../java/com/artipie/api/v1/ApiResponse.java | 71 + .../com/artipie/api/v1/ArtifactHandler.java | 604 +++ .../com/artipie/api/v1/AsyncApiVerticle.java | 321 ++ .../java/com/artipie/api/v1/AuthHandler.java | 804 +++ .../com/artipie/api/v1/CooldownHandler.java | 580 ++ .../com/artipie/api/v1/DashboardHandler.java | 221 + .../com/artipie/api/v1/RepositoryHandler.java | 490 ++ .../java/com/artipie/api/v1/RoleHandler.java | 313 ++ .../com/artipie/api/v1/SearchHandler.java | 276 + .../com/artipie/api/v1/SettingsHandler.java | 428 ++ .../artipie/api/v1/StorageAliasHandler.java | 372 ++ .../java/com/artipie/api/v1/UserHandler.java | 399 ++ .../java/com/artipie/auth/AuthFromDb.java | 97 + .../com/artipie/auth/JwtPasswordAuth.java | 3 +- .../artipie/auth/JwtPasswordAuthFactory.java | 3 +- .../main/java/com/artipie/auth/JwtTokens.java | 22 + .../java/com/artipie/auth/OktaOidcClient.java | 60 +- .../com/artipie/cluster/DbNodeRegistry.java | 280 + .../com/artipie/cluster/NodeRegistry.java | 114 + .../artipie/cooldown/CooldownRepository.java | 194 +- .../com/artipie/cooldown/CooldownSupport.java | 91 + .../com/artipie/cooldown/DbBlockRecord.java | 28 +- .../artipie/cooldown/JdbcCooldownService.java | 97 +- .../com/artipie/db/ArtifactDbFactory.java | 164 +- .../main/java/com/artipie/db/DbConsumer.java | 155 +- .../main/java/com/artipie/db/DbManager.java | 47 + .../java/com/artipie/db/DeadLetterWriter.java | 94 + .../java/com/artipie/db/dao/AuditLogDao.java | 47 + .../com/artipie/db/dao/AuthProviderDao.java | 126 + .../com/artipie/db/dao/RepositoryDao.java | 163 + .../main/java/com/artipie/db/dao/RoleDao.java | 139 + .../java/com/artipie/db/dao/SettingsDao.java | 94 + .../com/artipie/db/dao/StorageAliasDao.java | 128 + .../main/java/com/artipie/db/dao/UserDao.java | 294 + .../java/com/artipie/db/dao/UserTokenDao.java | 194 + .../db/migration/YamlToDbMigrator.java | 450 ++ .../diagnostics/BlockedThreadDiagnostics.java | 45 +- .../com/artipie/group/ArtifactNameParser.java | 342 ++ .../com/artipie/group/GroupMetadataCache.java | 29 + .../com/artipie/group/GroupNegativeCache.java | 652 --- .../java/com/artipie/group/GroupSlice.java | 529 +- .../com/artipie/group/MavenGroupSlice.java | 79 +- .../java/com/artipie/group/MemberSlice.java | 196 +- .../java/com/artipie/group/RoutingRule.java | 93 + .../com/artipie/group/WritableGroupSlice.java | 61 + .../com/artipie/http/ApiRoutingSlice.java | 132 +- .../java/com/artipie/http/HealthSlice.java | 71 +- .../main/java/com/artipie/http/MainSlice.java | 71 +- .../com/artipie/http/MergeShardsSlice.java | 50 +- .../artipie/http/slice/BrowsableSlice.java | 15 + .../http/slice/FileSystemBrowseSlice.java | 50 +- .../artipie/http/slice/RepoMetricsSlice.java | 8 +- .../artipie/importer/MetadataRegenerator.java | 16 +- .../com/artipie/index/DbArtifactIndex.java | 698 +++ .../com/artipie/jetty/http3/Http3Server.java | 1 + .../artipie/metrics/AsyncMetricsVerticle.java | 8 +- .../artipie/micrometer/MicrometerSlice.java | 8 +- .../com/artipie/proxy/OfflineAwareSlice.java | 81 + .../ArtipieQuartzConnectionProvider.java | 60 + .../scheduling/MetadataEventQueues.java | 78 +- .../com/artipie/scheduling/QuartzSchema.java | 319 ++ .../com/artipie/scheduling/QuartzService.java | 221 +- .../scheduling/TempFileCleanupJob.java | 262 + .../security/policy/CachedDbPolicy.java | 375 ++ .../com/artipie/settings/ArtipieSecurity.java | 19 +- .../com/artipie/settings/JwtSettings.java | 8 +- .../java/com/artipie/settings/RepoData.java | 47 +- .../java/com/artipie/settings/Settings.java | 18 + .../artipie/settings/SettingsFromPath.java | 16 +- .../com/artipie/settings/YamlSettings.java | 276 +- .../artipie/settings/cache/ArtipieCaches.java | 3 + .../artipie/settings/cache/CachedUsers.java | 5 +- .../cache/PublishingFiltersCache.java | 62 + .../settings/repo/CrudRepoSettings.java | 10 + .../settings/repo/DualCrudRepoSettings.java | 108 + .../com/artipie/settings/repo/RepoConfig.java | 37 + .../artipie/settings/users/DualCrudUsers.java | 90 + .../com/artipie/webhook/WebhookConfig.java | 61 + .../artipie/webhook/WebhookDispatcher.java | 155 + .../V100__create_settings_tables.sql | 95 + .../V101__create_user_tokens_table.sql | 15 + .../swagger-ui/swagger-initializer.js | 2 +- .../main/resources/swagger-ui/yaml/cache.yaml | 427 -- .../main/resources/swagger-ui/yaml/repo.yaml | 112 +- .../main/resources/swagger-ui/yaml/roles.yaml | 44 +- .../resources/swagger-ui/yaml/search.yaml | 305 ++ .../resources/swagger-ui/yaml/settings.yaml | 62 +- .../resources/swagger-ui/yaml/token-gen.yaml | 26 +- .../main/resources/swagger-ui/yaml/users.yaml | 122 +- .../com/artipie/api/RestApiServerBase.java | 4 +- .../api/perms/RestApiPermissionTest.java | 54 + .../com/artipie/api/v1/ApiResponseTest.java | 71 + .../artipie/api/v1/ArtifactHandlerTest.java | 67 + .../com/artipie/api/v1/AsyncApiTestBase.java | 209 + .../artipie/api/v1/AsyncApiVerticleTest.java | 37 + .../com/artipie/api/v1/AuthHandlerTest.java | 69 + .../artipie/api/v1/CooldownHandlerTest.java | 68 + .../artipie/api/v1/DashboardHandlerTest.java | 80 + .../artipie/api/v1/RepositoryHandlerTest.java | 145 + .../com/artipie/api/v1/RoleHandlerTest.java | 93 + .../com/artipie/api/v1/SearchHandlerTest.java | 120 + .../artipie/api/v1/SettingsHandlerTest.java | 56 + .../api/v1/StorageAliasHandlerTest.java | 100 + .../com/artipie/api/v1/UserHandlerTest.java | 111 + .../cache/CacheInvalidationPubSubTest.java | 294 + .../artipie/cluster/DbNodeRegistryTest.java | 242 + .../com/artipie/cluster/NodeRegistryTest.java | 48 + .../cooldown/JdbcCooldownServiceTest.java | 22 +- .../java/com/artipie/db/DbManagerTest.java | 77 + .../com/artipie/db/DeadLetterWriterTest.java | 87 + .../db/SettingsLayerIntegrationTest.java | 116 + .../com/artipie/db/dao/AuditLogDaoTest.java | 82 + .../artipie/db/dao/AuthProviderDaoTest.java | 121 + .../com/artipie/db/dao/RepositoryDaoTest.java | 139 + .../java/com/artipie/db/dao/RoleDaoTest.java | 121 + .../com/artipie/db/dao/SettingsDaoTest.java | 95 + .../artipie/db/dao/StorageAliasDaoTest.java | 121 + .../java/com/artipie/db/dao/UserDaoTest.java | 166 + .../db/migration/YamlToDbMigratorTest.java | 242 + .../artipie/group/ArtifactNameParserTest.java | 683 +++ .../artipie/group/GroupMetadataCacheTest.java | 97 + .../com/artipie/group/LocateHitRateTest.java | 402 ++ .../artipie/group/MavenGroupSliceTest.java | 52 + .../com/artipie/group/MemberSliceTest.java | 62 + .../com/artipie/group/RoutingRuleTest.java | 96 + .../artipie/group/WritableGroupSliceTest.java | 89 + .../com/artipie/http/ApiRoutingSliceTest.java | 51 +- .../com/artipie/http/HealthSliceTest.java | 99 +- .../artipie/http/SliceByPathPrefixTest.java | 159 +- .../artipie/index/DbArtifactIndexTest.java | 438 ++ .../artipie/proxy/OfflineAwareSliceTest.java | 66 + .../scheduling/QuartzServiceJdbcTest.java | 182 + .../artipie/scheduling/QuartzServiceTest.java | 14 + .../scheduling/TempFileCleanupJobTest.java | 225 + .../security/policy/CachedDbPolicyTest.java | 294 + .../artipie/settings/YamlSettingsTest.java | 23 + .../cache/PublishingFiltersCacheTest.java | 222 + .../artipie/webhook/WebhookConfigTest.java | 72 + .../webhook/WebhookDispatcherTest.java | 45 + artipie-ui/.gitignore | 4 + artipie-ui/Dockerfile | 17 + artipie-ui/README.md | 145 + artipie-ui/docker-entrypoint.sh | 14 + artipie-ui/env.d.ts | 9 + artipie-ui/index.html | 13 + artipie-ui/nginx/default.conf | 33 + artipie-ui/package-lock.json | 4758 +++++++++++++++++ artipie-ui/package.json | 40 + artipie-ui/public/config.json | 6 + artipie-ui/public/config.json.template | 6 + artipie-ui/src/App.vue | 14 + artipie-ui/src/api/__tests__/client.test.ts | 28 + artipie-ui/src/api/auth.ts | 82 + artipie-ui/src/api/client.ts | 40 + artipie-ui/src/api/repos.ts | 97 + artipie-ui/src/api/roles.ts | 30 + artipie-ui/src/api/search.ts | 26 + artipie-ui/src/api/settings.ts | 90 + artipie-ui/src/api/users.ts | 34 + artipie-ui/src/assets/main.css | 57 + .../src/components/common/HealthIndicator.vue | 39 + .../src/components/common/RepoTypeBadge.vue | 32 + .../src/components/layout/AppHeader.vue | 147 + .../src/components/layout/AppLayout.vue | 81 + .../src/components/layout/AppSidebar.vue | 119 + .../layout/__tests__/AppLayout.test.ts | 23 + .../__tests__/usePagination.test.ts | 32 + .../src/composables/useConfirmDelete.ts | 29 + .../src/composables/useCursorPagination.ts | 47 + .../src/composables/useDebouncedSearch.ts | 23 + artipie-ui/src/composables/usePagination.ts | 49 + artipie-ui/src/composables/usePermission.ts | 13 + artipie-ui/src/main.ts | 65 + artipie-ui/src/router/__tests__/index.test.ts | 24 + artipie-ui/src/router/index.ts | 149 + .../src/stores/__tests__/config.test.ts | 28 + artipie-ui/src/stores/__tests__/theme.test.ts | 34 + artipie-ui/src/stores/auth.ts | 108 + artipie-ui/src/stores/config.ts | 19 + artipie-ui/src/stores/notifications.ts | 34 + artipie-ui/src/stores/theme.ts | 25 + artipie-ui/src/types/index.ts | 238 + artipie-ui/src/utils/repoTypes.ts | 184 + artipie-ui/src/views/NotFoundView.vue | 11 + artipie-ui/src/views/admin/CooldownView.vue | 203 + artipie-ui/src/views/admin/RepoCreateView.vue | 323 ++ artipie-ui/src/views/admin/RepoEditView.vue | 68 + .../src/views/admin/RepoManagementView.vue | 131 + artipie-ui/src/views/admin/RoleDetailView.vue | 70 + artipie-ui/src/views/admin/RoleListView.vue | 535 ++ artipie-ui/src/views/admin/SettingsView.vue | 749 +++ .../src/views/admin/StorageAliasView.vue | 293 + artipie-ui/src/views/admin/UserDetailView.vue | 152 + artipie-ui/src/views/admin/UserListView.vue | 171 + artipie-ui/src/views/auth/LoginView.vue | 238 + .../src/views/auth/OAuthCallbackView.vue | 63 + .../views/auth/__tests__/LoginView.test.ts | 38 + .../src/views/dashboard/DashboardView.vue | 209 + .../dashboard/__tests__/DashboardView.test.ts | 34 + artipie-ui/src/views/profile/ProfileView.vue | 225 + artipie-ui/src/views/repos/RepoDetailView.vue | 304 ++ artipie-ui/src/views/repos/RepoListView.vue | 128 + .../repos/__tests__/RepoDetailView.test.ts | 33 + .../repos/__tests__/RepoListView.test.ts | 32 + artipie-ui/src/views/search/SearchView.vue | 316 ++ .../views/search/__tests__/SearchView.test.ts | 30 + artipie-ui/tsconfig.app.json | 21 + artipie-ui/tsconfig.json | 7 + artipie-ui/tsconfig.node.json | 9 + artipie-ui/vite.config.ts | 34 + asto/asto-artipie/pom.xml | 6 +- asto/asto-core/pom.xml | 2 +- .../main/java/com/artipie/asto/Content.java | 15 +- .../artipie/asto/cache/FromRemoteCache.java | 67 +- .../artipie/asto/cache/FromStorageCache.java | 65 +- .../asto/cache/OptimizedStorageCache.java | 2 +- .../asto/cache/StreamThroughCache.java | 311 ++ .../com/artipie/asto/events/EventQueue.java | 77 +- .../artipie/asto/events/EventsProcessor.java | 4 +- .../java/com/artipie/asto/fs/FileStorage.java | 27 +- .../main/java/com/artipie/asto/fs/RxFile.java | 5 +- .../lock/storage/LockCleanupScheduler.java | 182 + .../artipie/asto/lock/storage/Proposals.java | 43 + .../artipie/asto/streams/ContentAsStream.java | 3 +- .../java/com/artipie/asto/RxFileTest.java | 23 + .../asto/cache/StreamThroughCacheTest.java | 68 + .../artipie/asto/events/EventQueueTest.java | 145 + .../storage/LockCleanupSchedulerTest.java | 177 + asto/asto-etcd/pom.xml | 51 - .../java/com/artipie/asto/etcd/EtcdMeta.java | 40 - .../com/artipie/asto/etcd/EtcdStorage.java | 195 - .../artipie/asto/etcd/EtcdStorageFactory.java | 34 - .../com/artipie/asto/etcd/package-info.java | 11 - .../asto/etcd/EtcdStorageFactoryTest.java | 42 - .../artipie/asto/etcd/EtcdStorageITCase.java | 184 - .../etcd/EtcdStorageVerificationTest.java | 65 - .../com/artipie/asto/etcd/package-info.java | 12 - asto/asto-redis/pom.xml | 62 - .../com/artipie/asto/redis/RedisStorage.java | 274 - .../asto/redis/RedisStorageFactory.java | 46 - .../com/artipie/asto/redis/package-info.java | 11 - .../asto/redis/RedisStorageFactoryTest.java | 157 - .../artipie/asto/redis/RedisStorageTest.java | 290 - .../RedisStorageWhiteboxVerificationTest.java | 78 - .../com/artipie/asto/redis/package-info.java | 11 - asto/asto-s3/pom.xml | 14 +- .../com/artipie/asto/s3/DiskCacheStorage.java | 196 +- .../asto/s3/EstimatedContentCompliment.java | 1 + .../com/artipie/asto/s3/MultipartUpload.java | 55 +- .../asto/s3/S3ExpressStorageFactory.java | 13 +- .../java/com/artipie/asto/s3/S3Storage.java | 153 +- .../com/artipie/asto/s3/S3StorageFactory.java | 13 +- .../S3StorageWhiteboxVerificationTest.java | 8 + .../java/com/artipie/asto/s3/BucketTest.java | 8 + .../artipie/asto/s3/DiskCacheStorageTest.java | 170 + .../com/artipie/asto/s3/S3StorageTest.java | 48 + asto/asto-vertx-file/pom.xml | 4 +- .../com/artipie/asto/fs/VertxFileStorage.java | 77 + .../VertxFileStorageHierarchicalListTest.java | 141 + asto/pom.xml | 12 +- benchmark/README.md | 114 + benchmark/bench.sh | 287 + benchmark/docker-compose-bench.yml | 146 + benchmark/report/generate-report.sh | 210 + benchmark/scenarios/common.sh | 167 + benchmark/scenarios/docker-bench.sh | 369 ++ benchmark/scenarios/maven-bench.sh | 359 ++ benchmark/scenarios/npm-bench.sh | 386 ++ benchmark/setup/artipie-new.yml | 41 + benchmark/setup/artipie-old.yml | 41 + benchmark/setup/init-db.sql | 7 + benchmark/setup/log4j2-bench.xml | 14 + benchmark/setup/repos-new/docker_local.yaml | 5 + benchmark/setup/repos-new/docker_proxy.yaml | 7 + benchmark/setup/repos-new/maven.yaml | 5 + benchmark/setup/repos-new/maven_group.yaml | 5 + benchmark/setup/repos-new/maven_proxy.yaml | 7 + benchmark/setup/repos-new/npm.yaml | 6 + benchmark/setup/repos-new/npm_group.yaml | 5 + benchmark/setup/repos-new/npm_proxy.yaml | 9 + benchmark/setup/repos-old/docker_local.yaml | 5 + benchmark/setup/repos-old/docker_proxy.yaml | 7 + benchmark/setup/repos-old/maven.yaml | 5 + benchmark/setup/repos-old/maven_group.yaml | 5 + benchmark/setup/repos-old/maven_proxy.yaml | 7 + benchmark/setup/repos-old/npm.yaml | 6 + benchmark/setup/repos-old/npm_group.yaml | 5 + benchmark/setup/repos-old/npm_proxy.yaml | 9 + benchmark/setup/repos/docker_local.yaml | 5 + benchmark/setup/repos/docker_proxy.yaml | 7 + benchmark/setup/repos/maven.yaml | 5 + benchmark/setup/repos/maven_group.yaml | 5 + benchmark/setup/repos/maven_proxy.yaml | 7 + benchmark/setup/repos/npm.yaml | 6 + benchmark/setup/repos/npm_group.yaml | 5 + benchmark/setup/repos/npm_proxy.yaml | 9 + benchmark/setup/security/roles/admin.yaml | 2 + benchmark/setup/security/users/artipie.yaml | 5 + benchmark/setup/settings-new.xml | 33 + benchmark/setup/settings-old.xml | 33 + build-tools/pom.xml | 2 +- composer-adapter/pom.xml | 12 +- .../composer/http/proxy/CachedProxySlice.java | 247 +- .../proxy/ComposerProxyPackageProcessor.java | 78 +- .../http/proxy/MetadataUrlRewriter.java | 4 +- .../http/proxy/ProxyDownloadSlice.java | 178 +- conan-adapter/pom.xml | 10 +- conda-adapter/benchmarks/pom.xml | 6 +- conda-adapter/pom.xml | 12 +- .../java/com/artipie/conda/MultiRepodata.java | 2 + debian-adapter/benchmarks/pom.xml | 6 +- debian-adapter/pom.xml | 12 +- docker-adapter/pom.xml | 12 +- .../java/com/artipie/docker/Manifests.java | 12 + .../artipie/docker/asto/AstoManifests.java | 13 + .../com/artipie/docker/cache/CacheLayers.java | 31 +- .../artipie/docker/cache/CacheManifests.java | 299 +- .../com/artipie/docker/cache/CachingBlob.java | 217 + .../cache/DockerProxyCooldownInspector.java | 8 +- .../docker/composite/MultiReadManifests.java | 55 +- .../com/artipie/docker/http/DockerSlice.java | 16 +- .../com/artipie/docker/http/PathPatterns.java | 1 + .../artipie/docker/http/ReferrersSlice.java | 69 + .../docker/http/blobs/GetBlobsSlice.java | 15 + .../docker/http/blobs/HeadBlobsSlice.java | 15 + .../http/manifest/GetManifestSlice.java | 54 +- .../http/manifest/HeadManifestSlice.java | 20 +- .../http/manifest/PushManifestSlice.java | 83 +- .../com/artipie/docker/manifest/Manifest.java | 24 +- .../artipie/docker/proxy/ProxyManifests.java | 33 +- .../docker/asto/AstoManifestsTest.java | 26 +- .../docker/cache/CacheManifestsTest.java | 139 +- .../artipie/docker/cache/CachingBlobTest.java | 277 + .../docker/http/ReferrersSliceTest.java | 123 + .../manifest/GetManifestSliceMdcTest.java | 173 + .../manifest/HeadManifestSliceMdcTest.java | 166 + .../artipie/docker/manifest/ManifestTest.java | 32 + docs/CHANGELOG-AUTO1.md | 80 + docs/DEVELOPER_GUIDE.md | 790 ++- docs/ENVIRONMENT_VARIABLES.md | 208 + docs/RELEASE-NOTES-v1.21.0.md | 494 ++ docs/USER_GUIDE.md | 379 +- docs/ha-deployment/artipie-ha.yml | 158 + docs/ha-deployment/docker-compose-ha.yml | 247 + docs/ha-deployment/nginx-ha.conf | 169 + files-adapter/pom.xml | 12 +- .../com/artipie/files/FileProxySlice.java | 56 +- .../artipie/files/FilesCooldownInspector.java | 7 +- .../com/artipie/files/FileProxySliceTest.java | 8 +- gem-adapter/pom.xml | 10 +- .../src/main/java/com/artipie/gem/Gem.java | 6 +- .../java/com/artipie/gem/http/GemSlice.java | 65 + go-adapter/pom.xml | 12 +- .../goproxy/GoProxyPackageProcessor.java | 53 +- .../java/com/artipie/goproxy/Goproxy.java | 1 + .../com/artipie/http/CachedProxySlice.java | 8 +- .../com/artipie/http/GoCooldownInspector.java | 3 +- gradle-adapter/README.md | 167 - gradle-adapter/pom.xml | 98 - .../main/java/com/artipie/gradle/Gradle.java | 42 - .../gradle/GradleProxyPackageProcessor.java | 325 -- .../com/artipie/gradle/asto/AstoGradle.java | 49 - .../com/artipie/gradle/asto/package-info.java | 11 - .../artipie/gradle/http/CachedProxySlice.java | 677 --- .../gradle/http/GradleCooldownInspector.java | 319 -- .../gradle/http/GradleMetadataReader.java | 96 - .../gradle/http/GradlePathBuilder.java | 49 - .../artipie/gradle/http/GradleProxySlice.java | 154 - .../com/artipie/gradle/http/GradleSlice.java | 449 -- .../artipie/gradle/http/HeadProxySlice.java | 58 - .../com/artipie/gradle/http/PomParser.java | 97 - .../com/artipie/gradle/http/RepoHead.java | 60 - .../com/artipie/gradle/http/package-info.java | 11 - .../java/com/artipie/gradle/package-info.java | 11 - .../artipie/gradle/asto/AstoGradleTest.java | 62 - .../artipie/gradle/http/GradleProxyIT.java | 119 - .../artipie/gradle/http/GradleSliceTest.java | 166 - helm-adapter/benchmarks/pom.xml | 6 +- helm-adapter/pom.xml | 10 +- .../src/main/java/com/artipie/helm/Helm.java | 6 + hexpm-adapter/pom.xml | 8 +- http-client/pom.xml | 8 +- .../http/client/HttpClientSettings.java | 73 + .../http/client/auth/OAuthTokenFormat.java | 21 +- .../http/client/jetty/JettyClientSlice.java | 172 +- .../http/client/jetty/JettyClientSlices.java | 29 +- .../http/client/HttpClientSettingsTest.java | 33 + .../client/auth/OAuthTokenFormatTest.java | 22 +- .../JettyClientSliceChunkLifecycleTest.java | 19 +- .../jetty/JettyClientSliceGzipTest.java | 221 + maven-adapter/pom.xml | 12 +- .../maven/MavenProxyPackageProcessor.java | 53 +- .../artipie/maven/http/CachedProxySlice.java | 889 +-- .../artipie/maven/http/LocalMavenSlice.java | 8 +- .../artipie/maven/http/MavenProxySlice.java | 97 +- .../com/artipie/maven/http/MetadataCache.java | 82 +- .../com/artipie/maven/http/NegativeCache.java | 48 +- .../com/artipie/maven/http/UploadSlice.java | 7 +- .../artipie/maven/metadata/MavenMetadata.java | 4 +- .../maven/metadata/MavenTimestamp.java | 52 + .../maven/metadata/MetadataMerger.java | 4 +- .../artipie/maven/http/MetadataCacheTest.java | 38 +- .../maven/metadata/MavenTimestampTest.java | 59 + .../maven/metadata/MetadataMergerTest.java | 6 +- npm-adapter/pom.xml | 12 +- .../npm/cooldown/NpmMetadataParser.java | 8 +- .../npm/events/NpmProxyPackageProcessor.java | 53 +- .../java/com/artipie/npm/http/NpmSlice.java | 9 - .../npm/http/audit/GroupAuditSlice.java | 16 +- .../npm/http/search/InMemoryPackageIndex.java | 5 +- .../com/artipie/npm/misc/MetadataETag.java | 27 +- .../artipie/npm/misc/MetadataEnhancer.java | 22 +- .../npm/misc/NextSafeAvailablePort.java | 10 +- .../com/artipie/npm/proxy/HttpNpmRemote.java | 82 +- .../java/com/artipie/npm/proxy/NpmProxy.java | 104 +- .../artipie/npm/proxy/NpmProxyStorage.java | 9 + .../artipie/npm/proxy/RxNpmProxyStorage.java | 32 +- .../npm/proxy/http/CachedNpmProxySlice.java | 330 +- .../npm/proxy/http/DownloadAssetSlice.java | 20 +- .../npm/proxy/http/DownloadPackageSlice.java | 104 +- .../npm/proxy/http/NpmCooldownInspector.java | 8 +- .../artipie/npm/proxy/model/NpmPackage.java | 98 +- .../java/com/artipie/npm/NpmDeprecateIT.java | 3 +- .../java/com/artipie/npm/NpmDistTagsIT.java | 3 +- .../src/test/java/com/artipie/npm/NpmIT.java | 3 +- .../java/com/artipie/npm/NpmUnpublishIT.java | 4 +- .../java/com/artipie/npm/http/CurlPutIT.java | 3 +- .../artipie/npm/http/InstallCurlPutIT.java | 3 +- .../com/artipie/npm/proxy/NpmProxyTest.java | 22 +- nuget-adapter/pom.xml | 10 +- pom.xml | 30 +- pypi-adapter/pom.xml | 12 +- .../artipie/pypi/PyProxyPackageProcessor.java | 53 +- .../artipie/pypi/http/CachedPyProxySlice.java | 7 +- .../com/artipie/pypi/http/ProxySlice.java | 471 +- .../com/artipie/pypi/http/PyProxySlice.java | 3 +- .../com/artipie/pypi/http/ProxySliceTest.java | 8 +- rpm-adapter/benchmarks/pom.xml | 6 +- rpm-adapter/pom.xml | 12 +- .../src/main/java/com/artipie/rpm/Rpm.java | 60 - .../java/com/artipie/rpm/RpmMetadata.java | 2 + .../java/com/artipie/rpm/http/RpmSlice.java | 16 - .../java/com/artipie/rpm/meta/XmlAlter.java | 1 + .../rpm/http/RpmSliceDownloadITCase.java | 2 +- .../com/artipie/rpm/http/RpmSliceITCase.java | 2 +- .../artipie/rpm/http/RpmSliceS3ITCase.java | 2 +- vertx-server/pom.xml | 6 +- .../vertx/GuardedHttpServerResponse.java | 33 +- .../com/artipie/vertx/VertxSliceServer.java | 306 +- .../vertx/VertxSliceServerRobustnessTest.java | 85 + .../artipie/vertx/VertxSliceServerTest.java | 427 ++ 598 files changed, 60536 insertions(+), 10512 deletions(-) create mode 100644 .wiki/Configuration-HA.md create mode 100644 .wiki/Configuration-Health.md create mode 100644 artipie-backfill/README.md create mode 100644 artipie-backfill/pom.xml create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/ArtifactRecord.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/ArtipieMetaSidecar.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/BackfillCli.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/BatchInserter.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/BulkBackfillRunner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/ComposerScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/DebianScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/DockerScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/FileScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/GemScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/GoScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/HelmScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/MavenScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/NpmScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/ProgressReporter.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/PypiScanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/RepoConfigYaml.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/RepoEntry.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/RepoTypeNormalizer.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/Scanner.java create mode 100644 artipie-backfill/src/main/java/com/artipie/backfill/ScannerFactory.java create mode 100644 artipie-backfill/src/main/resources/META-INF/services/org.apache.logging.log4j.spi.Provider create mode 100644 artipie-backfill/src/main/resources/log4j2.xml create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/BackfillCliTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/BackfillIntegrationTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/BatchInserterTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/BulkBackfillRunnerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/ComposerScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/DebianScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/DockerScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/GemScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/GoScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/HelmScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/MavenScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/NpmScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/ProgressReporterTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/PypiScannerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/RepoConfigYamlTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/RepoTypeNormalizerTest.java create mode 100644 artipie-backfill/src/test/java/com/artipie/backfill/ScannerFactoryTest.java create mode 100644 artipie-core/src/main/java/com/artipie/asto/dedup/ContentAddressableStorage.java create mode 100644 artipie-core/src/main/java/com/artipie/cache/CacheInvalidationPubSub.java create mode 100644 artipie-core/src/main/java/com/artipie/cache/PublishingCleanable.java create mode 100644 artipie-core/src/main/java/com/artipie/cluster/ClusterEventBus.java create mode 100644 artipie-core/src/main/java/com/artipie/http/cache/BaseCachedProxySlice.java create mode 100644 artipie-core/src/main/java/com/artipie/http/cache/ConditionalRequest.java create mode 100644 artipie-core/src/main/java/com/artipie/http/cache/DedupStrategy.java create mode 100644 artipie-core/src/main/java/com/artipie/http/cache/DigestComputer.java create mode 100644 artipie-core/src/main/java/com/artipie/http/cache/NegativeCacheRegistry.java create mode 100644 artipie-core/src/main/java/com/artipie/http/cache/RequestDeduplicator.java create mode 100644 artipie-core/src/main/java/com/artipie/http/cache/SidecarFile.java create mode 100644 artipie-core/src/main/java/com/artipie/http/misc/ConfigDefaults.java create mode 100644 artipie-core/src/main/java/com/artipie/http/misc/DispatchedStorage.java create mode 100644 artipie-core/src/main/java/com/artipie/http/misc/RepoNameMeterFilter.java create mode 100644 artipie-core/src/main/java/com/artipie/http/misc/StorageExecutors.java create mode 100644 artipie-core/src/main/java/com/artipie/http/retry/RetrySlice.java create mode 100644 artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockRegistry.java create mode 100644 artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockSettings.java create mode 100644 artipie-core/src/main/java/com/artipie/http/timeout/BlockState.java create mode 100644 artipie-core/src/main/java/com/artipie/http/timeout/TimeoutSettings.java create mode 100644 artipie-core/src/main/java/com/artipie/index/ArtifactDocument.java create mode 100644 artipie-core/src/main/java/com/artipie/index/ArtifactIndex.java create mode 100644 artipie-core/src/main/java/com/artipie/index/SearchResult.java delete mode 100644 artipie-core/src/main/java/com/artipie/layout/GradleLayout.java create mode 100644 artipie-core/src/main/java/com/artipie/scheduling/JobDataRegistry.java create mode 100644 artipie-core/src/test/java/com/artipie/cache/ValkeyConnectionTest.java create mode 100644 artipie-core/src/test/java/com/artipie/cluster/ClusterEventBusTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/cache/BaseCachedProxySliceContentEncodingTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/cache/ConditionalRequestTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/cache/DedupStrategyTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/cache/DigestComputerTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/cache/NegativeCacheRegistryTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/cache/RequestDeduplicatorTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/cache/SidecarFileTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/misc/ConfigDefaultsTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/misc/DispatchedStorageTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/misc/RepoNameMeterFilterTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/misc/StorageExecutorsTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/retry/RetrySliceTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/slice/CircuitBreakerSliceTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/timeout/AutoBlockRegistryTest.java create mode 100644 artipie-core/src/test/java/com/artipie/http/timeout/TimeoutSettingsTest.java delete mode 100644 artipie-core/src/test/java/com/artipie/layout/GradleLayoutTest.java create mode 100644 artipie-main/docker-compose/artipie/security/users/ayd.asraf@auto1.local.yml delete mode 100644 artipie-main/src/main/java/com/artipie/adapters/gradle/GradleProxy.java delete mode 100644 artipie-main/src/main/java/com/artipie/api/CacheRest.java create mode 100644 artipie-main/src/main/java/com/artipie/api/SearchRest.java delete mode 100644 artipie-main/src/main/java/com/artipie/api/perms/ApiCachePermission.java create mode 100644 artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermission.java create mode 100644 artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermissionFactory.java create mode 100644 artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermission.java create mode 100644 artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermissionFactory.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/ApiResponse.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/ArtifactHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/AsyncApiVerticle.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/AuthHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/CooldownHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/DashboardHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/RepositoryHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/RoleHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/SearchHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/SettingsHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/StorageAliasHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/api/v1/UserHandler.java create mode 100644 artipie-main/src/main/java/com/artipie/auth/AuthFromDb.java create mode 100644 artipie-main/src/main/java/com/artipie/cluster/DbNodeRegistry.java create mode 100644 artipie-main/src/main/java/com/artipie/cluster/NodeRegistry.java create mode 100644 artipie-main/src/main/java/com/artipie/db/DbManager.java create mode 100644 artipie-main/src/main/java/com/artipie/db/DeadLetterWriter.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/AuditLogDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/AuthProviderDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/RepositoryDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/RoleDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/SettingsDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/StorageAliasDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/UserDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/dao/UserTokenDao.java create mode 100644 artipie-main/src/main/java/com/artipie/db/migration/YamlToDbMigrator.java create mode 100644 artipie-main/src/main/java/com/artipie/group/ArtifactNameParser.java delete mode 100644 artipie-main/src/main/java/com/artipie/group/GroupNegativeCache.java create mode 100644 artipie-main/src/main/java/com/artipie/group/RoutingRule.java create mode 100644 artipie-main/src/main/java/com/artipie/group/WritableGroupSlice.java create mode 100644 artipie-main/src/main/java/com/artipie/index/DbArtifactIndex.java create mode 100644 artipie-main/src/main/java/com/artipie/proxy/OfflineAwareSlice.java create mode 100644 artipie-main/src/main/java/com/artipie/scheduling/ArtipieQuartzConnectionProvider.java create mode 100644 artipie-main/src/main/java/com/artipie/scheduling/QuartzSchema.java create mode 100644 artipie-main/src/main/java/com/artipie/scheduling/TempFileCleanupJob.java create mode 100644 artipie-main/src/main/java/com/artipie/security/policy/CachedDbPolicy.java create mode 100644 artipie-main/src/main/java/com/artipie/settings/cache/PublishingFiltersCache.java create mode 100644 artipie-main/src/main/java/com/artipie/settings/repo/DualCrudRepoSettings.java create mode 100644 artipie-main/src/main/java/com/artipie/settings/users/DualCrudUsers.java create mode 100644 artipie-main/src/main/java/com/artipie/webhook/WebhookConfig.java create mode 100644 artipie-main/src/main/java/com/artipie/webhook/WebhookDispatcher.java create mode 100644 artipie-main/src/main/resources/db/migration/V100__create_settings_tables.sql create mode 100644 artipie-main/src/main/resources/db/migration/V101__create_user_tokens_table.sql delete mode 100644 artipie-main/src/main/resources/swagger-ui/yaml/cache.yaml create mode 100644 artipie-main/src/main/resources/swagger-ui/yaml/search.yaml create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/ApiResponseTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/ArtifactHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/AsyncApiTestBase.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/AsyncApiVerticleTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/AuthHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/CooldownHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/DashboardHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/RepositoryHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/RoleHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/SearchHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/SettingsHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/StorageAliasHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/api/v1/UserHandlerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/cache/CacheInvalidationPubSubTest.java create mode 100644 artipie-main/src/test/java/com/artipie/cluster/DbNodeRegistryTest.java create mode 100644 artipie-main/src/test/java/com/artipie/cluster/NodeRegistryTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/DbManagerTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/DeadLetterWriterTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/SettingsLayerIntegrationTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/dao/AuditLogDaoTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/dao/AuthProviderDaoTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/dao/RepositoryDaoTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/dao/RoleDaoTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/dao/SettingsDaoTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/dao/StorageAliasDaoTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/dao/UserDaoTest.java create mode 100644 artipie-main/src/test/java/com/artipie/db/migration/YamlToDbMigratorTest.java create mode 100644 artipie-main/src/test/java/com/artipie/group/ArtifactNameParserTest.java create mode 100644 artipie-main/src/test/java/com/artipie/group/GroupMetadataCacheTest.java create mode 100644 artipie-main/src/test/java/com/artipie/group/LocateHitRateTest.java create mode 100644 artipie-main/src/test/java/com/artipie/group/MemberSliceTest.java create mode 100644 artipie-main/src/test/java/com/artipie/group/RoutingRuleTest.java create mode 100644 artipie-main/src/test/java/com/artipie/group/WritableGroupSliceTest.java create mode 100644 artipie-main/src/test/java/com/artipie/index/DbArtifactIndexTest.java create mode 100644 artipie-main/src/test/java/com/artipie/proxy/OfflineAwareSliceTest.java create mode 100644 artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceJdbcTest.java create mode 100644 artipie-main/src/test/java/com/artipie/scheduling/TempFileCleanupJobTest.java create mode 100644 artipie-main/src/test/java/com/artipie/security/policy/CachedDbPolicyTest.java create mode 100644 artipie-main/src/test/java/com/artipie/settings/cache/PublishingFiltersCacheTest.java create mode 100644 artipie-main/src/test/java/com/artipie/webhook/WebhookConfigTest.java create mode 100644 artipie-main/src/test/java/com/artipie/webhook/WebhookDispatcherTest.java create mode 100644 artipie-ui/.gitignore create mode 100644 artipie-ui/Dockerfile create mode 100644 artipie-ui/README.md create mode 100755 artipie-ui/docker-entrypoint.sh create mode 100644 artipie-ui/env.d.ts create mode 100644 artipie-ui/index.html create mode 100644 artipie-ui/nginx/default.conf create mode 100644 artipie-ui/package-lock.json create mode 100644 artipie-ui/package.json create mode 100644 artipie-ui/public/config.json create mode 100644 artipie-ui/public/config.json.template create mode 100644 artipie-ui/src/App.vue create mode 100644 artipie-ui/src/api/__tests__/client.test.ts create mode 100644 artipie-ui/src/api/auth.ts create mode 100644 artipie-ui/src/api/client.ts create mode 100644 artipie-ui/src/api/repos.ts create mode 100644 artipie-ui/src/api/roles.ts create mode 100644 artipie-ui/src/api/search.ts create mode 100644 artipie-ui/src/api/settings.ts create mode 100644 artipie-ui/src/api/users.ts create mode 100644 artipie-ui/src/assets/main.css create mode 100644 artipie-ui/src/components/common/HealthIndicator.vue create mode 100644 artipie-ui/src/components/common/RepoTypeBadge.vue create mode 100644 artipie-ui/src/components/layout/AppHeader.vue create mode 100644 artipie-ui/src/components/layout/AppLayout.vue create mode 100644 artipie-ui/src/components/layout/AppSidebar.vue create mode 100644 artipie-ui/src/components/layout/__tests__/AppLayout.test.ts create mode 100644 artipie-ui/src/composables/__tests__/usePagination.test.ts create mode 100644 artipie-ui/src/composables/useConfirmDelete.ts create mode 100644 artipie-ui/src/composables/useCursorPagination.ts create mode 100644 artipie-ui/src/composables/useDebouncedSearch.ts create mode 100644 artipie-ui/src/composables/usePagination.ts create mode 100644 artipie-ui/src/composables/usePermission.ts create mode 100644 artipie-ui/src/main.ts create mode 100644 artipie-ui/src/router/__tests__/index.test.ts create mode 100644 artipie-ui/src/router/index.ts create mode 100644 artipie-ui/src/stores/__tests__/config.test.ts create mode 100644 artipie-ui/src/stores/__tests__/theme.test.ts create mode 100644 artipie-ui/src/stores/auth.ts create mode 100644 artipie-ui/src/stores/config.ts create mode 100644 artipie-ui/src/stores/notifications.ts create mode 100644 artipie-ui/src/stores/theme.ts create mode 100644 artipie-ui/src/types/index.ts create mode 100644 artipie-ui/src/utils/repoTypes.ts create mode 100644 artipie-ui/src/views/NotFoundView.vue create mode 100644 artipie-ui/src/views/admin/CooldownView.vue create mode 100644 artipie-ui/src/views/admin/RepoCreateView.vue create mode 100644 artipie-ui/src/views/admin/RepoEditView.vue create mode 100644 artipie-ui/src/views/admin/RepoManagementView.vue create mode 100644 artipie-ui/src/views/admin/RoleDetailView.vue create mode 100644 artipie-ui/src/views/admin/RoleListView.vue create mode 100644 artipie-ui/src/views/admin/SettingsView.vue create mode 100644 artipie-ui/src/views/admin/StorageAliasView.vue create mode 100644 artipie-ui/src/views/admin/UserDetailView.vue create mode 100644 artipie-ui/src/views/admin/UserListView.vue create mode 100644 artipie-ui/src/views/auth/LoginView.vue create mode 100644 artipie-ui/src/views/auth/OAuthCallbackView.vue create mode 100644 artipie-ui/src/views/auth/__tests__/LoginView.test.ts create mode 100644 artipie-ui/src/views/dashboard/DashboardView.vue create mode 100644 artipie-ui/src/views/dashboard/__tests__/DashboardView.test.ts create mode 100644 artipie-ui/src/views/profile/ProfileView.vue create mode 100644 artipie-ui/src/views/repos/RepoDetailView.vue create mode 100644 artipie-ui/src/views/repos/RepoListView.vue create mode 100644 artipie-ui/src/views/repos/__tests__/RepoDetailView.test.ts create mode 100644 artipie-ui/src/views/repos/__tests__/RepoListView.test.ts create mode 100644 artipie-ui/src/views/search/SearchView.vue create mode 100644 artipie-ui/src/views/search/__tests__/SearchView.test.ts create mode 100644 artipie-ui/tsconfig.app.json create mode 100644 artipie-ui/tsconfig.json create mode 100644 artipie-ui/tsconfig.node.json create mode 100644 artipie-ui/vite.config.ts create mode 100644 asto/asto-core/src/main/java/com/artipie/asto/cache/StreamThroughCache.java create mode 100644 asto/asto-core/src/main/java/com/artipie/asto/lock/storage/LockCleanupScheduler.java create mode 100644 asto/asto-core/src/test/java/com/artipie/asto/cache/StreamThroughCacheTest.java create mode 100644 asto/asto-core/src/test/java/com/artipie/asto/events/EventQueueTest.java create mode 100644 asto/asto-core/src/test/java/com/artipie/asto/lock/storage/LockCleanupSchedulerTest.java delete mode 100644 asto/asto-etcd/pom.xml delete mode 100644 asto/asto-etcd/src/main/java/com/artipie/asto/etcd/EtcdMeta.java delete mode 100644 asto/asto-etcd/src/main/java/com/artipie/asto/etcd/EtcdStorage.java delete mode 100644 asto/asto-etcd/src/main/java/com/artipie/asto/etcd/EtcdStorageFactory.java delete mode 100644 asto/asto-etcd/src/main/java/com/artipie/asto/etcd/package-info.java delete mode 100644 asto/asto-etcd/src/test/java/com/artipie/asto/etcd/EtcdStorageFactoryTest.java delete mode 100644 asto/asto-etcd/src/test/java/com/artipie/asto/etcd/EtcdStorageITCase.java delete mode 100644 asto/asto-etcd/src/test/java/com/artipie/asto/etcd/EtcdStorageVerificationTest.java delete mode 100644 asto/asto-etcd/src/test/java/com/artipie/asto/etcd/package-info.java delete mode 100644 asto/asto-redis/pom.xml delete mode 100644 asto/asto-redis/src/main/java/com/artipie/asto/redis/RedisStorage.java delete mode 100644 asto/asto-redis/src/main/java/com/artipie/asto/redis/RedisStorageFactory.java delete mode 100644 asto/asto-redis/src/main/java/com/artipie/asto/redis/package-info.java delete mode 100644 asto/asto-redis/src/test/java/com/artipie/asto/redis/RedisStorageFactoryTest.java delete mode 100644 asto/asto-redis/src/test/java/com/artipie/asto/redis/RedisStorageTest.java delete mode 100644 asto/asto-redis/src/test/java/com/artipie/asto/redis/RedisStorageWhiteboxVerificationTest.java delete mode 100644 asto/asto-redis/src/test/java/com/artipie/asto/redis/package-info.java create mode 100644 asto/asto-s3/src/test/java/com/artipie/asto/s3/DiskCacheStorageTest.java create mode 100644 asto/asto-vertx-file/src/test/java/com/artipie/asto/VertxFileStorageHierarchicalListTest.java create mode 100644 benchmark/README.md create mode 100755 benchmark/bench.sh create mode 100644 benchmark/docker-compose-bench.yml create mode 100755 benchmark/report/generate-report.sh create mode 100755 benchmark/scenarios/common.sh create mode 100755 benchmark/scenarios/docker-bench.sh create mode 100755 benchmark/scenarios/maven-bench.sh create mode 100755 benchmark/scenarios/npm-bench.sh create mode 100644 benchmark/setup/artipie-new.yml create mode 100644 benchmark/setup/artipie-old.yml create mode 100644 benchmark/setup/init-db.sql create mode 100644 benchmark/setup/log4j2-bench.xml create mode 100644 benchmark/setup/repos-new/docker_local.yaml create mode 100644 benchmark/setup/repos-new/docker_proxy.yaml create mode 100644 benchmark/setup/repos-new/maven.yaml create mode 100644 benchmark/setup/repos-new/maven_group.yaml create mode 100644 benchmark/setup/repos-new/maven_proxy.yaml create mode 100644 benchmark/setup/repos-new/npm.yaml create mode 100644 benchmark/setup/repos-new/npm_group.yaml create mode 100644 benchmark/setup/repos-new/npm_proxy.yaml create mode 100644 benchmark/setup/repos-old/docker_local.yaml create mode 100644 benchmark/setup/repos-old/docker_proxy.yaml create mode 100644 benchmark/setup/repos-old/maven.yaml create mode 100644 benchmark/setup/repos-old/maven_group.yaml create mode 100644 benchmark/setup/repos-old/maven_proxy.yaml create mode 100644 benchmark/setup/repos-old/npm.yaml create mode 100644 benchmark/setup/repos-old/npm_group.yaml create mode 100644 benchmark/setup/repos-old/npm_proxy.yaml create mode 100644 benchmark/setup/repos/docker_local.yaml create mode 100644 benchmark/setup/repos/docker_proxy.yaml create mode 100644 benchmark/setup/repos/maven.yaml create mode 100644 benchmark/setup/repos/maven_group.yaml create mode 100644 benchmark/setup/repos/maven_proxy.yaml create mode 100644 benchmark/setup/repos/npm.yaml create mode 100644 benchmark/setup/repos/npm_group.yaml create mode 100644 benchmark/setup/repos/npm_proxy.yaml create mode 100644 benchmark/setup/security/roles/admin.yaml create mode 100644 benchmark/setup/security/users/artipie.yaml create mode 100644 benchmark/setup/settings-new.xml create mode 100644 benchmark/setup/settings-old.xml create mode 100644 docker-adapter/src/main/java/com/artipie/docker/cache/CachingBlob.java create mode 100644 docker-adapter/src/main/java/com/artipie/docker/http/ReferrersSlice.java create mode 100644 docker-adapter/src/test/java/com/artipie/docker/cache/CachingBlobTest.java create mode 100644 docker-adapter/src/test/java/com/artipie/docker/http/ReferrersSliceTest.java create mode 100644 docker-adapter/src/test/java/com/artipie/docker/http/manifest/GetManifestSliceMdcTest.java create mode 100644 docker-adapter/src/test/java/com/artipie/docker/http/manifest/HeadManifestSliceMdcTest.java create mode 100644 docs/CHANGELOG-AUTO1.md create mode 100644 docs/ENVIRONMENT_VARIABLES.md create mode 100644 docs/RELEASE-NOTES-v1.21.0.md create mode 100644 docs/ha-deployment/artipie-ha.yml create mode 100644 docs/ha-deployment/docker-compose-ha.yml create mode 100644 docs/ha-deployment/nginx-ha.conf delete mode 100644 gradle-adapter/README.md delete mode 100644 gradle-adapter/pom.xml delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/Gradle.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/GradleProxyPackageProcessor.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/asto/AstoGradle.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/asto/package-info.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/CachedProxySlice.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/GradleCooldownInspector.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/GradleMetadataReader.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/GradlePathBuilder.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/GradleProxySlice.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/GradleSlice.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/HeadProxySlice.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/PomParser.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/RepoHead.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/http/package-info.java delete mode 100644 gradle-adapter/src/main/java/com/artipie/gradle/package-info.java delete mode 100644 gradle-adapter/src/test/java/com/artipie/gradle/asto/AstoGradleTest.java delete mode 100644 gradle-adapter/src/test/java/com/artipie/gradle/http/GradleProxyIT.java delete mode 100644 gradle-adapter/src/test/java/com/artipie/gradle/http/GradleSliceTest.java create mode 100644 http-client/src/test/java/com/artipie/http/client/jetty/JettyClientSliceGzipTest.java create mode 100644 maven-adapter/src/main/java/com/artipie/maven/metadata/MavenTimestamp.java create mode 100644 maven-adapter/src/test/java/com/artipie/maven/metadata/MavenTimestampTest.java diff --git a/.gitignore b/.gitignore index 502f01ca1..e8b378082 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ classes/ .storage/data/** .storage/data/test/** artipie-main/docker-compose/artipie/data/** +artipie-main/docker-compose/artipie/prod_repo/** artipie-main/docker-compose/artipie/artifacts/npm/node_modules/** artipie-main/docker-compose/artipie/cache/** artipie-main/docker-compose/artipie/artifacts/php/vendor/** @@ -29,4 +30,16 @@ artipie-main/docker-compose/artipie/artifacts/php/vendor/** artipie-main/docker-compose/.env # AI agent task/analysis documents - not part of product documentation -agents/ \ No newline at end of file +agents/ + +# Git worktrees +.worktrees/ +/benchmark/fixtures +/benchmark/results +/docs/plans +/docs/superpowers +*.csv +*.png +/.superpowers +/artipie-ui/mockups +artipie-backfill/dependency-reduced-pom.xml diff --git a/.wiki/Configuration-HA.md b/.wiki/Configuration-HA.md new file mode 100644 index 000000000..510c3aa46 --- /dev/null +++ b/.wiki/Configuration-HA.md @@ -0,0 +1,156 @@ +# HA Deployment + +Artipie supports multi-instance deployment for high availability. Instances +coordinate through a shared PostgreSQL database and a shared Valkey (Redis-compatible) +instance so that metadata, events, and job scheduling remain consistent across the +cluster. + +## Requirements + +| Component | Minimum version | Purpose | +|-----------------------|-----------------|-----------------------------------------| +| PostgreSQL | 15+ | Artifacts metadata, Quartz job store | +| Valkey / Redis | 7+ | Cross-instance event bus (pub/sub) | +| S3-compatible storage | - | Shared artifact storage | +| Load balancer (nginx) | - | Request distribution and health checks | + +All instances must be able to reach the same PostgreSQL database, the same +Valkey instance, and the same S3-compatible storage bucket. + +## Architecture + +Each Artipie instance registers itself in the `artipie_nodes` table on startup and +sends periodic heartbeats so that the cluster knows which nodes are alive. + +`ClusterEventBus` uses Valkey pub/sub channels to broadcast notifications across +instances. When one instance receives an artifact upload, the event is published to +Valkey so that other instances can update caches or indexes accordingly. + +Quartz is configured with JDBC clustering (`org.quartz.jobStore.isClustered = true`) +to prevent duplicate execution of scheduled jobs such as metadata flush or proxy +cache verification. Only one instance in the cluster will execute a given job trigger +at any point in time. + +## Configuration + +An example HA configuration file is provided at `docs/ha-deployment/artipie-ha.yml`. +The key points are: + +- **Storage must use S3** (not the local filesystem) so that all instances share the + same artifact data. A filesystem backend would result in each instance having its own + isolated copy of the data. + +- **All instances must share the same PostgreSQL database.** The `artifacts_database` + section in the main configuration file must point to the same host, port, and + database name on every instance. + +- **All instances must share the same Valkey instance.** The `valkey` section in the + main configuration must use the same connection details everywhere. + +- **JWT secrets must be identical across instances.** If one instance issues a JWT + token, any other instance must be able to validate it. Configure the same + `jwt_secret` value on every node. + +```yaml +meta: + storage: + type: s3 + bucket: artipie-data + region: us-east-1 + endpoint: http://minio:9000 + credentials: + type: basic + accessKeyId: minioadmin + secretAccessKey: minioadmin + artifacts_database: + postgres_host: postgres + postgres_port: 5432 + postgres_database: artipie + postgres_user: artipie + postgres_password: artipie + threads_count: 4 + interval_seconds: 2 + valkey: + host: valkey + port: 6379 +``` + +## nginx setup + +An example nginx configuration is provided at `docs/ha-deployment/nginx-ha.conf`. +The recommended setup uses: + +- `least_conn` load-balancing algorithm to distribute requests to the instance with + the fewest active connections. +- `keepalive 64` on the upstream block to reuse connections to backend instances and + reduce latency. +- Passive health checks via the `/.health` endpoint. nginx marks a backend as down + after a configurable number of failed requests and re-checks periodically. + +```nginx +upstream artipie { + least_conn; + server artipie-1:8080; + server artipie-2:8080; + server artipie-3:8080; + keepalive 64; +} + +server { + listen 80; + + location / { + proxy_pass http://artipie; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + + location /.health { + proxy_pass http://artipie; + proxy_connect_timeout 5s; + proxy_read_timeout 5s; + } +} +``` + +## Docker Compose + +A ready-to-use Docker Compose file for a 3-instance deployment is provided at +`docs/ha-deployment/docker-compose-ha.yml`. It includes: + +- 3 Artipie instances behind an nginx load balancer +- PostgreSQL 15 for metadata and Quartz job store +- Valkey 7 for the cross-instance event bus +- MinIO for S3-compatible shared storage +- nginx configured with `least_conn` and passive health checks + +To start the cluster: + +```bash +cd docs/ha-deployment +docker compose -f docker-compose-ha.yml up -d +``` + +After startup, the Artipie API is available through the nginx load balancer on port 80 +and each instance exposes its own health endpoint at `/.health`. + +## Monitoring + +The [health endpoint](Configuration-Health) at `GET /.health` returns per-component +status for each instance. In an HA deployment, configure your load balancer or external +monitoring system to poll `/.health` on each instance independently. + +Recommended monitoring setup: + +- **nginx passive health checks**: rely on the upstream `max_fails` / `fail_timeout` + directives to automatically remove unhealthy instances from the pool. +- **External monitoring**: poll each instance's `/.health` endpoint every 10 seconds. + Alert when any instance returns `unhealthy` (HTTP 503) for more than 3 consecutive + checks. +- **Prometheus metrics**: each instance exposes its own `/metrics/vertx` endpoint (see + [Metrics](Configuration-Metrics)). Aggregate across instances in Prometheus or + Grafana for a cluster-wide view. diff --git a/.wiki/Configuration-Health.md b/.wiki/Configuration-Health.md new file mode 100644 index 000000000..bc0c1a90e --- /dev/null +++ b/.wiki/Configuration-Health.md @@ -0,0 +1,120 @@ +# Health Checks + +Artipie exposes a built-in health endpoint that reports the status of every major +subsystem. The endpoint is designed for load balancer integration and external +monitoring. + +## Endpoint + +``` +GET /.health +``` + +No authentication is required. The endpoint is always available, even when JWT +authentication is configured for the rest of the API. + +## Response format + +```json +{ + "status": "healthy", + "components": { + "storage": {"status": "up"}, + "database": {"status": "up"}, + "valkey": {"status": "not_configured"}, + "quartz": {"status": "up"}, + "http_client": {"status": "up"} + } +} +``` + +The top-level `status` field is a roll-up of the individual component statuses. +Each entry in `components` reports the state of a single subsystem. + +## Status values + +| Top-level status | Condition | HTTP code | +|------------------|---------------------------------------------------|-----------| +| `healthy` | All components report `up` (or `not_configured`) | 200 | +| `degraded` | Exactly one non-storage component is down | 200 | +| `unhealthy` | Storage is down, OR two or more components are down | 503 | + +The distinction between `degraded` and `unhealthy` allows load balancers to keep +routing traffic to an instance that has a single non-critical failure (for example, +Valkey is temporarily unreachable) while removing instances that cannot serve +artifacts at all. + +## Component details + +### storage + +Tests the primary artifact storage by calling `list(Key.ROOT)` with a 5-second +timeout. If the storage does not respond within the timeout, the component is +reported as `down`. Because storage is the most critical subsystem, a storage +failure alone is enough to mark the instance as `unhealthy`. + +### database + +Tests the PostgreSQL connection pool by calling `connection.isValid(5)` through +HikariCP. This component is only checked when `artifacts_database` is configured +in the main Artipie configuration file. If PostgreSQL is not configured, the +component is omitted from the response. + +### valkey + +Tests connectivity to the Valkey (Redis-compatible) instance used for the +cross-instance event bus. When Valkey is not configured (single-instance +deployments), the status is reported as `not_configured` rather than `down`. + +### quartz + +Checks the Quartz scheduler state: +`scheduler.isStarted() && !scheduler.isShutdown() && !scheduler.isInStandbyMode()`. +If the scheduler has been shut down or placed in standby mode, the component is +reported as `down`. + +### http_client + +Checks that the Jetty HTTP client used for proxy and remote operations is running +and operational. A `down` status typically indicates a resource exhaustion problem +(thread pool or connection pool). + +## HTTP status codes + +| Code | Meaning | +|------|-------------------------------------------------------------| +| 200 | Instance is `healthy` or `degraded` -- safe to route traffic | +| 503 | Instance is `unhealthy` -- remove from load balancer pool | + +## Load balancer integration + +Use the `/.health` endpoint as the health check target for nginx, HAProxy, AWS ALB, +or any other load balancer. + +### nginx example + +```nginx +upstream artipie { + least_conn; + server artipie-1:8080 max_fails=3 fail_timeout=30s; + server artipie-2:8080 max_fails=3 fail_timeout=30s; + keepalive 64; +} +``` + +With the configuration above, nginx uses passive health checks: after 3 consecutive +failed requests (including 503 responses from `/.health`), the backend is marked as +unavailable for 30 seconds before nginx retries it. + +### Recommended settings + +| Parameter | Recommended value | Description | +|-----------------|-------------------|---------------------------------------| +| Check interval | 10s | How often to poll `/.health` | +| Failure threshold | 3 | Consecutive failures before marking down | +| Success threshold | 1 | Consecutive successes before marking up | +| Timeout | 5s | Maximum time to wait for a response | + +For [HA deployments](Configuration-HA), configure health checks independently for +each Artipie instance so that a single unhealthy node is removed from the pool +without affecting the others. diff --git a/.wiki/Configuration-Metadata.md b/.wiki/Configuration-Metadata.md index ddb7d4d9b..a068c651e 100644 --- a/.wiki/Configuration-Metadata.md +++ b/.wiki/Configuration-Metadata.md @@ -34,9 +34,61 @@ The database has a single table `artifacts` with the following structure: All fields are NOT NULL; a UNIQUE constraint is created on `(repo_name, name, version)`. +### Full-text search (tsvector) + +The `artifacts` table includes a `search_tokens` column of type `tsvector`. This column is +auto-populated by a PostgreSQL trigger on every INSERT and UPDATE. The trigger concatenates +`repo_name`, `name`, and `version` into a single text-search vector so no application-side +indexing is required. + +Full-text queries use `ts_rank()` to order results by relevance. When the query string +contains wildcard characters (`*` or `?`), the search engine automatically falls back to +`LIKE`-based matching so that glob-style patterns still work. + +### Connection pool environment variables + +The following environment variables tune the HikariCP connection pool used for PostgreSQL. +They can be set as system environment variables or passed via `-D` JVM flags: + +| Variable | Default | Description | +|------------------------------------|-----------|--------------------------------------------------| +| `ARTIPIE_DB_CONNECTION_TIMEOUT_MS` | 5000 | Maximum time (ms) to wait for a connection | +| `ARTIPIE_DB_IDLE_TIMEOUT_MS` | 600000 | Maximum time (ms) a connection may sit idle | +| `ARTIPIE_DB_MAX_LIFETIME_MS` | 1800000 | Maximum lifetime (ms) of a connection in the pool | + +These defaults are suitable for most single-instance deployments. In HA setups with many +concurrent writers you may need to lower `ARTIPIE_DB_IDLE_TIMEOUT_MS` or raise the pool +size to avoid connection starvation. + Migration note: earlier versions supported SQLite via `sqlite_data_file_path`. This is deprecated in favor of PostgreSQL. Please migrate your data and update the configuration to use the `postgres_*` settings. +## Artifact Index (Lucene) + +Artipie supports a Lucene-based artifact index for fast O(1) group repository lookups. +To enable this, add the following section to the main configuration file (`meta` section): + +```yaml +meta: + artifact_index: + enabled: true # Enable Lucene artifact index (default: false) + directory: /var/artipie/index # Path for Lucene index files (required if enabled) + warmup_on_startup: true # Scan repos on startup to populate index (default: true) +``` + +| Field | Required | Default | Description | +|--------------------|----------|---------|----------------------------------------------------------| +| enabled | no | false | Enable or disable the Lucene artifact index | +| directory | yes* | - | Filesystem path for Lucene index files (*required if enabled) | +| warmup_on_startup | no | true | Scan all repository storage on startup to populate index | + +When the index is enabled: +- On startup, `IndexWarmupService` scans all repository storage to build the initial index (unless `warmup_on_startup` is `false`) +- During warmup, group repositories fall back to querying all members (fan-out) +- Once warmup completes, group lookups return immediately from the index +- Artifact uploads and deletes automatically update the index via the event pipeline +- The REST API exposes search and stats endpoints under `/api/v1/search/` + ## Maven, NPM and PyPI proxy adapters [Maven-proxy](maven-proxy), [npm-proxy](npm-proxy) and [python-proxy](pypi-proxy) have some extra mechanism to process diff --git a/.wiki/Configuration-Metrics.md b/.wiki/Configuration-Metrics.md index 1ea772f68..e414ae571 100644 --- a/.wiki/Configuration-Metrics.md +++ b/.wiki/Configuration-Metrics.md @@ -46,6 +46,45 @@ Artipie gather the following metrics: All the metrics for storage operations report `error` events in the case of any errors, the events have `_error` postfix. +### Thread pool metrics + +These gauges report the current utilisation of the internal thread pools that handle +storage operations: + +| Name | Type | Description | +|----------------------------|-------|--------------------------------| +| artipie.pool.read.active | gauge | Active threads in READ pool | +| artipie.pool.read.queue | gauge | Queue size of READ pool | +| artipie.pool.write.active | gauge | Active threads in WRITE pool | +| artipie.pool.write.queue | gauge | Queue size of WRITE pool | +| artipie.pool.list.active | gauge | Active threads in LIST pool | +| artipie.pool.list.queue | gauge | Queue size of LIST pool | + +### Event queue metrics + +| Name | Type | Description | Tags | +|----------------------------|-------|----------------------------------------------------|------| +| artipie.events.queue.size | gauge | Total pending events in the metadata queue | - | +| artipie.proxy.queue.size | gauge | Per-repository proxy event queue size | repo | + +### Request deduplication metrics + +The `RequestDeduplicator` tracks the number of in-flight deduplicated requests. +The current count is available programmatically via `RequestDeduplicator.inFlightCount()`. +When metrics are enabled, this value is also exported as a gauge so it can be scraped by +Prometheus alongside the other Artipie metrics. + +### Metrics environment variables + +| Variable | Default | Description | +|-----------------------------------------|---------|------------------------------------------------------| +| `ARTIPIE_METRICS_MAX_REPOS` | 50 | Maximum number of per-repo metric tag values tracked | +| `ARTIPIE_METRICS_PERCENTILES_HISTOGRAM` | false | Publish percentile histogram buckets for summaries | + +Setting `ARTIPIE_METRICS_MAX_REPOS` limits the cardinality of the `repo` tag to avoid +unbounded memory growth in deployments with many repositories. When the limit is reached, +additional repositories are aggregated under a single `_other` tag value. + Tags description: | Name | Description | diff --git a/.wiki/Home.md b/.wiki/Home.md index bc60df87a..ad3278cb1 100644 --- a/.wiki/Home.md +++ b/.wiki/Home.md @@ -107,9 +107,13 @@ To add a new repository or update an existing repository, you have to simply cre configuration `yaml` files in the directory `/var/artipie/repo` or use the [REST API](./Rest-api) to create/update repositories dynamically without restart. -## New and notable +## New and notable (1.20.14) - Dynamic repositories: create, update, move and delete repositories at runtime via REST with no restart. See [REST API](./Rest-api). - Bearer auth everywhere: all repositories support bearer token authentication for uploads and downloads in addition to Basic auth. - PostgreSQL artifacts DB: artifacts metadata are written to PostgreSQL (SQLite support deprecated). See [Artifacts metadata](./Configuration-Metadata). - ARM64 support: official Docker images and the service run on `linux/amd64` and `linux/arm64` architectures. +- [HA Deployment](Configuration-HA): multi-instance deployment with shared PostgreSQL, Valkey, and S3 storage for high availability. +- [Health Checks](Configuration-Health): built-in `/.health` endpoint reports per-component status for load balancer integration. +- [Search API](./Rest-api#search-api): full-text artifact search with cursor pagination and relevance ranking. +- [Cooldown management](./Rest-api#cooldown-management): unblock artefacts or clear all cooldown entries per repository via REST API. diff --git a/.wiki/Rest-api.md b/.wiki/Rest-api.md index bdf031a9d..1926820aa 100644 --- a/.wiki/Rest-api.md +++ b/.wiki/Rest-api.md @@ -156,6 +156,88 @@ Both operations require update permissions on the repository. The first endpoint artefact version and any dependencies that were blocked alongside it. The second endpoint clears all pending cooldown entries for the repository. Successful calls return `204 No Content`. +## Search API + +When the artifact index is enabled (see [Configuration-Metadata](./Configuration-Metadata)), the following +search endpoints are available: + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `GET` | `/api/v1/search?query=...&cursor=...&limit=...` | Full-text artifact search with cursor pagination | +| `GET` | `/api/v1/search/locate?repo=...&name=...` | Locate a specific artifact by repository and name | +| `GET` | `/api/v1/search/reindex` | Trigger metadata reindex | +| `GET` | `/api/v1/search/stats` | Search index statistics (document count, warmup status) | + +All search endpoints require JWT authentication. The `/search/reindex` endpoint additionally requires +admin-level permissions. + +### Full-text search + +The `query` parameter supports free-text search. Results are ranked by relevance using +PostgreSQL `ts_rank()` when the query matches the tsvector index. Wildcard characters +(`*`, `?`) automatically fall back to `LIKE`-based matching. + +Use `cursor` and `limit` for pagination. The response includes a `next_cursor` value +that should be passed as `cursor` in the subsequent request to fetch the next page. + +**Example: search for an artifact** +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8086/api/v1/search?query=my-library&limit=10" +``` + +**Example: paginate through results** +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8086/api/v1/search?query=my-library&limit=10&cursor=eyJvZmZzZXQiOjEwfQ" +``` + +### Locate artifact + +The `/search/locate` endpoint finds a specific artifact by repository name and artifact +name. This is useful for determining whether a particular version exists in a given +repository. + +**Example: locate an artifact** +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8086/api/v1/search/locate?repo=my-maven&name=com.example:my-library" +``` + +### Reindex and stats + +**Example: trigger reindex** +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8086/api/v1/search/reindex" +``` + +**Example: check index stats** +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8086/api/v1/search/stats" +``` + +## Health API + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `GET` | `/.health` | System health check | + +The health endpoint does not require authentication. It returns per-component status +and an overall system status. See [Health Checks](./Configuration-Health) for full +details on response format, status values, and load balancer integration. + +## Import API + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v1/import` | Global import endpoint for bulk artifact migration | + +The import endpoint accepts bulk artifact data for migration into Artipie repositories. +This is useful when migrating artifacts from another repository manager. The endpoint +requires JWT authentication with admin-level permissions. + ## Storage aliases [Storage aliases](./Configuration-Storage#Storage-Aliases) can also be managed with Rest API, there are methods to read, create, update and remove aliases. Note, that concrete storage settings diff --git a/.wiki/_Sidebar.md b/.wiki/_Sidebar.md index 413f5ba90..9db368332 100644 --- a/.wiki/_Sidebar.md +++ b/.wiki/_Sidebar.md @@ -45,3 +45,5 @@ * [[Scripting support|Configuration-Scripting]] * [[Artifacts metadata|Configuration-Metadata]] * [[HTTP3 support|Configuration-http3]] + * [[HA Deployment|Configuration-HA]] + * [[Health Checks|Configuration-Health]] diff --git a/README.md b/README.md index 0811a1adc..4af6a26c9 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,15 @@ > **Auto1 Fork**: This is a production-hardened fork of the original [Artipie](https://github.com/artipie/artipie) project, significantly enhanced for enterprise-scale deployments. It includes performance optimizations, security features, and operational improvements developed for high-traffic production workloads. +## What's New in v1.20.14 + +- **HA Clustering** — PostgreSQL-backed node registry with heartbeat liveness, cross-instance events via Valkey pub/sub +- **Full-Text Search** — tsvector/GIN powered artifact search with relevance ranking +- **Health Checks** — 5-component health endpoint with degraded/unhealthy severity levels +- **Worker Pool Separation** — Independent thread pools for read, write, and list operations +- **Observability** — Pool utilization gauges, event queue depth metrics, structured ECS logging +- **Hardening** — 44 silent catches fixed, zero-copy request path, temp file bulk cleanup + ## What is Artipie? Artipie is a **binary artifact management platform** similar to [JFrog Artifactory](https://jfrog.com/artifactory/), [Sonatype Nexus](https://www.sonatype.com/product-nexus-repository), and [Apache Archiva](https://archiva.apache.org/). It provides a unified solution for hosting, proxying, and managing software packages across multiple ecosystems. @@ -21,11 +30,21 @@ Artipie is a **binary artifact management platform** similar to [JFrog Artifacto | **Enterprise Auth** | OAuth/OIDC integration (Keycloak, Okta with MFA), JWT, RBAC | | **Cloud-Native Storage** | Optimized S3-compatible storage with memory-efficient streaming | | **Observability** | Prometheus metrics, ECS JSON structured logging, Elastic APM | +| **HA Clustering** | PostgreSQL-backed node coordination with heartbeat liveness detection | +| **Full-Text Search** | Artifact search powered by PostgreSQL tsvector/GIN indexes | +| **Health Checks** | 5-component health probes: storage, database, Valkey, Quartz, HTTP client | +| **Worker Pool Separation** | Named I/O thread pools for read, write, and list operations | +| **Cache Invalidation** | Cross-instance cache invalidation via Valkey pub/sub | | **Dynamic Configuration** | Create, update, delete repositories at runtime via REST API | | **Production-Ready** | Docker Compose stack with PostgreSQL, Valkey (Redis), Nginx, monitoring | ### Fork-Specific Enhancements +- **HA Clustering**: PostgreSQL-backed node registry with heartbeat liveness detection and cross-instance event bus +- **Full-Text Artifact Search**: PostgreSQL tsvector/GIN index with relevance-ranked results +- **5-Component Health Checks**: Storage, database, Valkey, Quartz scheduler, and HTTP client probes +- **Named Worker Pools**: Separate thread pools for read (CPU x4), write (CPU x2), and list (CPU) operations +- **Cross-Instance Cache Invalidation**: Valkey pub/sub for distributed cache coherence - **Cooldown System**: Configurable delay on new package versions (supply chain attack prevention) - **Okta OIDC Integration**: Full Okta authentication with MFA support (TOTP + push) - **S3 Performance**: Memory-optimized streaming, retry improvements, connection pooling @@ -109,6 +128,8 @@ cp .env.example .env docker-compose up -d ``` +> **Multi-Instance HA Deployment**: For high-availability setups with multiple Artipie nodes behind a load balancer, see [`docs/ha-deployment/docker-compose-ha.yml`](docs/ha-deployment/docker-compose-ha.yml). The HA configuration includes PostgreSQL-backed node registry, Valkey pub/sub for cross-instance events, and nginx load balancing. + ### Included Services | Service | Port | Description | @@ -276,6 +297,9 @@ See [Cooldown System Documentation](docs/cooldown-fallback/README.md) for comple |----------|-------------| | [User Guide](docs/USER_GUIDE.md) | Installation, configuration, and usage | | [Developer Guide](docs/DEVELOPER_GUIDE.md) | Architecture and contributing | +| [Auto1 Fork Changelog](docs/CHANGELOG-AUTO1.md) | Release history for the Auto1 fork | +| [Environment Variables](docs/ENVIRONMENT_VARIABLES.md) | Complete environment variable reference | +| [HA Deployment](docs/ha-deployment/) | High-availability deployment configs (nginx, Docker Compose, Artipie) | | [API Routing](docs/API_ROUTING.md) | URL patterns and routing | | [Cooldown System](docs/cooldown-fallback/README.md) | Supply chain attack prevention | | [S3 Storage](docs/s3-optimizations/README.md) | S3 configuration and tuning | @@ -446,7 +470,7 @@ ECS JSON structured logging for Elasticsearch/Kibana: | Component | Version | |-----------|---------| -| Artipie | 1.20.12 | +| Artipie | 1.20.14 | | Java | 21+ | | Vert.x | 4.5.x | diff --git a/artipie-backfill/README.md b/artipie-backfill/README.md new file mode 100644 index 000000000..f99169986 --- /dev/null +++ b/artipie-backfill/README.md @@ -0,0 +1,239 @@ +# artipie-backfill + +Standalone CLI tool for backfilling the PostgreSQL `artifacts` table from disk storage. Scans artifact repositories on disk (in various package manager layouts) and populates a PostgreSQL database with artifact metadata. + +## Use Cases + +- **Initial indexing** — populate the database when enabling database-backed artifact indexing on existing repositories +- **Re-indexing** — rebuild artifact metadata after storage migrations or data recovery +- **Auditing** — dry-run mode to count and inspect artifacts without writing to the database + +## Supported Repository Types + +| Type | CLI Value | Description | +|------|-----------|-------------| +| Maven | `maven` | Standard Maven layout (`groupId/artifactId/version/`) | +| Gradle | `gradle` | Same as Maven with a different type identifier | +| Docker | `docker` | Docker v2 registry layout (`repositories/{image}/_manifests/tags/`) | +| NPM | `npm` | Artipie `.versions/` layout or legacy `meta.json` proxy layout | +| PyPI | `pypi` | Flat directory with `.whl`, `.tar.gz`, `.zip`, `.egg` files | +| Go | `go` | Hosted (`@v/list`) or proxy (`@v/*.info`) layouts | +| Helm | `helm` | `index.yaml` with chart tarballs | +| Composer/PHP | `composer`, `php` | `p2/{vendor}/*.json` or root `packages.json` layout | +| Debian | `deb`, `debian` | `dists/{codename}/{component}/binary-{arch}/Packages[.gz]` | +| Ruby Gems | `gem`, `gems` | `gems/` subdirectory or flat layout | +| Generic Files | `file` | Recursive file walk, uses relative path as artifact name | + +## Building + +```bash +mvn clean package -pl artipie-backfill -am +``` + +This produces a fat JAR via the Maven Shade plugin at: + +``` +artipie-backfill/target/artipie-backfill-.jar +``` + +## Usage + +``` +backfill-cli -t -p -r [options] +``` + +### Required Arguments + +| Flag | Long | Description | +|------|------|-------------| +| `-t` | `--type` | Scanner type (`maven`, `docker`, `npm`, `pypi`, `go`, `helm`, `composer`, `file`, `deb`, `gem`) | +| `-p` | `--path` | Root directory path of the repository to scan | +| `-r` | `--repo-name` | Logical repository name (stored in the `repo_name` column) | + +### Optional Arguments + +| Flag | Long | Default | Description | +|------|------|---------|-------------| +| | `--db-url` | *(required unless `--dry-run`)* | JDBC PostgreSQL URL | +| | `--db-user` | `artipie` | Database user | +| | `--db-password` | `artipie` | Database password | +| `-b` | `--batch-size` | `1000` | Number of records per batch insert | +| | `--owner` | `system` | Default owner written to the `owner` column | +| | `--log-interval` | `10000` | Log progress every N records | +| | `--dry-run` | `false` | Scan and count artifacts without writing to the database | +| `-h` | `--help` | | Print help and exit | + +### Examples + +**Dry run** — scan a Maven repository and report counts without database writes: + +```bash +java -jar artipie-backfill.jar \ + --type maven \ + --path /var/artipie/data/my-maven-repo \ + --repo-name internal-maven \ + --dry-run +``` + +**Full backfill** — scan and insert into PostgreSQL: + +```bash +java -jar artipie-backfill.jar \ + --type maven \ + --path /var/artipie/data/my-maven-repo \ + --repo-name internal-maven \ + --db-url jdbc:postgresql://db.example.com:5432/artipie \ + --db-user artipie \ + --db-password secret123 \ + --batch-size 500 +``` + +**Docker repository backfill:** + +```bash +java -jar artipie-backfill.jar \ + --type docker \ + --path /var/artipie/data/my-docker-repo \ + --repo-name docker-registry \ + --db-url jdbc:postgresql://localhost:5432/artipie +``` + +**NPM repository backfill with custom owner:** + +```bash +java -jar artipie-backfill.jar \ + --type npm \ + --path /var/artipie/data/npm-repo \ + --repo-name npm-internal \ + --db-url jdbc:postgresql://localhost:5432/artipie \ + --owner admin +``` + +## Database Schema + +The tool auto-creates the `artifacts` table and indexes if they do not exist: + +```sql +CREATE TABLE IF NOT EXISTS artifacts ( + id BIGSERIAL PRIMARY KEY, + repo_type VARCHAR NOT NULL, + repo_name VARCHAR NOT NULL, + name VARCHAR NOT NULL, + version VARCHAR NOT NULL, + size BIGINT NOT NULL, + created_date BIGINT NOT NULL, + release_date BIGINT, + owner VARCHAR NOT NULL, + UNIQUE (repo_name, name, version) +); +``` + +**Indexes:** + +| Index | Columns | +|-------|---------| +| `idx_artifacts_repo_lookup` | `repo_name, name, version` | +| `idx_artifacts_repo_type_name` | `repo_type, repo_name, name` | +| `idx_artifacts_created_date` | `created_date` | +| `idx_artifacts_owner` | `owner` | + +The insert uses `ON CONFLICT ... DO UPDATE` (upsert), so the tool is **idempotent** — running it multiple times against the same repository safely updates existing records. + +## Architecture + +``` +BackfillCli (entry point) +├── ScannerFactory → creates Scanner by --type +│ ├── MavenScanner (maven, gradle) +│ ├── DockerScanner (docker) +│ ├── NpmScanner (npm) +│ ├── PypiScanner (pypi) +│ ├── GoScanner (go) +│ ├── HelmScanner (helm) +│ ├── ComposerScanner (composer, php) +│ ├── DebianScanner (deb, debian) +│ ├── GemScanner (gem, gems) +│ └── FileScanner (file) +├── BatchInserter → buffered JDBC batch writer +└── ProgressReporter → throughput logging +``` + +### Key Components + +| Class | Responsibility | +|-------|---------------| +| `BackfillCli` | CLI argument parsing, wiring, and execution lifecycle | +| `Scanner` | Functional interface — `Stream scan(Path root, String repoName)` | +| `ScannerFactory` | Maps type strings (case-insensitive) to `Scanner` implementations | +| `ArtifactRecord` | Java record representing a row in the `artifacts` table | +| `BatchInserter` | Buffers records and flushes in batches via JDBC; falls back to individual inserts on batch failure | +| `ProgressReporter` | Thread-safe counter with periodic throughput logging | + +### Data Flow + +``` +Disk Storage → Scanner (lazy stream) → BatchInserter (buffered) → PostgreSQL +``` + +All scanners produce **lazy streams** (`java.util.stream.Stream`) to enable constant-memory processing of arbitrarily large repositories. + +## Error Handling + +- **Batch insert failure** — automatically falls back to individual record inserts so one bad record does not block the entire batch +- **Malformed metadata** — logged as a warning and skipped; processing continues +- **Missing files** — defaults to file system `mtime` when metadata timestamps are unavailable +- **Connection failure** — records in the failed batch are counted as skipped + +### Exit Codes + +| Code | Meaning | +|------|---------| +| `0` | Success | +| `1` | Validation error, invalid arguments, or processing failure | + +## Connection Pool + +Uses HikariCP with the following defaults: + +| Setting | Value | +|---------|-------| +| Max pool size | 5 | +| Min idle | 1 | +| Connection timeout | 5000 ms | +| Idle timeout | 30000 ms | + +## Testing + +### Unit Tests + +```bash +mvn test -pl artipie-backfill +``` + +Covers CLI parsing, batch inserter buffering/flushing, progress reporting, and scanner factory type mapping. + +### Integration Tests + +Full pipeline integration tests (dry-run mode) run with `mvn test`. PostgreSQL integration tests require the `BACKFILL_IT_DB_URL` environment variable: + +```bash +BACKFILL_IT_DB_URL=jdbc:postgresql://localhost:5432/artipie \ +BACKFILL_IT_DB_USER=artipie \ +BACKFILL_IT_DB_PASSWORD=artipie \ + mvn test -pl artipie-backfill +``` + +## Dependencies + +| Dependency | Version | Purpose | +|------------|---------|---------| +| Apache Commons CLI | 1.5.0 | CLI argument parsing | +| PostgreSQL JDBC | 42.7.1 | Database driver | +| HikariCP | 5.1.0 | Connection pooling | +| javax.json | — | JSON parsing (NPM, Composer, Go) | +| SnakeYAML | 2.0 | YAML parsing (Helm) | +| SLF4J + Log4j2 | 2.0.17 / 2.24.3 | Logging | + +## License + +[MIT](../LICENSE.txt) diff --git a/artipie-backfill/pom.xml b/artipie-backfill/pom.xml new file mode 100644 index 000000000..195374370 --- /dev/null +++ b/artipie-backfill/pom.xml @@ -0,0 +1,143 @@ + + + + 4.0.0 + + com.artipie + artipie + 1.22.0 + + artipie-backfill + 1.22.0 + jar + artipie-backfill + Standalone CLI for backfilling the PostgreSQL artifacts table from disk storage + 2020 + + ${project.basedir}/../LICENSE.header + + + + + commons-cli + commons-cli + 1.5.0 + + + org.postgresql + postgresql + 42.7.1 + + + com.zaxxer + HikariCP + 5.1.0 + + + javax.json + javax.json-api + ${javax.json.version} + + + org.glassfish + javax.json + ${javax.json.version} + + + org.yaml + snakeyaml + 2.0 + + + org.slf4j + slf4j-api + 2.0.17 + + + org.apache.logging.log4j + log4j-api + 2.24.3 + + + org.apache.logging.log4j + log4j-core + 2.24.3 + compile + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.3 + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + ${junit-platform.version} + test + + + org.hamcrest + hamcrest + 2.2 + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.1 + + + package + + shade + + + + + com.artipie.backfill.BackfillCli + + + + + + + + + + diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/ArtifactRecord.java b/artipie-backfill/src/main/java/com/artipie/backfill/ArtifactRecord.java new file mode 100644 index 000000000..0c1793d46 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/ArtifactRecord.java @@ -0,0 +1,33 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +/** + * Represents a single artifact record to be inserted into the PostgreSQL + * {@code artifacts} table. + * + * @param repoType Repository type identifier ("maven", "docker", "npm", etc.) + * @param repoName Repository name from the CLI {@code --repo-name} argument + * @param name Artifact coordinate (e.g. "com.example:mylib") + * @param version Version string + * @param size Artifact size in bytes + * @param createdDate Creation timestamp as epoch millis (file mtime) + * @param releaseDate Release timestamp as epoch millis, may be {@code null} + * @param owner Owner identifier, defaults to "system" + * @param pathPrefix Path prefix for group-repo lookup, may be {@code null} + * @since 1.20.13 + */ +public record ArtifactRecord( + String repoType, + String repoName, + String name, + String version, + long size, + long createdDate, + Long releaseDate, + String owner, + String pathPrefix +) { +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/ArtipieMetaSidecar.java b/artipie-backfill/src/main/java/com/artipie/backfill/ArtipieMetaSidecar.java new file mode 100644 index 000000000..c59540eba --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/ArtipieMetaSidecar.java @@ -0,0 +1,137 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonException; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonString; +import javax.json.JsonValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Reads the {@code .artipie-meta.json} sidecar file stored by + * {@code CachedArtifactMetadataStore} alongside cached proxy artifacts + * (Maven, PyPI). + * + *

Sidecar format:

+ * 
+ * {
+ *   "size": 12345,
+ *   "headers": [
+ *     {"name": "Last-Modified", "value": "Mon, 05 Jul 2021 10:08:46 GMT"},
+ *     ...
+ *   ],
+ *   "digests": {...}
+ * }
+ *
+ * + *

The {@code Last-Modified} value is the HTTP response header from the + * upstream registry, which is the artifact publish date — the source of + * {@code release_date} in production (via + * {@code MavenProxyPackageProcessor.releaseMillis()}).

+ * + * @since 1.20.13 + */ +final class ArtipieMetaSidecar { + + /** + * Sidecar file suffix appended to the artifact path. + */ + static final String SUFFIX = ".artipie-meta.json"; + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(ArtipieMetaSidecar.class); + + /** + * Private ctor — utility class, not instantiated. + */ + private ArtipieMetaSidecar() { + } + + /** + * Read the release date (epoch millis) from the + * {@code .artipie-meta.json} sidecar alongside the given artifact. + * + *

Returns empty if the sidecar is absent, the {@code headers} + * array is missing, no {@code Last-Modified} entry is present, or + * the date value cannot be parsed as RFC 1123.

+ * + * @param artifactPath Path to the artifact file + * @return Optional epoch millis, empty if sidecar is absent or + * unparseable + */ + static Optional readReleaseDate(final Path artifactPath) { + final Path sidecar = artifactPath.getParent() + .resolve(artifactPath.getFileName().toString() + SUFFIX); + if (!Files.isRegularFile(sidecar)) { + return Optional.empty(); + } + try (InputStream input = Files.newInputStream(sidecar); + JsonReader reader = Json.createReader(input)) { + final JsonObject json = reader.readObject(); + if (!json.containsKey("headers") + || json.isNull("headers") + || json.get("headers").getValueType() + != JsonValue.ValueType.ARRAY) { + return Optional.empty(); + } + final JsonArray headers = json.getJsonArray("headers"); + for (int idx = 0; idx < headers.size(); idx++) { + final JsonValue entry = headers.get(idx); + if (entry.getValueType() != JsonValue.ValueType.OBJECT) { + continue; + } + final JsonObject obj = (JsonObject) entry; + final JsonValue nameVal = obj.get("name"); + final JsonValue valueVal = obj.get("value"); + if (nameVal == null + || nameVal.getValueType() != JsonValue.ValueType.STRING + || valueVal == null + || valueVal.getValueType() + != JsonValue.ValueType.STRING) { + continue; + } + if ("Last-Modified".equalsIgnoreCase( + ((JsonString) nameVal).getString())) { + final String lm = ((JsonString) valueVal).getString(); + try { + return Optional.of( + Instant.from( + DateTimeFormatter.RFC_1123_DATE_TIME.parse(lm) + ).toEpochMilli() + ); + } catch (final DateTimeParseException ex) { + LOG.debug( + "Cannot parse Last-Modified '{}' in {}: {}", + lm, sidecar, ex.getMessage() + ); + return Optional.empty(); + } + } + } + } catch (final IOException | JsonException ex) { + LOG.debug( + "Cannot read artipie-meta sidecar {}: {}", + sidecar, ex.getMessage() + ); + } + return Optional.empty(); + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/BackfillCli.java b/artipie-backfill/src/main/java/com/artipie/backfill/BackfillCli.java new file mode 100644 index 000000000..2b4df40a3 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/BackfillCli.java @@ -0,0 +1,478 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import javax.sql.DataSource; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * CLI entry point for the artifact backfill tool. + * + *

Supports two modes:

+ *
    + *
  • Single-repo: {@code --type}, {@code --path}, {@code --repo-name} + * (original behaviour)
    • + *
  • Bulk: {@code --config-dir}, {@code --storage-root} — reads all + * {@code *.yaml} Artipie repo configs and scans each repo automatically
    • + *
+ * + * @since 1.20.13 + */ +public final class BackfillCli { + + /** + * SLF4J logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(BackfillCli.class); + + /** + * Default batch size for inserts. + */ + private static final int DEFAULT_BATCH_SIZE = 1000; + + /** + * Default progress log interval. + */ + private static final int DEFAULT_LOG_INTERVAL = 10000; + + /** + * Default database user. + */ + private static final String DEFAULT_DB_USER = "artipie"; + + /** + * Default database password. + */ + private static final String DEFAULT_DB_PASSWORD = "artipie"; + + /** + * Default owner. + */ + private static final String DEFAULT_OWNER = "system"; + + /** + * HikariCP maximum pool size. + */ + private static final int POOL_MAX_SIZE = 5; + + /** + * HikariCP minimum idle connections. + */ + private static final int POOL_MIN_IDLE = 1; + + /** + * HikariCP connection timeout in millis. + */ + private static final long POOL_CONN_TIMEOUT = 5000L; + + /** + * HikariCP idle timeout in millis. + */ + private static final long POOL_IDLE_TIMEOUT = 30000L; + + /** + * Private ctor to prevent instantiation. + */ + private BackfillCli() { + } + + /** + * CLI entry point. + * + * @param args Command-line arguments + */ + public static void main(final String... args) { + System.exit(run(args)); + } + + /** + * Core logic extracted for testability. Returns an exit code + * (0 = success, 1 = error). + * + * @param args Command-line arguments + * @return Exit code + */ + @SuppressWarnings("PMD.CyclomaticComplexity") + static int run(final String... args) { + final Options options = buildOptions(); + for (final String arg : args) { + if ("--help".equals(arg) || "-h".equals(arg)) { + printHelp(options); + return 0; + } + } + final CommandLine cmd; + try { + cmd = new DefaultParser().parse(options, args); + } catch (final ParseException ex) { + LOG.error("Failed to parse arguments: {}", ex.getMessage()); + printHelp(options); + return 1; + } + final boolean hasBulkFlags = + cmd.hasOption("config-dir") || cmd.hasOption("storage-root"); + final boolean hasSingleFlags = + cmd.hasOption("type") || cmd.hasOption("path") + || cmd.hasOption("repo-name"); + if (hasBulkFlags && hasSingleFlags) { + LOG.error( + "--config-dir/--storage-root cannot be combined with " + + "--type/--path/--repo-name" + ); + return 1; + } + if (cmd.hasOption("config-dir") && !cmd.hasOption("storage-root")) { + LOG.error("--config-dir requires --storage-root"); + return 1; + } + if (cmd.hasOption("storage-root") && !cmd.hasOption("config-dir")) { + LOG.error("--storage-root requires --config-dir"); + return 1; + } + if (!hasBulkFlags && !hasSingleFlags) { + LOG.error( + "Either --type/--path/--repo-name or " + + "--config-dir/--storage-root must be provided" + ); + printHelp(options); + return 1; + } + final boolean dryRun = cmd.hasOption("dry-run"); + final String dbUrl = cmd.getOptionValue("db-url"); + final String dbUser = cmd.getOptionValue("db-user", DEFAULT_DB_USER); + final String dbPassword = + cmd.getOptionValue("db-password", DEFAULT_DB_PASSWORD); + final int batchSize = Integer.parseInt( + cmd.getOptionValue( + "batch-size", String.valueOf(DEFAULT_BATCH_SIZE) + ) + ); + final String owner = cmd.getOptionValue("owner", DEFAULT_OWNER); + final int logInterval = Integer.parseInt( + cmd.getOptionValue( + "log-interval", String.valueOf(DEFAULT_LOG_INTERVAL) + ) + ); + if (cmd.hasOption("config-dir")) { + return runBulk( + cmd.getOptionValue("config-dir"), + cmd.getOptionValue("storage-root"), + dryRun, dbUrl, dbUser, dbPassword, + batchSize, owner, logInterval + ); + } + return runSingle( + cmd.getOptionValue("type"), + cmd.getOptionValue("path"), + cmd.getOptionValue("repo-name"), + dryRun, dbUrl, dbUser, dbPassword, + batchSize, owner, logInterval + ); + } + + /** + * Run bulk mode: scan all repos from the config directory. + * + * @param configDirStr Config directory path string + * @param storageRootStr Storage root path string + * @param dryRun Dry run flag + * @param dbUrl JDBC URL (may be null if dryRun) + * @param dbUser DB user + * @param dbPassword DB password + * @param batchSize Batch insert size + * @param owner Artifact owner + * @param logInterval Progress log interval + * @return Exit code + * @checkstyle ParameterNumberCheck (15 lines) + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + private static int runBulk( + final String configDirStr, + final String storageRootStr, + final boolean dryRun, + final String dbUrl, + final String dbUser, + final String dbPassword, + final int batchSize, + final String owner, + final int logInterval + ) { + final Path configDir = Paths.get(configDirStr); + final Path storageRoot = Paths.get(storageRootStr); + if (!Files.isDirectory(configDir)) { + LOG.error("--config-dir is not a directory: {}", configDirStr); + return 1; + } + if (!Files.isDirectory(storageRoot)) { + LOG.error("--storage-root is not a directory: {}", storageRootStr); + return 1; + } + if (!dryRun && (dbUrl == null || dbUrl.isEmpty())) { + LOG.error("--db-url is required unless --dry-run is set"); + return 1; + } + DataSource dataSource = null; + if (!dryRun) { + dataSource = buildDataSource(dbUrl, dbUser, dbPassword); + } + try { + return new BulkBackfillRunner( + configDir, storageRoot, dataSource, + owner, batchSize, dryRun, logInterval, System.err + ).run(); + } catch (final IOException ex) { + LOG.error("Bulk backfill failed: {}", ex.getMessage(), ex); + return 1; + } finally { + closeDataSource(dataSource); + } + } + + /** + * Run single-repo mode (original behaviour). + * + * @param type Scanner type + * @param pathStr Path string + * @param repoName Repo name + * @param dryRun Dry run flag + * @param dbUrl JDBC URL + * @param dbUser DB user + * @param dbPassword DB password + * @param batchSize Batch size + * @param owner Artifact owner + * @param logInterval Progress interval + * @return Exit code + * @checkstyle ParameterNumberCheck (15 lines) + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + private static int runSingle( + final String type, + final String pathStr, + final String repoName, + final boolean dryRun, + final String dbUrl, + final String dbUser, + final String dbPassword, + final int batchSize, + final String owner, + final int logInterval + ) { + if (type == null || pathStr == null || repoName == null) { + LOG.error( + "--type, --path, and --repo-name are all required in single-repo mode" + ); + return 1; + } + final Path root = Paths.get(pathStr); + if (!Files.exists(root) || !Files.isDirectory(root)) { + LOG.error( + "Path does not exist or is not a directory: {}", pathStr + ); + return 1; + } + if (!dryRun && (dbUrl == null || dbUrl.isEmpty())) { + LOG.error("--db-url is required unless --dry-run is set"); + return 1; + } + final Scanner scanner; + try { + scanner = ScannerFactory.create(type); + } catch (final IllegalArgumentException ex) { + LOG.error( + "Invalid scanner type '{}': {}", type, ex.getMessage() + ); + return 1; + } + LOG.info( + "Backfill starting: type={}, path={}, repo-name={}, " + + "batch-size={}, dry-run={}", + type, root, repoName, batchSize, dryRun + ); + DataSource dataSource = null; + if (!dryRun) { + dataSource = buildDataSource(dbUrl, dbUser, dbPassword); + } + final ProgressReporter progress = + new ProgressReporter(logInterval); + try (BatchInserter inserter = + new BatchInserter(dataSource, batchSize, dryRun)) { + try (Stream stream = + scanner.scan(root, repoName)) { + stream + .map(rec -> new ArtifactRecord( + rec.repoType(), rec.repoName(), rec.name(), + rec.version(), rec.size(), rec.createdDate(), + rec.releaseDate(), owner, rec.pathPrefix() + )) + .forEach(record -> { + inserter.accept(record); + progress.increment(); + }); + } + } catch (final Exception ex) { + LOG.error("Backfill failed: {}", ex.getMessage(), ex); + return 1; + } finally { + closeDataSource(dataSource); + } + progress.printFinalSummary(); + LOG.info("Backfill completed successfully"); + return 0; + } + + /** + * Build a HikariCP datasource. + * + * @param dbUrl JDBC URL + * @param dbUser DB user + * @param dbPassword DB password + * @return DataSource + */ + private static DataSource buildDataSource( + final String dbUrl, + final String dbUser, + final String dbPassword + ) { + final HikariConfig config = new HikariConfig(); + config.setJdbcUrl(dbUrl); + config.setUsername(dbUser); + config.setPassword(dbPassword); + config.setMaximumPoolSize(POOL_MAX_SIZE); + config.setMinimumIdle(POOL_MIN_IDLE); + config.setConnectionTimeout(POOL_CONN_TIMEOUT); + config.setIdleTimeout(POOL_IDLE_TIMEOUT); + config.setPoolName("Backfill-Pool"); + return new HikariDataSource(config); + } + + /** + * Close a HikariDataSource if non-null. + * + * @param dataSource DataSource to close (may be null) + */ + private static void closeDataSource(final DataSource dataSource) { + if (dataSource instanceof HikariDataSource) { + ((HikariDataSource) dataSource).close(); + } + } + + /** + * Build the CLI option definitions. + * + * @return Options instance + */ + private static Options buildOptions() { + final Options options = new Options(); + options.addOption( + Option.builder("t").longOpt("type") + .hasArg().argName("TYPE") + .desc("Scanner type — single-repo mode (maven, docker, npm, " + + "pypi, go, helm, composer, file, etc.)") + .build() + ); + options.addOption( + Option.builder("p").longOpt("path") + .hasArg().argName("PATH") + .desc("Root directory path to scan — single-repo mode") + .build() + ); + options.addOption( + Option.builder("r").longOpt("repo-name") + .hasArg().argName("NAME") + .desc("Repository name — single-repo mode") + .build() + ); + options.addOption( + Option.builder("C").longOpt("config-dir") + .hasArg().argName("DIR") + .desc("Directory of Artipie *.yaml repo configs — bulk mode") + .build() + ); + options.addOption( + Option.builder("R").longOpt("storage-root") + .hasArg().argName("DIR") + .desc("Storage root; each repo lives at // " + + "— bulk mode") + .build() + ); + options.addOption( + Option.builder().longOpt("db-url") + .hasArg().argName("URL") + .desc("JDBC PostgreSQL URL (required unless --dry-run)") + .build() + ); + options.addOption( + Option.builder().longOpt("db-user") + .hasArg().argName("USER") + .desc("Database user (default: artipie)") + .build() + ); + options.addOption( + Option.builder().longOpt("db-password") + .hasArg().argName("PASS") + .desc("Database password (default: artipie)") + .build() + ); + options.addOption( + Option.builder("b").longOpt("batch-size") + .hasArg().argName("SIZE") + .desc("Batch insert size (default: 1000)") + .build() + ); + options.addOption( + Option.builder().longOpt("owner") + .hasArg().argName("OWNER") + .desc("Default owner (default: system)") + .build() + ); + options.addOption( + Option.builder().longOpt("log-interval") + .hasArg().argName("N") + .desc("Progress log interval (default: 10000)") + .build() + ); + options.addOption( + Option.builder().longOpt("dry-run") + .desc("Scan only, do not write to database") + .build() + ); + options.addOption( + Option.builder("h").longOpt("help") + .desc("Print help and exit") + .build() + ); + return options; + } + + /** + * Print usage help to stdout. + * + * @param options CLI options + */ + private static void printHelp(final Options options) { + new HelpFormatter().printHelp( + "backfill-cli", + "Backfill the PostgreSQL artifacts table from disk storage", + options, + "", + true + ); + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/BatchInserter.java b/artipie-backfill/src/main/java/com/artipie/backfill/BatchInserter.java new file mode 100644 index 000000000..695d31805 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/BatchInserter.java @@ -0,0 +1,304 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Batches artifact records and inserts them into PostgreSQL using JDBC batch + * operations. Supports dry-run mode where records are counted but not + * persisted to the database. + * + *

On first call the {@code artifacts} table and its indexes are created + * if they do not already exist.

+ * + *

When a batch commit fails the inserter falls back to individual inserts + * so that a single bad record does not block the entire batch.

+ * + * @since 1.20.13 + */ +public final class BatchInserter implements AutoCloseable { + + /** + * SLF4J logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(BatchInserter.class); + + /** + * UPSERT SQL — must match DbConsumer parameter binding order exactly. + */ + private static final String UPSERT_SQL = String.join( + " ", + "INSERT INTO artifacts", + "(repo_type, repo_name, name, version, size,", + "created_date, release_date, owner, path_prefix)", + "VALUES (?,?,?,?,?,?,?,?,?)", + "ON CONFLICT (repo_name, name, version)", + "DO UPDATE SET repo_type = EXCLUDED.repo_type,", + "size = EXCLUDED.size,", + "created_date = EXCLUDED.created_date,", + "release_date = EXCLUDED.release_date,", + "owner = EXCLUDED.owner,", + "path_prefix = COALESCE(EXCLUDED.path_prefix, artifacts.path_prefix)" + ); + + /** + * JDBC data source. + */ + private final DataSource source; + + /** + * Maximum number of records per batch. + */ + private final int batchSize; + + /** + * When {@code true} records are counted but not written to the database. + */ + private final boolean dryRun; + + /** + * Buffer of records awaiting the next flush. + */ + private final List buffer; + + /** + * Total records successfully inserted (or counted in dry-run mode). + */ + private final AtomicLong insertedCount; + + /** + * Total records that could not be inserted. + */ + private final AtomicLong skippedCount; + + /** + * Whether the table DDL has already been executed in this session. + */ + private boolean tableCreated; + + /** + * Ctor. + * + * @param source JDBC data source + * @param batchSize Maximum records per batch flush + * @param dryRun If {@code true}, count only — no DB writes + */ + public BatchInserter(final DataSource source, final int batchSize, + final boolean dryRun) { + this.source = source; + this.batchSize = batchSize; + this.dryRun = dryRun; + this.buffer = new ArrayList<>(batchSize); + this.insertedCount = new AtomicLong(0L); + this.skippedCount = new AtomicLong(0L); + this.tableCreated = false; + } + + /** + * Accept a single artifact record. The record is buffered internally + * and flushed automatically when the buffer reaches {@code batchSize}. + * + * @param record Artifact record to insert + */ + public void accept(final ArtifactRecord record) { + this.buffer.add(record); + if (this.buffer.size() >= this.batchSize) { + this.flush(); + } + } + + /** + * Flush all buffered records to the database (or count them in dry-run). + */ + public void flush() { + if (this.buffer.isEmpty()) { + return; + } + if (this.dryRun) { + this.insertedCount.addAndGet(this.buffer.size()); + LOG.info("[dry-run] Would insert {} records (total: {})", + this.buffer.size(), this.insertedCount.get()); + this.buffer.clear(); + return; + } + this.ensureTable(); + final List batch = new ArrayList<>(this.buffer); + this.buffer.clear(); + try (Connection conn = this.source.getConnection()) { + conn.setAutoCommit(false); + try (PreparedStatement stmt = conn.prepareStatement(UPSERT_SQL)) { + for (final ArtifactRecord rec : batch) { + bindRecord(stmt, rec); + stmt.addBatch(); + } + stmt.executeBatch(); + conn.commit(); + this.insertedCount.addAndGet(batch.size()); + } catch (final SQLException ex) { + rollback(conn); + LOG.warn("Batch insert of {} records failed, falling back to " + + "individual inserts: {}", batch.size(), ex.getMessage()); + this.insertIndividually(batch); + } + } catch (final SQLException ex) { + LOG.warn("Failed to obtain DB connection for batch of {} records: {}", + batch.size(), ex.getMessage()); + this.skippedCount.addAndGet(batch.size()); + } + } + + /** + * Return total number of successfully inserted records. + * + * @return Inserted count + */ + public long getInsertedCount() { + return this.insertedCount.get(); + } + + /** + * Return total number of records that were skipped due to errors. + * + * @return Skipped count + */ + public long getSkippedCount() { + return this.skippedCount.get(); + } + + @Override + public void close() { + this.flush(); + LOG.info("BatchInserter closed — inserted: {}, skipped: {}", + this.insertedCount.get(), this.skippedCount.get()); + } + + /** + * Ensure the artifacts table and performance indexes exist. + * Called once per session on the first real flush. + */ + private void ensureTable() { + if (this.tableCreated) { + return; + } + try (Connection conn = this.source.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS artifacts(", + " id BIGSERIAL PRIMARY KEY,", + " repo_type VARCHAR NOT NULL,", + " repo_name VARCHAR NOT NULL,", + " name VARCHAR NOT NULL,", + " version VARCHAR NOT NULL,", + " size BIGINT NOT NULL,", + " created_date BIGINT NOT NULL,", + " release_date BIGINT,", + " owner VARCHAR NOT NULL,", + " UNIQUE (repo_name, name, version)", + ");" + ) + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_repo_lookup " + + "ON artifacts(repo_name, name, version)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_repo_type_name " + + "ON artifacts(repo_type, repo_name, name)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_created_date " + + "ON artifacts(created_date)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_owner " + + "ON artifacts(owner)" + ); + this.tableCreated = true; + LOG.info("Artifacts table and indexes verified/created"); + } catch (final SQLException ex) { + LOG.warn("Failed to create artifacts table: {}", ex.getMessage()); + } + } + + /** + * Fall back to inserting records one by one after a batch failure. + * + * @param records Records to insert individually + */ + private void insertIndividually(final List records) { + for (final ArtifactRecord rec : records) { + try (Connection conn = this.source.getConnection(); + PreparedStatement stmt = conn.prepareStatement(UPSERT_SQL)) { + conn.setAutoCommit(false); + bindRecord(stmt, rec); + stmt.executeUpdate(); + conn.commit(); + this.insertedCount.incrementAndGet(); + } catch (final SQLException ex) { + LOG.warn("Individual insert failed for {}/{}:{} — {}", + rec.repoName(), rec.name(), rec.version(), + ex.getMessage()); + this.skippedCount.incrementAndGet(); + } + } + } + + /** + * Bind an {@link ArtifactRecord} to a {@link PreparedStatement}. + * Parameter order must match the UPSERT_SQL and DbConsumer exactly. + * + * @param stmt Prepared statement + * @param rec Artifact record + * @throws SQLException On binding error + */ + private static void bindRecord(final PreparedStatement stmt, + final ArtifactRecord rec) throws SQLException { + stmt.setString(1, rec.repoType()); + stmt.setString(2, rec.repoName() == null + ? null : rec.repoName().trim()); + stmt.setString(3, rec.name()); + stmt.setString(4, rec.version()); + stmt.setLong(5, rec.size()); + stmt.setLong(6, rec.createdDate()); + if (rec.releaseDate() == null) { + stmt.setNull(7, Types.BIGINT); + } else { + stmt.setLong(7, rec.releaseDate()); + } + stmt.setString(8, rec.owner()); + if (rec.pathPrefix() == null) { + stmt.setNull(9, Types.VARCHAR); + } else { + stmt.setString(9, rec.pathPrefix()); + } + } + + /** + * Attempt to rollback the current transaction, logging any failure. + * + * @param conn JDBC connection + */ + private static void rollback(final Connection conn) { + try { + conn.rollback(); + } catch (final SQLException ex) { + LOG.warn("Rollback failed: {}", ex.getMessage()); + } + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/BulkBackfillRunner.java b/artipie-backfill/src/main/java/com/artipie/backfill/BulkBackfillRunner.java new file mode 100644 index 000000000..26258c500 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/BulkBackfillRunner.java @@ -0,0 +1,338 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Orchestrates a bulk backfill run over a directory of Artipie repo configs. + * + *

For each {@code *.yaml} file found (non-recursively, sorted alphabetically) + * in the config directory, derives the repo name from the filename stem and the + * scanner type from {@code repo.type}, then runs the appropriate {@link Scanner} + * against {@code storageRoot//}.

+ * + *

Per-repo failures (parse errors, unknown types, missing storage, scan + * exceptions) are all non-fatal: they are logged, recorded in the summary, + * and the next repo is processed. Only a {@code FAILED} status (scan exception) + * contributes to a non-zero exit code.

+ * + * @since 1.20.13 + */ +@SuppressWarnings("PMD.ExcessiveImports") +final class BulkBackfillRunner { + + /** + * SLF4J logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(BulkBackfillRunner.class); + + /** + * {@code .yaml} file extension constant. + */ + private static final String YAML_EXT = ".yaml"; + + /** + * Directory containing {@code *.yaml} Artipie repo config files. + */ + private final Path configDir; + + /** + * Root directory under which each repo's data lives at + * {@code //}. + */ + private final Path storageRoot; + + /** + * Shared JDBC data source. May be {@code null} when {@code dryRun} is + * {@code true}. + */ + private final DataSource dataSource; + + /** + * Owner string applied to all inserted artifact records. + */ + private final String owner; + + /** + * Batch insert size. + */ + private final int batchSize; + + /** + * If {@code true} count records but do not write to the database. + */ + private final boolean dryRun; + + /** + * Progress log interval (log every N records per repo). + */ + private final int logInterval; + + /** + * Print stream for the summary table (typically {@code System.err}). + */ + private final PrintStream out; + + /** + * Ctor. + * + * @param configDir Directory of repo YAML configs + * @param storageRoot Root for repo storage directories + * @param dataSource JDBC data source (may be null when dryRun=true) + * @param owner Owner string for artifact records + * @param batchSize JDBC batch insert size + * @param dryRun If true, count only, no DB writes + * @param logInterval Progress log every N records + * @param out Stream for summary output (typically System.err) + * @checkstyle ParameterNumberCheck (10 lines) + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + BulkBackfillRunner( + final Path configDir, + final Path storageRoot, + final DataSource dataSource, + final String owner, + final int batchSize, + final boolean dryRun, + final int logInterval, + final PrintStream out + ) { + this.configDir = configDir; + this.storageRoot = storageRoot; + this.dataSource = dataSource; + this.owner = owner; + this.batchSize = batchSize; + this.dryRun = dryRun; + this.logInterval = logInterval; + this.out = out; + } + + /** + * Run the bulk backfill over all {@code *.yaml} files in the config + * directory. + * + * @return Exit code: {@code 0} if all repos succeeded or were + * skipped/parse-errored, {@code 1} if any repo had a scan failure + * @throws IOException if the config directory cannot be listed + */ + int run() throws IOException { + final List results = new ArrayList<>(); + final Set seenNames = new HashSet<>(); + final List yamlFiles = new ArrayList<>(); + try (Stream listing = Files.list(this.configDir)) { + listing + .filter(Files::isRegularFile) + .forEach(p -> { + final String name = p.getFileName().toString(); + if (name.endsWith(YAML_EXT)) { + yamlFiles.add(p); + } else if (name.endsWith(".yml")) { + LOG.debug( + "Skipping '{}' — use .yaml extension, not .yml", + p.getFileName() + ); + } + }); + } + yamlFiles.sort(Path::compareTo); + for (final Path file : yamlFiles) { + results.add(this.processFile(file, seenNames)); + } + this.printSummary(results); + return results.stream() + .anyMatch(r -> r.status().startsWith("FAILED")) ? 1 : 0; + } + + /** + * Process one YAML file and return a result row. + * + * @param file Path to the {@code .yaml} file + * @param seenNames Set of repo name stems already processed + * @return Result row for the summary table + */ + private RepoResult processFile( + final Path file, + final Set seenNames + ) { + final String fileName = file.getFileName().toString(); + final String stem = fileName.endsWith(YAML_EXT) + ? fileName.substring(0, fileName.length() - YAML_EXT.length()) + : fileName; + if (!seenNames.add(stem)) { + LOG.warn( + "Duplicate repo name '{}' (from '{}'), skipping", stem, fileName + ); + return new RepoResult( + stem, "-", -1L, -1L, "SKIPPED (duplicate repo name)" + ); + } + final RepoEntry entry; + try { + entry = RepoConfigYaml.parse(file); + } catch (final IOException ex) { + LOG.warn("PARSE_ERROR for '{}': {}", fileName, ex.getMessage()); + return new RepoResult( + stem, "-", -1L, -1L, + "PARSE_ERROR (" + ex.getMessage() + ")" + ); + } + final String rawType = entry.rawType(); + final Scanner scanner; + try { + scanner = ScannerFactory.create(rawType); + } catch (final IllegalArgumentException ex) { + LOG.warn( + "Unknown type '{}' for repo '{}', skipping", + rawType, stem + ); + return new RepoResult( + stem, "[UNKNOWN]", -1L, -1L, + "SKIPPED (unknown type: " + rawType + ")" + ); + } + final Path storagePath = this.storageRoot.resolve(stem); + if (!Files.exists(storagePath)) { + LOG.warn( + "Storage path missing for repo '{}': {}", stem, storagePath + ); + return new RepoResult( + stem, rawType, -1L, -1L, "SKIPPED (storage path missing)" + ); + } + return this.scanRepo(stem, rawType, scanner, storagePath); + } + + /** + * Scan one repo directory and return a result row. + * + * @param repoName Repo name (for logging and record insertion) + * @param scannerType Normalised scanner type string (for display) + * @param scanner Scanner instance + * @param storagePath Root directory to scan + * @return Result row + */ + private RepoResult scanRepo( + final String repoName, + final String scannerType, + final Scanner scanner, + final Path storagePath + ) { + LOG.info( + "Scanning repo '{}' (type={}) at {}", + repoName, scannerType, storagePath + ); + final ProgressReporter reporter = + new ProgressReporter(this.logInterval); + long inserted = -1L; + long dbSkipped = -1L; + boolean failed = false; + String failMsg = null; + final BatchInserter inserter = new BatchInserter( + this.dataSource, this.batchSize, this.dryRun + ); + try ( + inserter; + Stream stream = + scanner.scan(storagePath, repoName) + ) { + stream + .map(r -> new ArtifactRecord( + r.repoType(), r.repoName(), r.name(), + r.version(), r.size(), r.createdDate(), + r.releaseDate(), this.owner, r.pathPrefix() + )) + .forEach(rec -> { + inserter.accept(rec); + reporter.increment(); + }); + } catch (final Exception ex) { + // inserter.close() was called by try-with-resources before this catch block. + // For FAILED rows, use -1L sentinel per design. + failed = true; + failMsg = ex.getMessage(); + LOG.error( + "Scan FAILED for repo '{}': {}", repoName, ex.getMessage(), ex + ); + } + // inserter.close() has been called (flushed remaining batch). Read final counts. + if (!failed) { + inserted = inserter.getInsertedCount(); + dbSkipped = inserter.getSkippedCount(); + } + reporter.printFinalSummary(); + if (failed) { + return new RepoResult( + repoName, scannerType, -1L, -1L, + "FAILED (" + failMsg + ")" + ); + } + return new RepoResult(repoName, scannerType, inserted, dbSkipped, "OK"); + } + + /** + * Print the summary table to the output stream. + * + * @param results List of result rows + */ + private void printSummary(final List results) { + this.out.printf( + "%nBulk backfill complete — %d repos processed%n", + results.size() + ); + for (final RepoResult row : results) { + final String counts; + if (row.inserted() < 0) { + counts = String.format("%-30s", "-"); + } else { + counts = String.format( + "inserted=%-10d skipped=%-6d", + row.inserted(), row.dbSkipped() + ); + } + this.out.printf( + " %-20s [%-12s] %s %s%n", + row.repoName(), row.displayType(), counts, row.status() + ); + } + final long failCount = results.stream() + .filter(r -> r.status().startsWith("FAILED")).count(); + if (failCount > 0) { + this.out.printf("%nExit code: 1 (%d repo(s) failed)%n", failCount); + } else { + this.out.println("\nExit code: 0"); + } + } + + /** + * One row in the bulk run summary. + * + * @param repoName Repo name + * @param displayType Type string for display + * @param inserted Records inserted (or -1 if not applicable) + * @param dbSkipped Records skipped at DB level (or -1 if not applicable) + * @param status Status string + */ + private record RepoResult( + String repoName, + String displayType, + long inserted, + long dbSkipped, + String status + ) { + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/ComposerScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/ComposerScanner.java new file mode 100644 index 000000000..01f86ee93 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/ComposerScanner.java @@ -0,0 +1,373 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.json.Json; +import javax.json.JsonException; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for Composer (PHP) repositories. + * + *

Supports two layouts:

+ *
    + *
  • p2 (Satis-style): per-package JSON files under + * {@code p2/{vendor}/{package}.json}. Files ending with {@code ~dev.json} + * are skipped.
    • + *
  • packages.json: a single root-level file containing + * all package metadata.
    • + *
+ * + *

The p2 layout is checked first; if the {@code p2/} directory exists, + * {@code packages.json} is ignored even if present.

+ * + * @since 1.20.13 + */ +final class ComposerScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(ComposerScanner.class); + + /** + * Repository type string stored in every produced artifact record + * (e.g. {@code "composer"} or {@code "php"}). + */ + private final String repoType; + + /** + * Ctor with default repo type {@code "composer"}. + */ + ComposerScanner() { + this("composer"); + } + + /** + * Ctor. + * + * @param repoType Repository type string for artifact records + */ + ComposerScanner(final String repoType) { + this.repoType = repoType; + } + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + final Path p2dir = root.resolve("p2"); + if (Files.isDirectory(p2dir)) { + return this.scanP2(root, repoName, p2dir); + } + final Path packagesJson = root.resolve("packages.json"); + if (Files.isRegularFile(packagesJson) && Files.size(packagesJson) > 0) { + final List from = + this.parseJsonFile(root, repoName, packagesJson) + .collect(Collectors.toList()); + if (!from.isEmpty()) { + return from.stream(); + } + LOG.debug( + "packages.json has no packages, trying vendor-dir layout" + ); + } + return this.scanVendorDirs(root, repoName); + } + + /** + * Scan the p2 directory layout. Walks all {@code .json} files under + * {@code p2/}, skipping any that end with {@code ~dev.json}. + * + * @param root Repository root directory + * @param repoName Logical repository name + * @param p2dir Path to the p2 directory + * @return Stream of artifact records + * @throws IOException If an I/O error occurs + */ + private Stream scanP2(final Path root, + final String repoName, final Path p2dir) throws IOException { + return Files.walk(p2dir) + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".json")) + .filter(path -> !path.getFileName().toString().endsWith("~dev.json")) + .flatMap(path -> this.parseJsonFile(root, repoName, path)); + } + + /** + * Scan the Artipie Composer proxy layout. + * + *

The Artipie Composer proxy caches per-package metadata as + * {@code {vendor}/{package}.json} files directly under the repository + * root (no {@code p2/} prefix). Each file uses the standard Composer + * {@code {"packages":{...}}} JSON format.

+ * + *

Files ending with {@code ~dev.json} and 0-byte files are skipped.

+ * + * @param root Repository root directory + * @param repoName Logical repository name + * @return Stream of artifact records + * @throws IOException If an I/O error occurs + */ + private Stream scanVendorDirs(final Path root, + final String repoName) throws IOException { + return Files.list(root) + .filter(Files::isDirectory) + .filter(dir -> !dir.getFileName().toString().startsWith(".")) + .flatMap( + vendorDir -> { + try { + return Files.list(vendorDir) + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".json")) + .filter( + path -> !path.getFileName().toString().endsWith("~dev.json") + ) + .filter( + path -> { + try { + return Files.size(path) > 0L; + } catch (final IOException ex) { + LOG.debug("Cannot stat {}, skipping: {}", path, ex.getMessage()); + return false; + } + } + ) + .flatMap(path -> this.parseJsonFile(root, repoName, path)); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + } + ); + } + + /** + * Parse a single Composer JSON file and produce artifact records. + * + * @param root Repository root directory + * @param repoName Logical repository name + * @param jsonPath Path to the JSON file + * @return Stream of artifact records + */ + private Stream parseJsonFile(final Path root, + final String repoName, final Path jsonPath) { + final JsonObject json; + try (InputStream input = Files.newInputStream(jsonPath); + JsonReader reader = Json.createReader(input)) { + json = reader.readObject(); + } catch (final JsonException ex) { + LOG.warn("Malformed JSON in {}: {}", jsonPath, ex.getMessage()); + return Stream.empty(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + if (!json.containsKey("packages") + || json.isNull("packages") + || json.get("packages").getValueType() != JsonValue.ValueType.OBJECT) { + LOG.debug("Missing or invalid 'packages' key in {}", jsonPath); + return Stream.empty(); + } + final JsonObject packages = json.getJsonObject("packages"); + final long mtime; + try { + mtime = Files.readAttributes(jsonPath, BasicFileAttributes.class) + .lastModifiedTime().toMillis(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + final boolean proxyMode = this.repoType.endsWith("-proxy"); + final List records = new ArrayList<>(); + for (final String packageName : packages.keySet()) { + if (packages.isNull(packageName) + || packages.get(packageName).getValueType() + != JsonValue.ValueType.OBJECT) { + LOG.debug("Skipping non-object package entry: {}", packageName); + continue; + } + final JsonObject versions = packages.getJsonObject(packageName); + for (final String version : versions.keySet()) { + if (versions.isNull(version) + || versions.get(version).getValueType() + != JsonValue.ValueType.OBJECT) { + LOG.debug( + "Skipping non-object version entry: {} {}", + packageName, version + ); + continue; + } + final JsonObject versionObj = versions.getJsonObject(version); + // For proxy repos, only record versions that have cached + // dist artifacts on disk. The metadata JSON lists all upstream + // versions but only downloaded ones have actual files. + // Check both .zip (new format) and plain (legacy). + if (proxyMode) { + final Path distDir = root.resolve("dist") + .resolve(packageName); + final Path zipFile = distDir.resolve(version + ".zip"); + final Path legacyFile = distDir.resolve(version); + if (!Files.exists(zipFile) && !Files.exists(legacyFile)) { + continue; + } + } + long size = ComposerScanner.resolveDistSize( + root, versionObj + ); + // For proxy repos, if dist URL resolution failed, read size + // directly from the cached file on disk + if (size == 0L && proxyMode) { + final Path distDir = root.resolve("dist") + .resolve(packageName); + final Path zipFile = distDir.resolve(version + ".zip"); + final Path legacyFile = distDir.resolve(version); + try { + if (Files.isRegularFile(zipFile)) { + size = Files.size(zipFile); + } else if (Files.isRegularFile(legacyFile)) { + size = Files.size(legacyFile); + } + } catch (final IOException ignored) { + // keep size = 0 + } + } + final String pathPrefix = proxyMode + ? packageName + "/" + version : null; + records.add( + new ArtifactRecord( + this.repoType, + repoName, + packageName, + version, + size, + mtime, + null, + "system", + pathPrefix + ) + ); + } + } + return records.stream(); + } + + /** + * Resolve the dist artifact size for a version entry. + * + *

Tries to extract the {@code dist.url} field and resolve it as a + * local file path. For HTTP URLs the path component is extracted and + * attempted relative to the repository root. If the file cannot be + * found the size is 0.

+ * + * @param root Repository root directory + * @param versionObj Version metadata JSON object + * @return Size in bytes, or 0 if the artifact cannot be found + */ + private static long resolveDistSize(final Path root, + final JsonObject versionObj) { + if (!versionObj.containsKey("dist") + || versionObj.isNull("dist") + || versionObj.get("dist").getValueType() + != JsonValue.ValueType.OBJECT) { + return 0L; + } + final JsonObject dist = versionObj.getJsonObject("dist"); + if (!dist.containsKey("url") + || dist.isNull("url") + || dist.get("url").getValueType() != JsonValue.ValueType.STRING) { + return 0L; + } + final String url = dist.getString("url"); + return ComposerScanner.sizeFromUrl(root, url); + } + + /** + * Attempt to resolve a dist URL to a local file and return its size. + * + * @param root Repository root directory + * @param url The dist URL string + * @return File size in bytes, or 0 if the file is not found + */ + private static long sizeFromUrl(final Path root, final String url) { + String localPath; + if (url.startsWith("http://") || url.startsWith("https://")) { + try { + localPath = URI.create(url).getPath(); + } catch (final IllegalArgumentException ex) { + LOG.debug("Cannot parse dist URL '{}': {}", url, ex.getMessage()); + return 0L; + } + } else { + localPath = url; + } + if (localPath == null || localPath.isEmpty()) { + return 0L; + } + if (localPath.startsWith("/")) { + localPath = localPath.substring(1); + } + final Path resolved = root.resolve(localPath); + if (Files.isRegularFile(resolved)) { + try { + return Files.size(resolved); + } catch (final IOException ex) { + LOG.debug("Cannot stat {}: {}", resolved, ex.getMessage()); + return 0L; + } + } + final int lastSlash = localPath.lastIndexOf('/'); + if (lastSlash >= 0) { + final String filename = localPath.substring(lastSlash + 1); + final Path fallback = root.resolve(filename); + if (Files.isRegularFile(fallback)) { + try { + return Files.size(fallback); + } catch (final IOException ex) { + LOG.debug( + "Cannot stat fallback {}: {}", + fallback, ex.getMessage() + ); + return 0L; + } + } + } + // Final fallback: progressively strip leading path segments. + // Handles Artipie local PHP repos where the dist URL contains + // a full HTTP path like "/prefix/api/composer/repo/artifacts/...". + String stripped = localPath; + while (stripped.contains("/")) { + stripped = stripped.substring(stripped.indexOf('/') + 1); + if (stripped.isEmpty()) { + break; + } + final Path candidate = root.resolve(stripped); + if (Files.isRegularFile(candidate)) { + try { + return Files.size(candidate); + } catch (final IOException ex) { + LOG.debug( + "Cannot stat candidate {}: {}", + candidate, ex.getMessage() + ); + return 0L; + } + } + } + return 0L; + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/DebianScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/DebianScanner.java new file mode 100644 index 000000000..9634b6b3e --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/DebianScanner.java @@ -0,0 +1,213 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for Debian repositories. + * + *

Walks the repository directory tree to find {@code Packages} and + * {@code Packages.gz} index files under the standard Debian layout + * ({@code dists/{codename}/{component}/binary-{arch}/}). Each stanza + * in a Packages file describes one {@code .deb} package. The scanner + * extracts the {@code Package}, {@code Version}, and {@code Size} + * fields from each stanza.

+ * + *

When both {@code Packages} and {@code Packages.gz} exist in the + * same directory, only {@code Packages.gz} is used to avoid + * double-counting.

+ * + * @since 1.20.13 + */ +final class DebianScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(DebianScanner.class); + + /** + * Name of the uncompressed Packages index file. + */ + private static final String PACKAGES = "Packages"; + + /** + * Name of the gzip-compressed Packages index file. + */ + private static final String PACKAGES_GZ = "Packages.gz"; + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + final List indexFiles = Files.walk(root) + .filter(Files::isRegularFile) + .filter(DebianScanner::isPackagesFile) + .collect(Collectors.toList()); + final List deduped = DebianScanner.dedup(indexFiles); + return deduped.stream() + .flatMap(path -> DebianScanner.parseIndex(path, repoName)); + } + + /** + * Check whether a file is a Packages or Packages.gz index file. + * + * @param path File path to check + * @return True if the filename is "Packages" or "Packages.gz" + */ + private static boolean isPackagesFile(final Path path) { + final String name = path.getFileName().toString(); + return PACKAGES.equals(name) || PACKAGES_GZ.equals(name); + } + + /** + * Deduplicate index files by parent directory. + * When both Packages and Packages.gz exist in the same directory, + * prefer Packages.gz. + * + * @param files List of discovered index files + * @return Deduplicated list preferring .gz files + */ + private static List dedup(final List files) { + final Map byParent = new HashMap<>(); + for (final Path file : files) { + final Path parent = file.getParent(); + final Path existing = byParent.get(parent); + if (existing == null) { + byParent.put(parent, file); + } else if (file.getFileName().toString().equals(PACKAGES_GZ)) { + byParent.put(parent, file); + } + } + return new ArrayList<>(byParent.values()); + } + + /** + * Parse a single Packages or Packages.gz file into artifact records. + * + * @param path Path to the index file + * @param repoName Logical repository name + * @return Stream of artifact records parsed from the index + */ + private static Stream parseIndex(final Path path, + final String repoName) { + try { + final long mtime = Files.getLastModifiedTime(path).toMillis(); + final List records = new ArrayList<>(); + try ( + InputStream fis = Files.newInputStream(path); + InputStream input = path.getFileName().toString().equals(PACKAGES_GZ) + ? new GZIPInputStream(fis) : fis; + BufferedReader reader = new BufferedReader( + new InputStreamReader(input, StandardCharsets.UTF_8) + ) + ) { + String pkg = null; + String version = null; + String arch = null; + long size = 0L; + String line = reader.readLine(); + while (line != null) { + if (line.isEmpty()) { + if (pkg != null && version != null) { + records.add( + new ArtifactRecord( + "deb", + repoName, + DebianScanner.formatName(pkg, arch), + version, + size, + mtime, + null, + "system", + null + ) + ); + } else if (pkg != null || version != null) { + LOG.debug( + "Skipping incomplete stanza (Package={}, Version={}) in {}", + pkg, version, path + ); + } + pkg = null; + version = null; + arch = null; + size = 0L; + } else if (line.startsWith("Package:")) { + pkg = line.substring("Package:".length()).trim(); + } else if (line.startsWith("Version:")) { + version = line.substring("Version:".length()).trim(); + } else if (line.startsWith("Architecture:")) { + arch = line.substring("Architecture:".length()).trim(); + } else if (line.startsWith("Size:")) { + try { + size = Long.parseLong( + line.substring("Size:".length()).trim() + ); + } catch (final NumberFormatException ex) { + LOG.debug( + "Invalid Size value in {}: {}", + path, line + ); + size = 0L; + } + } + line = reader.readLine(); + } + if (pkg != null && version != null) { + records.add( + new ArtifactRecord( + "deb", + repoName, + DebianScanner.formatName(pkg, arch), + version, + size, + mtime, + null, + "system", + null + ) + ); + } + } + return records.stream(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + } + + /** + * Format the artifact name. The Debian adapter stores artifact names + * as {@code package_architecture} (e.g. {@code curl_amd64}). + * If architecture is missing, uses just the package name. + * + * @param pkg Package name + * @param arch Architecture string, or null if not present + * @return Formatted name + */ + private static String formatName(final String pkg, final String arch) { + if (arch != null && !arch.isEmpty()) { + return String.join("_", pkg, arch); + } + return pkg; + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/DockerScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/DockerScanner.java new file mode 100644 index 000000000..ba677626b --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/DockerScanner.java @@ -0,0 +1,386 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonException; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for Docker v2 registry repositories. + * + *

Walks the Docker registry storage layout looking for image repositories + * under {@code repositories/}, reads tag link files to resolve manifest + * digests, and parses manifest JSON to compute artifact sizes.

+ * + * @since 1.20.13 + */ +final class DockerScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(DockerScanner.class); + + /** + * Name of the manifests metadata directory. + */ + private static final String MANIFESTS_DIR = "_manifests"; + + /** + * Name of the tags subdirectory. + */ + private static final String TAGS_DIR = "tags"; + + /** + * Repository type string stored in every produced artifact record + * (e.g. {@code "docker"} or {@code "docker-proxy"}). + */ + private final String repoType; + + /** + * When {@code true} this is a proxy repo — image names match the + * upstream pull path with no prefix. When {@code false} (local/hosted) + * the Artipie Docker push path includes the registry name in the image + * path, so we prepend {@code repoName + "/"} to match production. + */ + private final boolean isProxy; + + /** + * Ctor for local (hosted) Docker repos. + */ + DockerScanner() { + this("docker", false); + } + + /** + * Ctor. + * + * @param isProxy {@code true} for proxy repos, {@code false} for local + */ + DockerScanner(final boolean isProxy) { + this(isProxy ? "docker-proxy" : "docker", isProxy); + } + + /** + * Ctor. + * + * @param repoType Repository type string for artifact records + * @param isProxy {@code true} for proxy repos, {@code false} for local + */ + DockerScanner(final String repoType, final boolean isProxy) { + this.repoType = repoType; + this.isProxy = isProxy; + } + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + final Path reposDir = DockerScanner.resolveReposDir(root); + if (reposDir == null) { + LOG.warn("No repositories directory found under {}", root); + return Stream.empty(); + } + final Path blobsRoot = reposDir.getParent().resolve("blobs"); + final List images = DockerScanner.findImages(reposDir); + final List records = new ArrayList<>(); + for (final Path imageDir : images) { + final String rawImageName = + reposDir.relativize(imageDir).toString(); + final String imageName = this.isProxy + ? rawImageName + : repoName + "/" + rawImageName; + final Path tagsDir = imageDir + .resolve(DockerScanner.MANIFESTS_DIR) + .resolve(DockerScanner.TAGS_DIR); + if (!Files.isDirectory(tagsDir)) { + continue; + } + try (Stream tagDirs = Files.list(tagsDir)) { + final List tagList = tagDirs + .filter(Files::isDirectory) + .toList(); + for (final Path tagDir : tagList) { + final ArtifactRecord record = this.processTag( + blobsRoot, repoName, imageName, tagDir + ); + if (record != null) { + records.add(record); + } + } + } + } + return records.stream(); + } + + /** + * Resolve the repositories directory. Checks common Docker registry + * v2 layouts: + *
    + *
  • {@code root/repositories/}
    • + *
  • {@code root/docker/registry/v2/repositories/}
    • + *
+ * Falls back to walking for a directory named {@code repositories} + * that contains image dirs with {@code _manifests/}. + * + * @param root Registry root path + * @return Path to the repositories directory, or null if not found + * @throws IOException If an I/O error occurs during directory walk + */ + private static Path resolveReposDir(final Path root) throws IOException { + final Path direct = root.resolve("repositories"); + if (Files.isDirectory(direct)) { + return direct; + } + final Path v2 = root.resolve("docker/registry/v2/repositories"); + if (Files.isDirectory(v2)) { + return v2; + } + try (Stream walk = Files.walk(root)) { + return walk.filter(Files::isDirectory) + .filter( + p -> "repositories".equals(p.getFileName().toString()) + ) + .findFirst() + .orElse(null); + } + } + + /** + * Walk the repositories directory to find all image directories. + * An image directory is one that contains {@code _manifests/tags/}. + * + * @param reposDir The repositories root directory + * @return List of image directory paths + * @throws IOException If an I/O error occurs + */ + private static List findImages(final Path reposDir) + throws IOException { + final List images = new ArrayList<>(); + try (Stream walker = Files.walk(reposDir)) { + walker.filter(Files::isDirectory) + .filter( + dir -> { + final Path manifests = dir + .resolve(DockerScanner.MANIFESTS_DIR) + .resolve(DockerScanner.TAGS_DIR); + return Files.isDirectory(manifests); + } + ) + .forEach(images::add); + } + return images; + } + + /** + * Process a single tag directory and produce an artifact record. + * + * @param blobsRoot Path to the blobs directory + * @param repoName Logical repository name + * @param imageName Image name (relative path from repositories dir) + * @param tagDir Tag directory path + * @return ArtifactRecord, or null if tag should be skipped + */ + private ArtifactRecord processTag(final Path blobsRoot, + final String repoName, final String imageName, final Path tagDir) { + final String tag = tagDir.getFileName().toString(); + final Path linkFile = tagDir.resolve("current").resolve("link"); + if (!Files.isRegularFile(linkFile)) { + LOG.debug("No link file at {}", linkFile); + return null; + } + final String digest; + try { + digest = Files.readString(linkFile, StandardCharsets.UTF_8).trim(); + } catch (final IOException ex) { + LOG.warn("Cannot read link file {}: {}", linkFile, ex.getMessage()); + return null; + } + if (digest.isEmpty()) { + LOG.debug("Empty link file at {}", linkFile); + return null; + } + final long createdDate = DockerScanner.linkMtime(linkFile); + final long size = DockerScanner.resolveSize(blobsRoot, digest); + return new ArtifactRecord( + this.repoType, + repoName, + imageName, + tag, + size, + createdDate, + null, + "system", + null + ); + } + + /** + * Resolve the total size of an artifact from its manifest digest. + * For image manifests with layers, sums config.size + layers[].size. + * For manifest lists, uses the manifest blob file's own size. + * Returns 0 if the blob is missing or manifest is corrupt. + * + * @param blobsRoot Path to the blobs directory + * @param digest Digest string like "sha256:abc123..." + * @return Total size in bytes + */ + private static long resolveSize(final Path blobsRoot, + final String digest) { + final Path blobPath = DockerScanner.digestToPath(blobsRoot, digest); + if (blobPath == null || !Files.isRegularFile(blobPath)) { + LOG.debug("Blob not found for digest {}", digest); + return 0L; + } + final JsonObject manifest; + try (InputStream input = Files.newInputStream(blobPath); + JsonReader reader = Json.createReader(input)) { + manifest = reader.readObject(); + } catch (final JsonException ex) { + LOG.warn( + "Corrupted manifest JSON for digest {}: {}", + digest, ex.getMessage() + ); + return 0L; + } catch (final IOException ex) { + LOG.warn("Cannot read blob {}: {}", blobPath, ex.getMessage()); + return 0L; + } + if (manifest.containsKey("manifests") + && manifest.get("manifests").getValueType() + == JsonValue.ValueType.ARRAY) { + return DockerScanner.resolveManifestListSize( + blobsRoot, manifest.getJsonArray("manifests") + ); + } + return DockerScanner.sumLayersAndConfig(manifest); + } + + /** + * Sum config.size and all layers[].size from an image manifest. + * + * @param manifest Parsed manifest JSON object + * @return Total size in bytes, or 0 if fields are missing + */ + private static long sumLayersAndConfig(final JsonObject manifest) { + long total = 0L; + if (manifest.containsKey("config") + && manifest.get("config").getValueType() + == JsonValue.ValueType.OBJECT) { + final JsonObject config = manifest.getJsonObject("config"); + if (config.containsKey("size")) { + total += config.getJsonNumber("size").longValue(); + } + } + if (manifest.containsKey("layers") + && manifest.get("layers").getValueType() + == JsonValue.ValueType.ARRAY) { + final JsonArray layers = manifest.getJsonArray("layers"); + for (final JsonValue layer : layers) { + if (layer.getValueType() == JsonValue.ValueType.OBJECT) { + final JsonObject layerObj = layer.asJsonObject(); + if (layerObj.containsKey("size")) { + total += layerObj.getJsonNumber("size").longValue(); + } + } + } + } + return total; + } + + /** + * Resolve the total size of a manifest list by summing the sizes + * of all child image manifests' layers and configs. + * + * @param blobsRoot Path to the blobs directory + * @param children The "manifests" JSON array from the manifest list + * @return Total size in bytes across all child manifests + */ + private static long resolveManifestListSize(final Path blobsRoot, + final JsonArray children) { + long total = 0L; + for (final JsonValue entry : children) { + if (entry.getValueType() != JsonValue.ValueType.OBJECT) { + continue; + } + final JsonObject child = entry.asJsonObject(); + final String childDigest = child.getString("digest", null); + if (childDigest == null || childDigest.isEmpty()) { + continue; + } + final Path childPath = + DockerScanner.digestToPath(blobsRoot, childDigest); + if (childPath == null || !Files.isRegularFile(childPath)) { + LOG.debug("Child manifest blob not found: {}", childDigest); + continue; + } + try (InputStream input = Files.newInputStream(childPath); + JsonReader reader = Json.createReader(input)) { + final JsonObject childManifest = reader.readObject(); + total += DockerScanner.sumLayersAndConfig(childManifest); + } catch (final JsonException | IOException ex) { + LOG.warn("Cannot read child manifest {}: {}", + childDigest, ex.getMessage()); + } + } + return total; + } + + /** + * Convert a digest string to a blob file path. + * + * @param blobsRoot Root blobs directory + * @param digest Digest like "sha256:abc123def..." + * @return Path to the data file, or null if digest format is invalid + */ + private static Path digestToPath(final Path blobsRoot, + final String digest) { + final String[] parts = digest.split(":", 2); + if (parts.length != 2 || parts[1].length() < 2) { + LOG.warn("Invalid digest format: {}", digest); + return null; + } + final String algorithm = parts[0]; + final String hex = parts[1]; + return blobsRoot + .resolve(algorithm) + .resolve(hex.substring(0, 2)) + .resolve(hex) + .resolve("data"); + } + + /** + * Get the last-modified time of the link file as epoch millis. + * + * @param linkFile Path to the link file + * @return Epoch millis + */ + private static long linkMtime(final Path linkFile) { + try { + return Files.readAttributes(linkFile, BasicFileAttributes.class) + .lastModifiedTime().toMillis(); + } catch (final IOException ex) { + LOG.debug( + "Cannot read mtime of {}: {}", linkFile, ex.getMessage() + ); + return System.currentTimeMillis(); + } + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/FileScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/FileScanner.java new file mode 100644 index 000000000..9781ddaa9 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/FileScanner.java @@ -0,0 +1,101 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +/** + * Scanner for generic file repositories. + * + *

Walks the directory tree rooted at the given path, filters out + * hidden files (names starting with {@code .}), and maps every + * regular file to an {@link ArtifactRecord} with {@code repoType="file"}, + * an empty version string, the file size, and the last-modified time + * as the creation date.

+ * + * @since 1.20.13 + */ +final class FileScanner implements Scanner { + + /** + * Repository type string stored in every produced artifact record + * (e.g. {@code "file"} or {@code "file-proxy"}). + */ + private final String repoType; + + /** + * Owner string to set on every produced record. + */ + private final String owner; + + /** + * Ctor with default repo type {@code "file"} and owner {@code "system"}. + */ + FileScanner() { + this("file", "system"); + } + + /** + * Ctor with given repo type and default owner {@code "system"}. + * + * @param repoType Repository type string for artifact records + */ + FileScanner(final String repoType) { + this(repoType, "system"); + } + + /** + * Ctor. + * + * @param repoType Repository type string for artifact records + * @param owner Owner identifier for produced records + */ + FileScanner(final String repoType, final String owner) { + this.repoType = repoType; + this.owner = owner; + } + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + return Files.walk(root) + .filter(Files::isRegularFile) + .filter(path -> !path.getFileName().toString().startsWith(".")) + .map(path -> this.toRecord(root, repoName, path)); + } + + /** + * Convert a file path to an artifact record. + * + * @param root Repository root directory + * @param repoName Logical repository name + * @param path File path + * @return Artifact record + */ + private ArtifactRecord toRecord(final Path root, final String repoName, + final Path path) { + try { + final String relative = root.relativize(path) + .toString().replace('\\', '/').replace('/', '.'); + return new ArtifactRecord( + this.repoType, + repoName, + relative, + "UNKNOWN", + Files.size(path), + Files.getLastModifiedTime(path).toMillis(), + null, + this.owner, + null + ); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/GemScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/GemScanner.java new file mode 100644 index 000000000..bcd971d0e --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/GemScanner.java @@ -0,0 +1,122 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for Ruby gem repositories. + * + *

Walks the repository directory tree looking for {@code .gem} files. + * If a {@code gems/} subdirectory exists under the root, only that + * subdirectory is scanned; otherwise the root itself is scanned + * (flat layout). Each {@code .gem} filename is parsed with a regex + * to extract the gem name and version.

+ * + *

The filename convention is + * {@code {name}-{version}(-{platform}).gem}. Gem names may contain + * hyphens (e.g. {@code net-http}, {@code ruby-ole}), so the version + * is identified as the first hyphen-separated segment that starts + * with a digit.

+ * + * @since 1.20.13 + */ +final class GemScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(GemScanner.class); + + /** + * Pattern for gem filenames. + * Captures the gem name (which may contain hyphens) and the + * version (which starts with a digit). An optional platform + * suffix (e.g. {@code -x86_64-linux}) is allowed but not + * captured. + * Examples: + *
    + *
  • {@code rails-7.0.4.gem} -> name=rails, version=7.0.4
    • + *
  • {@code net-http-0.3.2.gem} -> name=net-http, version=0.3.2
    • + *
  • {@code nokogiri-1.13.8-x86_64-linux.gem} -> name=nokogiri, version=1.13.8
    • + *
  • {@code ruby-ole-1.2.12.7.gem} -> name=ruby-ole, version=1.2.12.7
    • + *
+ */ + private static final Pattern GEM_PATTERN = Pattern.compile( + "^(?.+?)-(?\\d[A-Za-z0-9._]*)(?:-[A-Za-z0-9_]+(?:-[A-Za-z0-9_]+)*)?[.]gem$" + ); + + /** + * Name of the standard gems subdirectory. + */ + private static final String GEMS_DIR = "gems"; + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + final Path base; + if (Files.isDirectory(root.resolve(GemScanner.GEMS_DIR))) { + base = root.resolve(GemScanner.GEMS_DIR); + } else { + base = root; + } + return Files.walk(base, 1) + .filter(Files::isRegularFile) + .filter(path -> !path.getFileName().toString().startsWith(".")) + .filter(path -> path.getFileName().toString().endsWith(".gem")) + .flatMap(path -> this.tryParse(repoName, path)); + } + + /** + * Attempt to parse a gem file path into an artifact record. + * + * @param repoName Logical repository name + * @param path File path to parse + * @return Stream with a single record, or empty if filename does not match + */ + private Stream tryParse(final String repoName, + final Path path) { + final String filename = path.getFileName().toString(); + final Matcher matcher = GEM_PATTERN.matcher(filename); + if (!matcher.matches()) { + LOG.debug( + "Skipping non-conforming gem filename: {}", filename + ); + return Stream.empty(); + } + final String name = matcher.group("name"); + final String version = matcher.group("version"); + try { + final BasicFileAttributes attrs = Files.readAttributes( + path, BasicFileAttributes.class + ); + return Stream.of( + new ArtifactRecord( + "gem", + repoName, + name, + version, + attrs.size(), + attrs.lastModifiedTime().toMillis(), + null, + "system", + null + ) + ); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/GoScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/GoScanner.java new file mode 100644 index 000000000..10b088322 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/GoScanner.java @@ -0,0 +1,331 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import javax.json.Json; +import javax.json.JsonException; +import javax.json.JsonObject; +import javax.json.JsonReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for Go module repositories. + * + *

Walks every {@code @v} directory in the tree. For each one:

+ *
    + *
  • If a {@code list} file is present, versions are read from it + * and the corresponding {@code .zip} files are resolved.
    • + *
  • Otherwise, all {@code .zip} files in the directory are + * enumerated directly. The paired {@code .info} file is used + * for date resolution when available.
    • + *
+ *

This per-directory dispatch ensures proxy repos where some modules + * have a {@code list} file and others do not are both captured.

+ * + * @since 1.20.13 + */ +final class GoScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(GoScanner.class); + + /** + * Repository type string stored in every produced artifact record + * (e.g. {@code "go"} or {@code "go-proxy"}). + */ + private final String repoType; + + /** + * Ctor with default repo type {@code "go"}. + */ + GoScanner() { + this("go"); + } + + /** + * Ctor. + * + * @param repoType Repository type string for artifact records + */ + GoScanner(final String repoType) { + this.repoType = repoType; + } + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + final List records = new ArrayList<>(); + try (Stream walk = Files.walk(root)) { + walk.filter(Files::isDirectory) + .filter(p -> "@v".equals(p.getFileName().toString())) + .forEach(atVDir -> { + final Path listFile = atVDir.resolve("list"); + if (Files.isRegularFile(listFile)) { + this.processListFile(root, repoName, listFile) + .forEach(records::add); + } else { + this.processZipDir(root, repoName, atVDir) + .forEach(records::add); + } + }); + } + return records.stream(); + } + + /** + * Enumerate {@code .zip} files in an {@code @v} directory that has no + * {@code list} file (proxy-cached module with no version list). + * + *

The paired {@code .info} file is used for date resolution when + * present; falls back to the zip file mtime.

+ * + * @param root Repository root + * @param repoName Logical repository name + * @param atVDir The {@code @v} directory to scan + * @return Stream of artifact records + */ + private Stream processZipDir(final Path root, + final String repoName, final Path atVDir) { + final Path moduleDir = atVDir.getParent(); + final String modulePath = root.relativize(moduleDir) + .toString().replace('\\', '/'); + final List records = new ArrayList<>(); + try (Stream dirStream = Files.list(atVDir)) { + dirStream.filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(".zip")) + .forEach(zipFile -> { + final String fname = zipFile.getFileName().toString(); + final String version = fname.substring( + 0, fname.length() - ".zip".length() + ); + if (version.isEmpty()) { + return; + } + final long createdDate = GoScanner.resolveCreatedDate( + atVDir, version, GoScanner.fileMtime(zipFile) + ); + final long size = GoScanner.resolveZipSize(atVDir, version); + final String stripped = GoScanner.stripV(version); + final String pathPrefix = this.repoType.endsWith("-proxy") + ? modulePath + "/@v/" + stripped : null; + records.add(new ArtifactRecord( + this.repoType, repoName, modulePath, stripped, + size, createdDate, null, "system", pathPrefix + )); + }); + } catch (final IOException ex) { + LOG.debug("Cannot list @v dir {}: {}", atVDir, ex.getMessage()); + } + return records.stream(); + } + + /** + * Process a single {@code @v/list} file and produce artifact records + * for every version listed inside it. + * + * @param root Repository root directory + * @param repoName Logical repository name + * @param listFile Path to the {@code @v/list} file + * @return Stream of artifact records, one per version + */ + private Stream processListFile(final Path root, + final String repoName, final Path listFile) { + final Path atVDir = listFile.getParent(); + final Path moduleDir = atVDir.getParent(); + final String modulePath = root.relativize(moduleDir).toString() + .replace('\\', '/'); + final List lines; + try { + lines = Files.readAllLines(listFile); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + final long listMtime = GoScanner.fileMtime(listFile); + final List records = new ArrayList<>(); + final boolean hasVersions = + lines.stream().anyMatch(l -> !l.trim().isEmpty()); + if (hasVersions) { + for (final String line : lines) { + final String version = line.trim(); + if (version.isEmpty()) { + continue; + } + final Path zipFile = atVDir.resolve( + String.format("%s.zip", version) + ); + if (!Files.isRegularFile(zipFile)) { + LOG.debug( + "Skipping {} {} — zip not cached", modulePath, version + ); + continue; + } + final long createdDate = GoScanner.resolveCreatedDate( + atVDir, version, listMtime + ); + final long size = GoScanner.resolveZipSize(atVDir, version); + final String stripped = GoScanner.stripV(version); + final String pathPrefix = this.repoType.endsWith("-proxy") + ? modulePath + "/@v/" + stripped : null; + records.add( + new ArtifactRecord( + this.repoType, + repoName, + modulePath, + stripped, + size, + createdDate, + null, + "system", + pathPrefix + ) + ); + } + } else { + // Empty list file — scan @v directory directly for .zip files. + // Proxy-cached modules where only a specific version was fetched + // (no list request) will have an empty list but a present .zip. + try (Stream dirStream = Files.list(atVDir)) { + dirStream.filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(".zip")) + .forEach(zipFile -> { + final String fname = zipFile.getFileName().toString(); + final String ver = fname.substring( + 0, fname.length() - ".zip".length() + ); + if (ver.isEmpty()) { + return; + } + final long createdDate = GoScanner.resolveCreatedDate( + atVDir, ver, listMtime + ); + final long size = + GoScanner.resolveZipSize(atVDir, ver); + final String stripped = GoScanner.stripV(ver); + final String pathPrefix = this.repoType.endsWith("-proxy") + ? modulePath + "/@v/" + stripped : null; + records.add(new ArtifactRecord( + this.repoType, repoName, modulePath, stripped, + size, createdDate, null, "system", pathPrefix + )); + }); + } catch (final IOException ex) { + LOG.debug( + "Cannot list @v dir {}: {}", atVDir, ex.getMessage() + ); + } + } + return records.stream(); + } + + /** + * Resolve the creation date for a version. Reads the {@code .info} JSON + * file and parses the {@code "Time"} field. Falls back to the list file + * mtime if the {@code .info} file is missing or cannot be parsed. + * + * @param atVDir Path to the {@code @v} directory + * @param version Version string (e.g. {@code v1.0.0}) + * @param fallback Fallback epoch millis (list file mtime) + * @return Epoch millis + */ + private static long resolveCreatedDate(final Path atVDir, + final String version, final long fallback) { + final Path infoFile = atVDir.resolve( + String.format("%s.info", version) + ); + if (!Files.isRegularFile(infoFile)) { + return fallback; + } + try (InputStream input = Files.newInputStream(infoFile); + JsonReader reader = Json.createReader(input)) { + final JsonObject json = reader.readObject(); + if (json.containsKey("Time") && !json.isNull("Time")) { + final String time = json.getString("Time"); + return Instant.parse(time).toEpochMilli(); + } + } catch (final JsonException ex) { + LOG.warn( + "Invalid JSON in {}: {}", infoFile, ex.getMessage() + ); + } catch (final Exception ex) { + LOG.warn( + "Cannot parse .info file {}: {}", infoFile, ex.getMessage() + ); + } + return fallback; + } + + /** + * Resolve the zip file size for a version. Returns 0 if the zip + * file does not exist. + * + * @param atVDir Path to the {@code @v} directory + * @param version Version string (e.g. {@code v1.0.0}) + * @return File size in bytes, or 0 if not found + */ + private static long resolveZipSize(final Path atVDir, + final String version) { + final Path zipFile = atVDir.resolve( + String.format("%s.zip", version) + ); + if (Files.isRegularFile(zipFile)) { + try { + return Files.size(zipFile); + } catch (final IOException ex) { + LOG.debug( + "Cannot stat zip file {}: {}", zipFile, ex.getMessage() + ); + return 0L; + } + } + return 0L; + } + + /** + * Strip the leading {@code v} prefix from a Go version string. + * The Go adapter stores versions without the {@code v} prefix + * (e.g. {@code 1.0.0} instead of {@code v1.0.0}). + * + * @param version Version string, possibly starting with "v" + * @return Version without "v" prefix + */ + private static String stripV(final String version) { + if (version.startsWith("v") || version.startsWith("V")) { + return version.substring(1); + } + return version; + } + + /** + * Get the last-modified time of a file as epoch millis. + * + * @param path Path to the file + * @return Epoch millis + */ + private static long fileMtime(final Path path) { + try { + return Files.readAttributes(path, BasicFileAttributes.class) + .lastModifiedTime().toMillis(); + } catch (final IOException ex) { + LOG.debug( + "Cannot read mtime of {}: {}", path, ex.getMessage() + ); + return System.currentTimeMillis(); + } + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/HelmScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/HelmScanner.java new file mode 100644 index 000000000..b5682f7d4 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/HelmScanner.java @@ -0,0 +1,194 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +/** + * Scanner for Helm chart repositories. + * + *

Reads {@code index.yaml} from the repository root, parses it with + * SnakeYAML, and emits one {@link ArtifactRecord} per chart version. + * The {@code .tgz} file referenced in the {@code urls} list is resolved + * relative to the root directory to determine artifact size.

+ * + * @since 1.20.13 + */ +final class HelmScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(HelmScanner.class); + + @Override + @SuppressWarnings("unchecked") + public Stream scan(final Path root, final String repoName) + throws IOException { + final Path indexPath = root.resolve("index.yaml"); + if (!Files.isRegularFile(indexPath)) { + LOG.debug("No index.yaml found in {}", root); + return Stream.empty(); + } + final Map index; + try (InputStream input = Files.newInputStream(indexPath)) { + index = new Yaml().load(input); + } + if (index == null || !index.containsKey("entries")) { + LOG.debug("No 'entries' key in index.yaml at {}", indexPath); + return Stream.empty(); + } + final Object entriesObj = index.get("entries"); + if (!(entriesObj instanceof Map)) { + LOG.warn("'entries' is not a map in {}", indexPath); + return Stream.empty(); + } + final Map entries = (Map) entriesObj; + final long indexMtime = HelmScanner.indexMtime(indexPath); + final List records = new ArrayList<>(); + for (final Map.Entry entry : entries.entrySet()) { + final String chartName = entry.getKey(); + final Object versionsObj = entry.getValue(); + if (!(versionsObj instanceof List)) { + LOG.debug("Skipping chart {} with non-list versions", chartName); + continue; + } + final List> versionsList = + (List>) versionsObj; + for (final Map versionMap : versionsList) { + if (versionMap == null) { + continue; + } + final Object versionObj = versionMap.get("version"); + if (versionObj == null) { + LOG.debug("Skipping entry in {} with null version", chartName); + continue; + } + final String version = versionObj.toString(); + final long createdDate = HelmScanner.parseCreated( + versionMap.get("created"), indexMtime + ); + final long size = HelmScanner.resolveSize( + root, versionMap.get("urls") + ); + records.add( + new ArtifactRecord( + "helm", + repoName, + chartName, + version, + size, + createdDate, + null, + "system", + null + ) + ); + } + } + return records.stream(); + } + + /** + * Parse the {@code created} field from a version map entry. + * Falls back to the index.yaml mtime if parsing fails. + * + * @param created The created field value (String, possibly ISO-8601) + * @param fallback Fallback epoch millis (index.yaml mtime) + * @return Epoch millis + */ + private static long parseCreated(final Object created, final long fallback) { + if (created == null) { + return fallback; + } + final String text = created.toString(); + if (text.isEmpty()) { + return fallback; + } + try { + return OffsetDateTime.parse(text, DateTimeFormatter.ISO_OFFSET_DATE_TIME) + .toInstant() + .toEpochMilli(); + } catch (final DateTimeParseException ex) { + LOG.debug("Cannot parse created timestamp '{}': {}", text, ex.getMessage()); + return fallback; + } + } + + /** + * Resolve the .tgz file size from the {@code urls} list. + * + * @param root Repository root directory + * @param urlsObj The urls field (expected List of String) + * @return File size in bytes, or 0 if not found + */ + @SuppressWarnings("unchecked") + private static long resolveSize(final Path root, final Object urlsObj) { + if (!(urlsObj instanceof List)) { + return 0L; + } + final List urls = (List) urlsObj; + if (urls.isEmpty()) { + return 0L; + } + final Object firstUrl = urls.get(0); + if (firstUrl == null) { + return 0L; + } + String filename = firstUrl.toString(); + if (filename.startsWith("http://") || filename.startsWith("https://")) { + try { + final String path = URI.create(filename).getPath(); + final int lastSlash = path.lastIndexOf('/'); + filename = lastSlash >= 0 ? path.substring(lastSlash + 1) : path; + } catch (final IllegalArgumentException ex) { + LOG.debug("Cannot parse URL '{}': {}", filename, ex.getMessage()); + return 0L; + } + } + final Path tgzPath = root.resolve(filename); + if (Files.isRegularFile(tgzPath)) { + try { + return Files.size(tgzPath); + } catch (final IOException ex) { + LOG.debug("Cannot stat {}: {}", tgzPath, ex.getMessage()); + return 0L; + } + } + return 0L; + } + + /** + * Get the last-modified time of the index.yaml file as epoch millis. + * + * @param indexPath Path to index.yaml + * @return Epoch millis + */ + private static long indexMtime(final Path indexPath) { + try { + return Files.readAttributes(indexPath, BasicFileAttributes.class) + .lastModifiedTime().toMillis(); + } catch (final IOException ex) { + LOG.debug("Cannot read mtime of {}: {}", indexPath, ex.getMessage()); + return System.currentTimeMillis(); + } + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/MavenScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/MavenScanner.java new file mode 100644 index 000000000..28232aa70 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/MavenScanner.java @@ -0,0 +1,194 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for Maven (and Gradle) repositories. + * + *

Walks the Maven directory structure for artifact files + * ({@code .jar}, {@code .war}, {@code .aar}, {@code .zip}, + * {@code .pom}) and + * infers groupId, artifactId, and version from the standard Maven + * directory convention:

+ * 
groupId-as-path/artifactId/version/artifactId-version.ext
+ * + *

Works for both local/hosted repos (with {@code maven-metadata.xml}) + * and proxy/cache repos (without metadata). Sidecar files + * ({@code .sha1}, {@code .md5}, {@code .json}, etc.) are skipped. + * When multiple files share the same GAV (e.g. {@code .jar} + {@code .pom}), + * only one record is emitted using the largest file size.

+ * + * @since 1.20.13 + */ +final class MavenScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = LoggerFactory.getLogger(MavenScanner.class); + + /** + * Artifact file extensions to match, in priority order. + */ + private static final List EXTENSIONS = Arrays.asList( + ".jar", ".war", ".aar", ".zip", ".pom", ".module" + ); + + /** + * Repository type identifier. + */ + private final String repoType; + + /** + * Ctor. + * + * @param repoType Repository type ("maven" or "gradle") + */ + MavenScanner(final String repoType) { + this.repoType = repoType; + } + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + final Map dedup = new HashMap<>(); + // Tracks which keys already have a non-POM (binary) artifact winner. + final Set hasBinary = new HashSet<>(); + try (Stream walk = Files.walk(root)) { + walk.filter(Files::isRegularFile) + .filter(MavenScanner::isMavenArtifact) + .forEach(path -> { + final ArtifactRecord record = this.parseFromPath( + root, repoName, path + ); + if (record != null) { + final String key = String.format( + "%s:%s", record.name(), record.version() + ); + final String fname = path.getFileName().toString(); + final boolean incoming = !fname.endsWith(".pom") + && !fname.endsWith(".module"); + if (!dedup.containsKey(key)) { + dedup.put(key, record); + if (incoming) { + hasBinary.add(key); + } + } else if (incoming && !hasBinary.contains(key)) { + // Replace POM-only entry with binary + dedup.put(key, record); + hasBinary.add(key); + } else if (incoming + && record.size() > dedup.get(key).size()) { + // Both binary — keep the larger one + dedup.put(key, record); + } + // POM incoming when binary already exists: ignored + } + }); + } + return dedup.values().stream(); + } + + /** + * Check if a file is a Maven artifact (not a sidecar/metadata file). + * + * @param path File path to check + * @return True if the file is a Maven artifact + */ + private static boolean isMavenArtifact(final Path path) { + final String name = path.getFileName().toString(); + if (name.startsWith(".")) { + return false; + } + if (name.endsWith(".md5") || name.endsWith(".sha1") + || name.endsWith(".sha256") || name.endsWith(".sha512") + || name.endsWith(".asc") || name.endsWith(".sig") + || name.endsWith(".json") || name.endsWith(".xml")) { + return false; + } + for (final String ext : EXTENSIONS) { + if (name.endsWith(ext)) { + return true; + } + } + return false; + } + + /** + * Parse an artifact record from the Maven directory structure. + * Path convention: root/groupId-parts/artifactId/version/file.ext + * + * @param root Repository root + * @param repoName Logical repository name + * @param path Path to the artifact file + * @return Artifact record, or null if path structure is invalid + */ + private ArtifactRecord parseFromPath(final Path root, + final String repoName, final Path path) { + final Path relative = root.relativize(path); + final int count = relative.getNameCount(); + // Need at least: groupId-part / artifactId / version / file + if (count < 4) { + LOG.debug("Path too short for Maven layout: {}", relative); + return null; + } + final String version = relative.getName(count - 2).toString(); + final String artifactId = relative.getName(count - 3).toString(); + final StringBuilder groupBuilder = new StringBuilder(); + for (int idx = 0; idx < count - 3; idx++) { + if (idx > 0) { + groupBuilder.append('.'); + } + groupBuilder.append(relative.getName(idx).toString()); + } + final String groupId = groupBuilder.toString(); + if (groupId.isEmpty()) { + return null; + } + final char sep = this.repoType.startsWith("gradle") ? ':' : '.'; + final String name = groupId + sep + artifactId; + long size = 0L; + long mtime; + try { + final BasicFileAttributes attrs = Files.readAttributes( + path, BasicFileAttributes.class + ); + size = attrs.size(); + mtime = attrs.lastModifiedTime().toMillis(); + } catch (final IOException ex) { + mtime = System.currentTimeMillis(); + } + // Proxy repos need the directory path for artifact lookup; + // local/hosted repos use NULL (no prefix stored in production). + final String pathPrefix; + if (this.repoType.endsWith("-proxy")) { + final Path relParent = relative.getParent(); + pathPrefix = relParent != null + ? relParent.toString().replace('\\', '/') : null; + } else { + pathPrefix = null; + } + final Long releaseDate = + ArtipieMetaSidecar.readReleaseDate(path).orElse(null); + return new ArtifactRecord( + this.repoType, repoName, name, version, + size, mtime, releaseDate, "system", pathPrefix + ); + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/NpmScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/NpmScanner.java new file mode 100644 index 000000000..db595ce6d --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/NpmScanner.java @@ -0,0 +1,517 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import javax.json.Json; +import javax.json.JsonException; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonString; +import javax.json.JsonValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for NPM repositories. + * + *

Supports two scanning modes:

+ *
    + *
  • Versions-directory mode (primary): Walks for + * {@code .versions/} directories, reads version JSON files, + * resolves tarball sizes from sibling {@code -/} directories. + * Used for real Artipie NPM storage layout.
    • + *
  • Meta.json mode (fallback): Walks for {@code meta.json} + * files, parses them to extract package name, versions, tarball + * sizes, and creation dates. Used for legacy/proxy layouts.
    • + *
+ * + * @since 1.20.13 + */ +final class NpmScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(NpmScanner.class); + + /** + * Whether this is a proxy repository (used to determine path_prefix). + * Artipie production always stores {@code "npm"} as repo_type for both + * local and proxy NPM repositories; the proxy/local distinction is + * captured solely through whether {@code path_prefix} is NULL or not. + */ + private final boolean proxyMode; + + /** + * Ctor for a local (hosted) NPM repository. + */ + NpmScanner() { + this(false); + } + + /** + * Ctor. + * + * @param proxyMode True if this is an npm-proxy repository + */ + NpmScanner(final boolean proxyMode) { + this.proxyMode = proxyMode; + } + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + final boolean hasVersionsDirs; + try (Stream walk = Files.walk(root)) { + hasVersionsDirs = walk + .filter(Files::isDirectory) + .anyMatch( + p -> ".versions".equals( + p.getFileName().toString() + ) + ); + } + if (hasVersionsDirs) { + return this.scanVersionsDirs(root, repoName); + } + LOG.info( + "No .versions directories found, falling back to meta.json mode" + ); + return this.scanMetaJson(root, repoName); + } + + /** + * Scan using .versions/ directories (Artipie NPM layout). + * + *

Layout for unscoped packages:

+ * 
+     * package-name/
+     *   .versions/
+     *     1.0.0.json
+     *     1.0.1.json
+     *   -/
+     *     @scope/
+     *       package-name-1.0.0.tgz
+     *
+ * + *

Layout for scoped packages:

+ * 
+     * @scope/
+     *   package-name/
+     *     .versions/
+     *       1.0.0.json
+     *     -/
+     *       @scope/
+     *         package-name-1.0.0.tgz
+     *
+ * + * @param root Repository root + * @param repoName Logical repository name + * @return Stream of artifact records + * @throws IOException If an I/O error occurs + */ + private Stream scanVersionsDirs(final Path root, + final String repoName) throws IOException { + final List records = new ArrayList<>(); + try (Stream walk = Files.walk(root)) { + walk.filter(Files::isDirectory) + .filter( + p -> ".versions".equals(p.getFileName().toString()) + ) + .forEach(versionsDir -> { + final Path packageDir = versionsDir.getParent(); + if (packageDir == null) { + return; + } + final String packageName = NpmScanner.resolvePackageName( + root, packageDir + ); + if (packageName.isEmpty()) { + return; + } + try (Stream files = Files.list(versionsDir)) { + files.filter(Files::isRegularFile) + .filter( + f -> f.getFileName().toString() + .endsWith(".json") + ) + .forEach(jsonFile -> { + final String fname = + jsonFile.getFileName().toString(); + final String version = fname.substring( + 0, fname.length() - ".json".length() + ); + if (version.isEmpty()) { + return; + } + final Optional tgzOpt = + NpmScanner.findTgzFile( + packageDir, packageName, version + ); + final long size = tgzOpt.map( + p -> { + try { + return Files.size(p); + } catch (final IOException ex) { + return 0L; + } + } + ).orElse(0L); + final String pathPrefix = + this.proxyMode + ? tgzOpt.map( + p -> root.relativize(p) + .toString().replace('\\', '/') + ).orElse(null) : null; + final Long releaseDate = tgzOpt + .flatMap(NpmScanner::readNpmReleaseDate) + .orElse(null); + final long mtime = + NpmScanner.fileMtime(jsonFile); + records.add( + new ArtifactRecord( + "npm", + repoName, + packageName, + version, + size, + mtime, + releaseDate, + "system", + pathPrefix + ) + ); + }); + } catch (final IOException ex) { + LOG.warn( + "Cannot list .versions dir {}: {}", + versionsDir, ex.getMessage() + ); + } + }); + } + return records.stream(); + } + + /** + * Resolve the NPM package name from the directory structure. + * For scoped packages (@scope/name), the parent of packageDir + * starts with {@code @}. For unscoped packages, packageDir + * name is the full package name. + * + * @param root Repository root + * @param packageDir Directory containing .versions/ + * @return Package name (e.g., "lodash" or "@scope/button") + */ + private static String resolvePackageName(final Path root, + final Path packageDir) { + final Path relative = root.relativize(packageDir); + final int count = relative.getNameCount(); + if (count == 0) { + return ""; + } + final String dirName = relative.getName(count - 1).toString(); + if (count >= 2) { + final String parentName = + relative.getName(count - 2).toString(); + if (parentName.startsWith("@")) { + return parentName + "/" + dirName; + } + } + return dirName; + } + + /** + * Find the tarball file for a given package version by searching + * the {@code -/} subdirectory tree for a matching {@code .tgz} file. + * + * @param packageDir Package directory (parent of .versions/) + * @param packageName Full package name + * @param version Version string + * @return Optional path to the tgz file, empty if not found + */ + private static Optional findTgzFile(final Path packageDir, + final String packageName, final String version) { + final Path dashDir = packageDir.resolve("-"); + if (!Files.isDirectory(dashDir)) { + return Optional.empty(); + } + final String artifactName; + final int slash = packageName.indexOf('/'); + if (slash >= 0) { + artifactName = packageName.substring(slash + 1); + } else { + artifactName = packageName; + } + final String tgzName = artifactName + "-" + version + ".tgz"; + try (Stream walk = Files.walk(dashDir)) { + return walk.filter(Files::isRegularFile) + .filter(p -> tgzName.equals(p.getFileName().toString())) + .findFirst(); + } catch (final IOException ex) { + LOG.debug( + "Cannot walk dash dir {}: {}", dashDir, ex.getMessage() + ); + return Optional.empty(); + } + } + + /** + * Read the NPM release date from a tgz sidecar {@code .meta} file. + * + *

Artipie NPM proxy stores metadata alongside each cached tgz as + * {@code {path}.meta}, a JSON file containing: + * {@code {"last-modified":"RFC_1123_DATE","content-type":"..."}}. + * The {@code last-modified} value is the {@code Last-Modified} HTTP + * response header from the upstream NPM registry, which is the + * package publish date — and the source of {@code release_date} in + * production (via {@code NpmProxyPackageProcessor.releaseMillis()}). + *

+ * + * @param tgzPath Path to the {@code .tgz} file + * @return Optional epoch millis, empty if sidecar is absent or unparseable + */ + private static Optional readNpmReleaseDate(final Path tgzPath) { + final Path metaPath = tgzPath.getParent() + .resolve(tgzPath.getFileName().toString() + ".meta"); + if (!Files.isRegularFile(metaPath)) { + return Optional.empty(); + } + try (InputStream input = Files.newInputStream(metaPath); + JsonReader reader = Json.createReader(input)) { + final JsonObject json = reader.readObject(); + if (!json.containsKey("last-modified") + || json.isNull("last-modified")) { + return Optional.empty(); + } + final String lm = json.getString("last-modified"); + return Optional.of( + Instant.from( + DateTimeFormatter.RFC_1123_DATE_TIME.parse(lm) + ).toEpochMilli() + ); + } catch (final IOException | JsonException | DateTimeParseException ex) { + LOG.debug( + "Cannot read NPM release date from {}: {}", + metaPath, ex.getMessage() + ); + return Optional.empty(); + } + } + + /** + * Scan using meta.json files (legacy/fallback mode). + * + * @param root Repository root + * @param repoName Logical repository name + * @return Stream of artifact records + * @throws IOException If an I/O error occurs + */ + private Stream scanMetaJson(final Path root, + final String repoName) throws IOException { + return Files.walk(root) + .filter(Files::isRegularFile) + .filter( + path -> "meta.json".equals(path.getFileName().toString()) + ) + .flatMap(path -> this.parseMetaJson(root, repoName, path)); + } + + /** + * Parse a single meta.json file and produce artifact records. + * + * @param root Repository root directory + * @param repoName Logical repository name + * @param metaPath Path to the meta.json file + * @return Stream of artifact records, one per version + */ + private Stream parseMetaJson(final Path root, + final String repoName, final Path metaPath) { + final JsonObject json; + try (InputStream input = Files.newInputStream(metaPath); + JsonReader reader = Json.createReader(input)) { + json = reader.readObject(); + } catch (final JsonException ex) { + LOG.warn("Malformed JSON in {}: {}", metaPath, ex.getMessage()); + return Stream.empty(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + if (!json.containsKey("name") + || json.isNull("name")) { + LOG.warn("Missing 'name' field in {}", metaPath); + return Stream.empty(); + } + final String packageName = json.getString("name"); + if (!json.containsKey("versions") + || json.isNull("versions") + || json.get("versions").getValueType() + != JsonValue.ValueType.OBJECT) { + LOG.warn( + "Missing or invalid 'versions' field in {}", metaPath + ); + return Stream.empty(); + } + final JsonObject versions = json.getJsonObject("versions"); + final JsonObject time = json.containsKey("time") + && !json.isNull("time") + && json.get("time").getValueType() + == JsonValue.ValueType.OBJECT + ? json.getJsonObject("time") : null; + final long metaMtime; + try { + metaMtime = Files.readAttributes( + metaPath, BasicFileAttributes.class + ).lastModifiedTime().toMillis(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + final List records = new ArrayList<>(); + for (final String version : versions.keySet()) { + final Optional tarball = this.resolveMetaTarball( + root, metaPath, versions.getJsonObject(version) + ); + if (tarball.isEmpty()) { + LOG.debug( + "Skipping {} {} — tarball not cached", packageName, version + ); + continue; + } + final long size; + try { + size = Files.size(tarball.get()); + } catch (final IOException ex) { + LOG.debug( + "Cannot stat tarball {}: {}", + tarball.get(), ex.getMessage() + ); + continue; + } + final String pathPrefix = this.proxyMode + ? root.relativize(tarball.get()).toString().replace('\\', '/') + : null; + final Long releaseDate = + NpmScanner.readNpmReleaseDate(tarball.get()).orElse(null); + final long createdDate = NpmScanner.resolveCreatedDate( + time, version, metaMtime + ); + records.add( + new ArtifactRecord( + "npm", + repoName, + packageName, + version, + size, + createdDate, + releaseDate, + "system", + pathPrefix + ) + ); + } + return records.stream(); + } + + /** + * Resolve the tarball path for a version entry from meta.json. + * + * @param root Repository root directory + * @param metaPath Path to the meta.json file + * @param versionObj Version metadata JSON object + * @return Optional path to the tarball file, empty if not found + */ + private Optional resolveMetaTarball(final Path root, + final Path metaPath, final JsonObject versionObj) { + if (!versionObj.containsKey("dist") + || versionObj.isNull("dist") + || versionObj.get("dist").getValueType() + != JsonValue.ValueType.OBJECT) { + return Optional.empty(); + } + final JsonObject dist = versionObj.getJsonObject("dist"); + if (!dist.containsKey("tarball") + || dist.isNull("tarball") + || dist.get("tarball").getValueType() + != JsonValue.ValueType.STRING) { + return Optional.empty(); + } + final String tarball = + ((JsonString) dist.get("tarball")).getString(); + final String stripped = tarball.startsWith("/") + ? tarball.substring(1) : tarball; + final Path resolved = root.resolve(stripped); + if (Files.isRegularFile(resolved)) { + return Optional.of(resolved); + } + final Path filename = resolved.getFileName(); + if (filename != null) { + final Path fallback = + metaPath.getParent().resolve(filename.toString()); + if (Files.isRegularFile(fallback)) { + return Optional.of(fallback); + } + } + return Optional.empty(); + } + + /** + * Resolve the created date for a version from meta.json time field. + * + * @param time The "time" JSON object from the root, or null + * @param version Version string to look up + * @param metaMtime Meta.json file last-modified time in epoch millis + * @return Created date as epoch millis + */ + private static long resolveCreatedDate(final JsonObject time, + final String version, final long metaMtime) { + if (time != null && time.containsKey(version) + && !time.isNull(version) + && time.get(version).getValueType() + == JsonValue.ValueType.STRING) { + try { + final String iso = time.getString(version); + return Instant.parse(iso).toEpochMilli(); + } catch (final Exception ex) { + LOG.debug( + "Cannot parse time for version {}: {}", + version, ex.getMessage() + ); + } + } + return metaMtime; + } + + /** + * Get the last-modified time of a file as epoch millis. + * + * @param path Path to the file + * @return Epoch millis + */ + private static long fileMtime(final Path path) { + try { + return Files.readAttributes(path, BasicFileAttributes.class) + .lastModifiedTime().toMillis(); + } catch (final IOException ex) { + LOG.debug( + "Cannot read mtime of {}: {}", path, ex.getMessage() + ); + return System.currentTimeMillis(); + } + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/ProgressReporter.java b/artipie-backfill/src/main/java/com/artipie/backfill/ProgressReporter.java new file mode 100644 index 000000000..611910deb --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/ProgressReporter.java @@ -0,0 +1,114 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Thread-safe progress reporter that tracks scanned records and errors, + * logging throughput statistics at a configurable interval. + * + * @since 1.20.13 + */ +public final class ProgressReporter { + + /** + * SLF4J logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(ProgressReporter.class); + + /** + * Number of records between periodic log messages. + */ + private final int logInterval; + + /** + * Total records scanned / processed. + */ + private final AtomicLong scanned; + + /** + * Total errors encountered. + */ + private final AtomicLong errors; + + /** + * Timestamp (epoch millis) when this reporter was created. + */ + private final long startTime; + + /** + * Ctor. + * + * @param logInterval Log progress every N records + */ + public ProgressReporter(final int logInterval) { + this.logInterval = logInterval; + this.scanned = new AtomicLong(0L); + this.errors = new AtomicLong(0L); + this.startTime = System.currentTimeMillis(); + } + + /** + * Increment the scanned counter. Every {@code logInterval} records a + * progress line with throughput (records/sec) is logged. + */ + public void increment() { + final long count = this.scanned.incrementAndGet(); + if (count % this.logInterval == 0) { + final long elapsed = System.currentTimeMillis() - this.startTime; + final double secs = elapsed / 1_000.0; + final double throughput = secs > 0 ? count / secs : 0; + LOG.info("Progress: {} records scanned ({} errors) — {}/sec", + count, this.errors.get(), + String.format("%.1f", throughput)); + } + } + + /** + * Record an error. + */ + public void recordError() { + this.errors.incrementAndGet(); + } + + /** + * Return the current scanned count. + * + * @return Number of records scanned so far + */ + public long getScanned() { + return this.scanned.get(); + } + + /** + * Return the current error count. + * + * @return Number of errors recorded so far + */ + public long getErrors() { + return this.errors.get(); + } + + /** + * Log final summary with total scanned, errors, elapsed time, and + * overall throughput. + */ + public void printFinalSummary() { + final long elapsed = System.currentTimeMillis() - this.startTime; + final double secs = elapsed / 1_000.0; + final long total = this.scanned.get(); + final double throughput = secs > 0 ? total / secs : 0; + LOG.info("=== Backfill Summary ==="); + LOG.info("Total scanned : {}", total); + LOG.info("Total errors : {}", this.errors.get()); + LOG.info("Elapsed time : {}s", String.format("%.1f", secs)); + LOG.info("Throughput : {}/sec", + String.format("%.1f", throughput)); + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/PypiScanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/PypiScanner.java new file mode 100644 index 000000000..486c61572 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/PypiScanner.java @@ -0,0 +1,170 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scanner for PyPI repositories. + * + *

Walks the repository directory tree up to depth 2 + * (package-dir/filename), filters for recognized Python distribution + * file extensions ({@code .whl}, {@code .tar.gz}, {@code .zip}, + * {@code .egg}), and regex-parses each filename to extract the + * package name and version. Package names are normalized per PEP 503 + * (lowercase, consecutive {@code [-_.]} replaced with a single + * {@code -}).

+ * + * @since 1.20.13 + */ +final class PypiScanner implements Scanner { + + /** + * Logger. + */ + private static final Logger LOG = + LoggerFactory.getLogger(PypiScanner.class); + + /** + * Repository type string stored in every produced artifact record + * (e.g. {@code "pypi"} or {@code "pypi-proxy"}). + */ + private final String repoType; + + /** + * Ctor with default repo type {@code "pypi"}. + */ + PypiScanner() { + this("pypi"); + } + + /** + * Ctor. + * + * @param repoType Repository type string for artifact records + */ + PypiScanner(final String repoType) { + this.repoType = repoType; + } + + /** + * Pattern for wheel filenames. + * Example: my_package-1.0.0-py3-none-any.whl + */ + private static final Pattern WHEEL = Pattern.compile( + "(?[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?)-(?[0-9][A-Za-z0-9.!+_-]*?)(-\\d+)?-[A-Za-z0-9._]+-[A-Za-z0-9._]+-[A-Za-z0-9._]+\\.whl" + ); + + /** + * Pattern for sdist filenames (tar.gz, zip, egg). + * Example: requests-2.28.0.tar.gz + */ + private static final Pattern SDIST = Pattern.compile( + "(?[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?)-(?[0-9][A-Za-z0-9.!+_-]*)\\.(tar\\.gz|zip|egg)" + ); + + @Override + public Stream scan(final Path root, final String repoName) + throws IOException { + return Files.walk(root) + .filter(Files::isRegularFile) + .filter(path -> { + for (Path part : root.relativize(path)) { + if (part.toString().startsWith(".")) { + return false; + } + } + return true; + }) + .filter(PypiScanner::hasRecognizedExtension) + .flatMap(path -> this.tryParse(root, repoName, path)); + } + + /** + * Attempt to parse a file path into an artifact record. + * + * @param root Repository root directory + * @param repoName Logical repository name + * @param path File path to parse + * @return Stream with a single record, or empty if filename does not match + */ + private Stream tryParse(final Path root, + final String repoName, final Path path) { + final String filename = path.getFileName().toString(); + Matcher matcher = WHEEL.matcher(filename); + if (!matcher.matches()) { + matcher = SDIST.matcher(filename); + } + if (!matcher.matches()) { + LOG.debug( + "Skipping non-conforming filename: {}", filename + ); + return Stream.empty(); + } + final String name = normalizePep503(matcher.group("name")); + final String version = matcher.group("version"); + try { + final BasicFileAttributes attrs = Files.readAttributes( + path, BasicFileAttributes.class + ); + // Proxy repos need the full relative path for artifact lookup; + // local/hosted repos use NULL (no prefix stored in production). + final String pathPrefix = this.repoType.endsWith("-proxy") + ? root.relativize(path).toString().replace('\\', '/') : null; + final Long releaseDate = + ArtipieMetaSidecar.readReleaseDate(path).orElse(null); + return Stream.of( + new ArtifactRecord( + this.repoType, + repoName, + name, + version, + attrs.size(), + attrs.lastModifiedTime().toMillis(), + releaseDate, + "system", + pathPrefix + ) + ); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + } + + /** + * Check whether a file path has a recognized Python distribution extension. + * + * @param path File path to check + * @return True if the file has a recognized extension + */ + private static boolean hasRecognizedExtension(final Path path) { + final String name = path.getFileName().toString().toLowerCase(Locale.ROOT); + return name.endsWith(".whl") + || name.endsWith(".tar.gz") + || name.endsWith(".zip") + || name.endsWith(".egg"); + } + + /** + * Normalize a package name per PEP 503: lowercase and replace + * consecutive runs of {@code [-_.]} with a single hyphen. + * + * @param name Raw package name + * @return Normalized name + */ + private static String normalizePep503(final String name) { + return name.toLowerCase(Locale.ROOT).replaceAll("[-_.]+", "-"); + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/RepoConfigYaml.java b/artipie-backfill/src/main/java/com/artipie/backfill/RepoConfigYaml.java new file mode 100644 index 000000000..bc4e92750 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/RepoConfigYaml.java @@ -0,0 +1,81 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import org.yaml.snakeyaml.Yaml; + +/** + * Parses one Artipie YAML repo config file into a {@link RepoEntry}. + * + *

Expected minimal YAML structure: + * 

+ * repo:
+ *   type: docker
+ *
+ * Additional fields (storage, remotes, url, etc.) are ignored. + *

+ * + * @since 1.20.13 + */ +final class RepoConfigYaml { + + /** + * Private ctor — utility class, not instantiable. + */ + private RepoConfigYaml() { + } + + /** + * Parse a single {@code .yaml} Artipie repo config file. + * + * @param file Path to the {@code .yaml} file + * @return Parsed {@link RepoEntry} with repo name (filename stem) and raw type + * @throws IOException if the file is unreadable, YAML is malformed, + * or {@code repo.type} is missing + */ + @SuppressWarnings("unchecked") + static RepoEntry parse(final Path file) throws IOException { + final String filename = file.getFileName().toString(); + final String repoName; + if (filename.endsWith(".yaml")) { + repoName = filename.substring(0, filename.length() - ".yaml".length()); + } else { + repoName = filename; + } + final Map doc; + try (InputStream in = Files.newInputStream(file)) { + doc = new Yaml().load(in); + } catch (final Exception ex) { + throw new IOException( + String.format("Failed to parse YAML in '%s': %s", filename, ex.getMessage()), + ex + ); + } + if (doc == null) { + throw new IOException( + String.format("Empty YAML file: '%s'", filename) + ); + } + final Object repoObj = doc.get("repo"); + if (!(repoObj instanceof Map)) { + throw new IOException( + String.format("Missing or invalid 'repo' key in '%s'", filename) + ); + } + final Map repo = (Map) repoObj; + final Object typeObj = repo.get("type"); + if (typeObj == null) { + throw new IOException( + String.format("Missing 'repo.type' in '%s'", filename) + ); + } + return new RepoEntry(repoName, typeObj.toString()); + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/RepoEntry.java b/artipie-backfill/src/main/java/com/artipie/backfill/RepoEntry.java new file mode 100644 index 000000000..867eadcd4 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/RepoEntry.java @@ -0,0 +1,15 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +/** + * Parsed result of one Artipie repo YAML config file. + * + * @param repoName Repo name derived from the YAML filename stem (e.g. {@code go.yaml} → {@code go}) + * @param rawType Raw {@code repo.type} string from the YAML (e.g. {@code docker-proxy}) + * @since 1.20.13 + */ +record RepoEntry(String repoName, String rawType) { +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/RepoTypeNormalizer.java b/artipie-backfill/src/main/java/com/artipie/backfill/RepoTypeNormalizer.java new file mode 100644 index 000000000..db980841c --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/RepoTypeNormalizer.java @@ -0,0 +1,39 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +/** + * Normalises raw Artipie repo type strings to scanner type keys + * understood by {@link ScannerFactory}. + * + *

Currently only strips the {@code -proxy} suffix + * (e.g. {@code docker-proxy} → {@code docker}). + * Other compound suffixes (e.g. {@code -hosted}, {@code -group}) are out of + * scope and will surface as unknown types in {@link ScannerFactory}.

+ * + * @since 1.20.13 + */ +final class RepoTypeNormalizer { + + /** + * Private ctor — utility class, not instantiable. + */ + private RepoTypeNormalizer() { + } + + /** + * Normalize a raw repo type by stripping the {@code -proxy} suffix. + * + * @param rawType Raw {@code repo.type} value from the YAML config + * @return Normalised scanner type string + */ + static String normalize(final String rawType) { + final String suffix = "-proxy"; + if (rawType.endsWith(suffix)) { + return rawType.substring(0, rawType.length() - suffix.length()); + } + return rawType; + } +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/Scanner.java b/artipie-backfill/src/main/java/com/artipie/backfill/Scanner.java new file mode 100644 index 000000000..bc528f9f5 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/Scanner.java @@ -0,0 +1,31 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.stream.Stream; + +/** + * Scans a repository root directory and produces a lazy stream of + * {@link ArtifactRecord} instances. Implementations must ensure the + * returned stream is lazy so that arbitrarily large repositories can + * be processed with constant memory. + * + * @since 1.20.13 + */ +@FunctionalInterface +public interface Scanner { + + /** + * Scan the given repository root and produce artifact records. + * + * @param root Path to the repository root directory on disk + * @param repoName Logical repository name + * @return Lazy stream of artifact records + * @throws IOException If an I/O error occurs while scanning + */ + Stream scan(Path root, String repoName) throws IOException; +} diff --git a/artipie-backfill/src/main/java/com/artipie/backfill/ScannerFactory.java b/artipie-backfill/src/main/java/com/artipie/backfill/ScannerFactory.java new file mode 100644 index 000000000..99b74cc62 --- /dev/null +++ b/artipie-backfill/src/main/java/com/artipie/backfill/ScannerFactory.java @@ -0,0 +1,97 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +/** + * Factory that maps repository type strings to {@link Scanner} implementations. + * + * @since 1.20.13 + */ +public final class ScannerFactory { + + /** + * Private ctor to prevent instantiation. + */ + private ScannerFactory() { + } + + /** + * Create a scanner for the given repository type. + * + *

Accepts both plain types (e.g. {@code "maven"}) and proxy variants + * (e.g. {@code "maven-proxy"}). The raw type string is passed through to + * the scanner so that the correct {@code repo_type} value is stored in + * the database (matching production).

+ * + * @param type Repository type string, raw from YAML + * (e.g. "maven", "docker-proxy", "php") + * @return Scanner implementation for the given type + * @throws IllegalArgumentException If the type is not recognized + */ + public static Scanner create(final String type) { + final String lower = type.toLowerCase(java.util.Locale.ROOT); + final Scanner scanner; + switch (lower) { + case "maven": + case "maven-proxy": + scanner = new MavenScanner(lower); + break; + case "gradle": + case "gradle-proxy": + scanner = new MavenScanner(lower); + break; + case "docker": + scanner = new DockerScanner(lower, false); + break; + case "docker-proxy": + scanner = new DockerScanner(lower, true); + break; + case "npm": + scanner = new NpmScanner(false); + break; + case "npm-proxy": + scanner = new NpmScanner(true); + break; + case "pypi": + case "pypi-proxy": + scanner = new PypiScanner(lower); + break; + case "go": + case "go-proxy": + scanner = new GoScanner(lower); + break; + case "helm": + case "helm-proxy": + scanner = new HelmScanner(); + break; + case "composer": + case "composer-proxy": + case "php": + case "php-proxy": + scanner = new ComposerScanner(lower); + break; + case "file": + case "file-proxy": + scanner = new FileScanner(lower); + break; + case "deb": + case "deb-proxy": + case "debian": + case "debian-proxy": + scanner = new DebianScanner(); + break; + case "gem": + case "gem-proxy": + case "gems": + scanner = new GemScanner(); + break; + default: + throw new IllegalArgumentException( + String.format("Unknown repository type: %s", type) + ); + } + return scanner; + } +} diff --git a/artipie-backfill/src/main/resources/META-INF/services/org.apache.logging.log4j.spi.Provider b/artipie-backfill/src/main/resources/META-INF/services/org.apache.logging.log4j.spi.Provider new file mode 100644 index 000000000..f2a6da017 --- /dev/null +++ b/artipie-backfill/src/main/resources/META-INF/services/org.apache.logging.log4j.spi.Provider @@ -0,0 +1 @@ +org.apache.logging.log4j.core.impl.Log4jProvider diff --git a/artipie-backfill/src/main/resources/log4j2.xml b/artipie-backfill/src/main/resources/log4j2.xml new file mode 100644 index 000000000..9a4a05422 --- /dev/null +++ b/artipie-backfill/src/main/resources/log4j2.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/BackfillCliTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/BackfillCliTest.java new file mode 100644 index 000000000..86a3cd95f --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/BackfillCliTest.java @@ -0,0 +1,273 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link BackfillCli}. + * + *

All tests exercise the {@code run()} method which returns an + * exit code (0 = success, 1 = error) instead of calling + * {@code System.exit()}.

+ * + * @since 1.20.13 + */ +final class BackfillCliTest { + + /** + * Dry-run with a file scanner should succeed (exit code 0) and + * process all non-hidden regular files in the temp directory. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + void dryRunWithFileScanner(@TempDir final Path tmp) throws IOException { + Files.createFile(tmp.resolve("file1.txt")); + Files.write(tmp.resolve("file2.dat"), new byte[]{1, 2, 3}); + Files.createFile(tmp.resolve(".hidden")); + final int code = BackfillCli.run( + "--type", "file", + "--path", tmp.toString(), + "--repo-name", "test", + "--dry-run" + ); + MatcherAssert.assertThat( + "Dry-run with file scanner should succeed", + code, + Matchers.is(0) + ); + } + + /** + * Running with no arguments should fail (exit code 1) because + * required options are missing. + */ + @Test + void missingRequiredArgs() { + final int code = BackfillCli.run(); + MatcherAssert.assertThat( + "Missing required args should return exit code 1", + code, + Matchers.is(1) + ); + } + + /** + * Running with a non-existent path should fail (exit code 1). + */ + @Test + void invalidPath() { + final int code = BackfillCli.run( + "--type", "file", + "--path", "/nonexistent/directory/that/does/not/exist", + "--repo-name", "test", + "--dry-run" + ); + MatcherAssert.assertThat( + "Non-existent path should return exit code 1", + code, + Matchers.is(1) + ); + } + + /** + * Running with an unknown scanner type should fail (exit code 1). + * + * @param tmp Temporary directory created by JUnit + */ + @Test + void invalidType(@TempDir final Path tmp) { + final int code = BackfillCli.run( + "--type", "unknown_type_xyz", + "--path", tmp.toString(), + "--repo-name", "test", + "--dry-run" + ); + MatcherAssert.assertThat( + "Unknown scanner type should return exit code 1", + code, + Matchers.is(1) + ); + } + + /** + * Running with --help should succeed (exit code 0). + */ + @Test + void helpFlag() { + final int code = BackfillCli.run("--help"); + MatcherAssert.assertThat( + "Help flag should return exit code 0", + code, + Matchers.is(0) + ); + } + + /** + * Running without --db-url and without --dry-run should fail + * (exit code 1) because the database URL is required for real runs. + * + * @param tmp Temporary directory created by JUnit + */ + @Test + void dbUrlRequiredWithoutDryRun(@TempDir final Path tmp) { + final int code = BackfillCli.run( + "--type", "file", + "--path", tmp.toString(), + "--repo-name", "test" + ); + MatcherAssert.assertThat( + "Missing --db-url without --dry-run should return exit code 1", + code, + Matchers.is(1) + ); + } + + /** + * Dry-run with nested directories should process files recursively + * and skip hidden files. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + void dryRunWithNestedDirectories(@TempDir final Path tmp) + throws IOException { + final Path sub = tmp.resolve("subdir"); + Files.createDirectory(sub); + Files.createFile(tmp.resolve("root-file.txt")); + Files.createFile(sub.resolve("nested-file.txt")); + Files.createFile(sub.resolve(".hidden-nested")); + final int code = BackfillCli.run( + "--type", "file", + "--path", tmp.toString(), + "--repo-name", "nested-test", + "--dry-run" + ); + MatcherAssert.assertThat( + "Dry-run with nested directories should succeed", + code, + Matchers.is(0) + ); + } + + /** + * --config-dir without --storage-root should fail (exit code 1). + * + * @param tmp JUnit temp directory + * @throws IOException if directory setup fails + */ + @Test + void configDirWithoutStorageRootFails(@TempDir final Path tmp) + throws IOException { + Files.createDirectories(tmp); + final int code = BackfillCli.run( + "--config-dir", tmp.toString(), + "--dry-run" + ); + MatcherAssert.assertThat( + "--config-dir without --storage-root should return exit code 1", + code, + Matchers.is(1) + ); + } + + /** + * --storage-root without --config-dir should fail (exit code 1). + * + * @param tmp JUnit temp directory + * @throws IOException if directory setup fails + */ + @Test + void storageRootWithoutConfigDirFails(@TempDir final Path tmp) + throws IOException { + Files.createDirectories(tmp); + final int code = BackfillCli.run( + "--storage-root", tmp.toString(), + "--dry-run" + ); + MatcherAssert.assertThat( + "--storage-root without --config-dir should return exit code 1", + code, + Matchers.is(1) + ); + } + + /** + * --config-dir combined with --type should fail (mutually exclusive). + * + * @param tmp JUnit temp directory + * @throws IOException if directory setup fails + */ + @Test + void configDirAndTypeTogether(@TempDir final Path tmp) throws IOException { + Files.createDirectories(tmp); + final int code = BackfillCli.run( + "--config-dir", tmp.toString(), + "--storage-root", tmp.toString(), + "--type", "file", + "--dry-run" + ); + MatcherAssert.assertThat( + "--config-dir and --type together should return exit code 1", + code, + Matchers.is(1) + ); + } + + /** + * Valid --config-dir + --storage-root in dry-run mode → exit code 0. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void bulkModeWithConfigDirSucceeds(@TempDir final Path tmp) + throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + Files.createDirectories(storageRoot); + Files.writeString(configDir.resolve("myrepo.yaml"), "repo:\n type: file\n"); + Files.createDirectories(storageRoot.resolve("myrepo")); + Files.writeString(storageRoot.resolve("myrepo").resolve("f.txt"), "hi"); + final int code = BackfillCli.run( + "--config-dir", configDir.toString(), + "--storage-root", storageRoot.toString(), + "--dry-run" + ); + MatcherAssert.assertThat( + "Valid bulk mode dry-run should return exit code 0", + code, + Matchers.is(0) + ); + } + + /** + * --type alone without --path and --repo-name should fail (exit code 1). + * + * @throws IOException if test setup fails + */ + @Test + void typeWithoutPathAndRepoNameFails() throws IOException { + final int code = BackfillCli.run( + "--type", "file", + "--dry-run" + ); + MatcherAssert.assertThat( + "--type without --path and --repo-name should return exit code 1", + code, + Matchers.is(1) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/BackfillIntegrationTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/BackfillIntegrationTest.java new file mode 100644 index 000000000..786940e98 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/BackfillIntegrationTest.java @@ -0,0 +1,533 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.io.TempDir; + +/** + * Integration tests for the backfill CLI pipeline. + * + *

Dry-run tests (always run) exercise the full pipeline + * {@code BackfillCli -> ScannerFactory -> Scanner -> BatchInserter(dry-run)} + * for every supported scanner type with minimal but valid sample data.

+ * + *

PostgreSQL tests (gated behind the {@code BACKFILL_IT_DB_URL} + * environment variable) verify actual database inserts and + * UPSERT idempotency against a real PostgreSQL instance.

+ * + * @since 1.20.13 + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +final class BackfillIntegrationTest { + + // --------------------------------------------------------------- + // Dry-run tests (always run) + // --------------------------------------------------------------- + + /** + * Maven scanner dry-run: creates a minimal maven-metadata.xml with + * one version directory containing a JAR and verifies exit code 0. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(1) + void dryRunMavenScanner(@TempDir final Path tmp) throws IOException { + final Path artifact = tmp.resolve("com/example/mylib"); + Files.createDirectories(artifact); + Files.writeString( + artifact.resolve("maven-metadata.xml"), + String.join( + "\n", + "", + "", + " com.example", + " mylib", + " ", + " ", + " 1.0.0", + " ", + " ", + "" + ), + StandardCharsets.UTF_8 + ); + final Path ver = artifact.resolve("1.0.0"); + Files.createDirectories(ver); + Files.write(ver.resolve("mylib-1.0.0.jar"), new byte[64]); + final int code = BackfillCli.run( + "--type", "maven", + "--path", tmp.toString(), + "--repo-name", "it-maven", + "--dry-run" + ); + MatcherAssert.assertThat( + "Maven dry-run should succeed", + code, + Matchers.is(0) + ); + } + + /** + * Docker scanner dry-run: creates a minimal Docker registry layout + * with one image, one tag, and a manifest blob. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(2) + void dryRunDockerScanner(@TempDir final Path tmp) throws IOException { + final String digest = "sha256:aabbccdd11223344"; + final Path linkDir = tmp + .resolve("repositories") + .resolve("alpine") + .resolve("_manifests") + .resolve("tags") + .resolve("3.18") + .resolve("current"); + Files.createDirectories(linkDir); + Files.writeString( + linkDir.resolve("link"), digest, StandardCharsets.UTF_8 + ); + final String hex = digest.split(":", 2)[1]; + final Path blobDir = tmp.resolve("blobs") + .resolve("sha256") + .resolve(hex.substring(0, 2)) + .resolve(hex); + Files.createDirectories(blobDir); + Files.writeString( + blobDir.resolve("data"), + String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"config\": { \"size\": 100, \"digest\": \"sha256:cfg\" },", + " \"layers\": [", + " { \"size\": 500, \"digest\": \"sha256:l1\" }", + " ]", + "}" + ), + StandardCharsets.UTF_8 + ); + final int code = BackfillCli.run( + "--type", "docker", + "--path", tmp.toString(), + "--repo-name", "it-docker", + "--dry-run" + ); + MatcherAssert.assertThat( + "Docker dry-run should succeed", + code, + Matchers.is(0) + ); + } + + /** + * NPM scanner dry-run: creates a meta.json with one scoped package + * and one version entry. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(3) + void dryRunNpmScanner(@TempDir final Path tmp) throws IOException { + final Path pkgDir = tmp.resolve("@scope/widget"); + Files.createDirectories(pkgDir); + Files.writeString( + pkgDir.resolve("meta.json"), + String.join( + "\n", + "{", + " \"name\": \"@scope/widget\",", + " \"versions\": {", + " \"2.0.0\": {", + " \"name\": \"@scope/widget\",", + " \"version\": \"2.0.0\",", + " \"dist\": {", + " \"tarball\": \"/@scope/widget/-/" + + "@scope/widget-2.0.0.tgz\"", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final int code = BackfillCli.run( + "--type", "npm", + "--path", tmp.toString(), + "--repo-name", "it-npm", + "--dry-run" + ); + MatcherAssert.assertThat( + "NPM dry-run should succeed", + code, + Matchers.is(0) + ); + } + + /** + * PyPI scanner dry-run: creates a wheel file in a package directory. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(4) + void dryRunPypiScanner(@TempDir final Path tmp) throws IOException { + final Path pkgDir = tmp.resolve("requests"); + Files.createDirectories(pkgDir); + Files.write( + pkgDir.resolve("requests-2.31.0-py3-none-any.whl"), + new byte[80] + ); + final int code = BackfillCli.run( + "--type", "pypi", + "--path", tmp.toString(), + "--repo-name", "it-pypi", + "--dry-run" + ); + MatcherAssert.assertThat( + "PyPI dry-run should succeed", + code, + Matchers.is(0) + ); + } + + /** + * Go scanner dry-run: creates a module {@code @v} directory with + * a version list file and a .info JSON file. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(5) + void dryRunGoScanner(@TempDir final Path tmp) throws IOException { + final Path atv = tmp.resolve("example.com/mod/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "v1.0.0\n", + StandardCharsets.UTF_8 + ); + Files.writeString( + atv.resolve("v1.0.0.info"), + "{\"Version\":\"v1.0.0\"," + + "\"Time\":\"2024-01-01T00:00:00Z\"}", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v1.0.0.zip"), new byte[128]); + final int code = BackfillCli.run( + "--type", "go", + "--path", tmp.toString(), + "--repo-name", "it-go", + "--dry-run" + ); + MatcherAssert.assertThat( + "Go dry-run should succeed", + code, + Matchers.is(0) + ); + } + + /** + * Helm scanner dry-run: creates an index.yaml with one chart entry + * and a corresponding .tgz file. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(6) + void dryRunHelmScanner(@TempDir final Path tmp) throws IOException { + Files.writeString( + tmp.resolve("index.yaml"), + String.join( + "\n", + "apiVersion: v1", + "entries:", + " mychart:", + " - name: mychart", + " version: 0.1.0", + " urls:", + " - mychart-0.1.0.tgz", + " created: '2024-06-01T00:00:00+00:00'" + ), + StandardCharsets.UTF_8 + ); + Files.write(tmp.resolve("mychart-0.1.0.tgz"), new byte[256]); + final int code = BackfillCli.run( + "--type", "helm", + "--path", tmp.toString(), + "--repo-name", "it-helm", + "--dry-run" + ); + MatcherAssert.assertThat( + "Helm dry-run should succeed", + code, + Matchers.is(0) + ); + } + + /** + * Composer scanner dry-run: creates a p2 layout with one package + * JSON file containing one vendor/package with one version. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(7) + void dryRunComposerScanner(@TempDir final Path tmp) throws IOException { + final Path vendorDir = tmp.resolve("p2").resolve("vendor"); + Files.createDirectories(vendorDir); + Files.writeString( + vendorDir.resolve("lib.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"vendor/lib\": {", + " \"1.0.0\": {", + " \"name\": \"vendor/lib\",", + " \"version\": \"1.0.0\",", + " \"dist\": {", + " \"url\": \"https://example.com/lib.zip\",", + " \"type\": \"zip\"", + " }", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final int code = BackfillCli.run( + "--type", "composer", + "--path", tmp.toString(), + "--repo-name", "it-composer", + "--dry-run" + ); + MatcherAssert.assertThat( + "Composer dry-run should succeed", + code, + Matchers.is(0) + ); + } + + /** + * File scanner dry-run: creates a couple of plain files and one + * hidden file that should be skipped. + * + * @param tmp Temporary directory created by JUnit + * @throws IOException If temp file creation fails + */ + @Test + @Order(8) + void dryRunFileScanner(@TempDir final Path tmp) throws IOException { + Files.createFile(tmp.resolve("readme.txt")); + Files.write(tmp.resolve("data.bin"), new byte[32]); + Files.createFile(tmp.resolve(".hidden")); + final int code = BackfillCli.run( + "--type", "file", + "--path", tmp.toString(), + "--repo-name", "it-file", + "--dry-run" + ); + MatcherAssert.assertThat( + "File dry-run should succeed", + code, + Matchers.is(0) + ); + } + + // --------------------------------------------------------------- + // PostgreSQL tests (gated behind BACKFILL_IT_DB_URL) + // --------------------------------------------------------------- + + /** + * Insert records into a real PostgreSQL instance via the CLI pipeline + * and verify the row count matches the expected number. + * + *

Requires the following environment variables:

+ *
    + *
  • {@code BACKFILL_IT_DB_URL} - JDBC URL, e.g. + * {@code jdbc:postgresql://localhost:5432/artipie}
    • + *
  • {@code BACKFILL_IT_DB_USER} - (optional, default: artipie)
    • + *
  • {@code BACKFILL_IT_DB_PASSWORD} - (optional, default: artipie)
    • + *
+ * + * @param tmp Temporary directory created by JUnit + * @throws Exception If I/O or SQL operations fail + */ + @Test + @Order(10) + @EnabledIfEnvironmentVariable(named = "BACKFILL_IT_DB_URL", matches = ".+") + void insertsRecordsIntoPostgres(@TempDir final Path tmp) throws Exception { + final String dbUrl = System.getenv("BACKFILL_IT_DB_URL"); + final String dbUser = System.getenv().getOrDefault( + "BACKFILL_IT_DB_USER", "artipie" + ); + final String dbPassword = System.getenv().getOrDefault( + "BACKFILL_IT_DB_PASSWORD", "artipie" + ); + final String repoName = "it-pg-maven-" + System.nanoTime(); + final Path artifact = tmp.resolve("org/test/pglib"); + Files.createDirectories(artifact); + Files.writeString( + artifact.resolve("maven-metadata.xml"), + String.join( + "\n", + "", + "", + " org.test", + " pglib", + " ", + " ", + " 1.0.0", + " 2.0.0", + " ", + " ", + "" + ), + StandardCharsets.UTF_8 + ); + final Path ver1 = artifact.resolve("1.0.0"); + Files.createDirectories(ver1); + Files.write(ver1.resolve("pglib-1.0.0.jar"), new byte[100]); + final Path ver2 = artifact.resolve("2.0.0"); + Files.createDirectories(ver2); + Files.write(ver2.resolve("pglib-2.0.0.jar"), new byte[200]); + final int code = BackfillCli.run( + "--type", "maven", + "--path", tmp.toString(), + "--repo-name", repoName, + "--db-url", dbUrl, + "--db-user", dbUser, + "--db-password", dbPassword, + "--batch-size", "10" + ); + MatcherAssert.assertThat( + "CLI should succeed inserting into PostgreSQL", + code, + Matchers.is(0) + ); + final long count; + try (Connection conn = + DriverManager.getConnection(dbUrl, dbUser, dbPassword); + Statement stmt = conn.createStatement(); + ResultSet rset = stmt.executeQuery( + "SELECT count(*) FROM artifacts WHERE repo_name = '" + + repoName + "'" + )) { + rset.next(); + count = rset.getLong(1); + } + MatcherAssert.assertThat( + "Should have inserted exactly 2 records", + count, + Matchers.is(2L) + ); + } + + /** + * Run the same backfill again and verify the UPSERT does not + * duplicate rows (idempotency check). + * + * @param tmp Temporary directory created by JUnit + * @throws Exception If I/O or SQL operations fail + */ + @Test + @Order(11) + @EnabledIfEnvironmentVariable(named = "BACKFILL_IT_DB_URL", matches = ".+") + void upsertIsIdempotent(@TempDir final Path tmp) throws Exception { + final String dbUrl = System.getenv("BACKFILL_IT_DB_URL"); + final String dbUser = System.getenv().getOrDefault( + "BACKFILL_IT_DB_USER", "artipie" + ); + final String dbPassword = System.getenv().getOrDefault( + "BACKFILL_IT_DB_PASSWORD", "artipie" + ); + final String repoName = "it-pg-idempotent-" + System.nanoTime(); + final Path artifact = tmp.resolve("org/test/idem"); + Files.createDirectories(artifact); + Files.writeString( + artifact.resolve("maven-metadata.xml"), + String.join( + "\n", + "", + "", + " org.test", + " idem", + " ", + " ", + " 1.0.0", + " ", + " ", + "" + ), + StandardCharsets.UTF_8 + ); + final Path ver = artifact.resolve("1.0.0"); + Files.createDirectories(ver); + Files.write(ver.resolve("idem-1.0.0.jar"), new byte[50]); + final String[] args = { + "--type", "maven", + "--path", tmp.toString(), + "--repo-name", repoName, + "--db-url", dbUrl, + "--db-user", dbUser, + "--db-password", dbPassword, + "--batch-size", "10", + }; + final int firstRun = BackfillCli.run(args); + MatcherAssert.assertThat( + "First run should succeed", + firstRun, + Matchers.is(0) + ); + final int secondRun = BackfillCli.run(args); + MatcherAssert.assertThat( + "Second run (upsert) should succeed", + secondRun, + Matchers.is(0) + ); + final long count; + try (Connection conn = + DriverManager.getConnection(dbUrl, dbUser, dbPassword); + Statement stmt = conn.createStatement(); + ResultSet rset = stmt.executeQuery( + "SELECT count(*) FROM artifacts WHERE repo_name = '" + + repoName + "'" + )) { + rset.next(); + count = rset.getLong(1); + } + MatcherAssert.assertThat( + "UPSERT should not duplicate; count should still be 1", + count, + Matchers.is(1L) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/BatchInserterTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/BatchInserterTest.java new file mode 100644 index 000000000..ff5048fa9 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/BatchInserterTest.java @@ -0,0 +1,174 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link BatchInserter}. + * + *

These tests exercise dry-run counting, flush-threshold logic, and + * close-flushes-remaining behavior. Full database integration tests + * (PostgreSQL upsert, parameter binding, error fall-back) are deferred + * to Task 12.

+ * + * @since 1.20.13 + */ +final class BatchInserterTest { + + /** + * In dry-run mode, records are counted but nothing is written to the + * database. The {@code insertedCount} reflects the number of records + * that would have been inserted. + */ + @Test + void dryRunCountsWithoutDbInteraction() { + try (BatchInserter inserter = new BatchInserter(null, 100, true)) { + for (int idx = 0; idx < 5; idx++) { + inserter.accept(sampleRecord(idx)); + } + inserter.flush(); + MatcherAssert.assertThat( + "Dry-run should count all accepted records", + inserter.getInsertedCount(), + Matchers.is(5L) + ); + MatcherAssert.assertThat( + "Dry-run should have zero skipped", + inserter.getSkippedCount(), + Matchers.is(0L) + ); + } + } + + /** + * Verify that dry-run auto-flushes when the buffer reaches batchSize. + */ + @Test + void dryRunAutoFlushesAtBatchSize() { + try (BatchInserter inserter = new BatchInserter(null, 3, true)) { + inserter.accept(sampleRecord(1)); + inserter.accept(sampleRecord(2)); + MatcherAssert.assertThat( + "Before reaching batchSize, insertedCount should be 0", + inserter.getInsertedCount(), + Matchers.is(0L) + ); + inserter.accept(sampleRecord(3)); + MatcherAssert.assertThat( + "After reaching batchSize, auto-flush should have counted 3", + inserter.getInsertedCount(), + Matchers.is(3L) + ); + } + } + + /** + * Verify that close() flushes remaining records that haven't reached + * batchSize yet. + */ + @Test + void closeFlushesRemainingRecords() { + final BatchInserter inserter = new BatchInserter(null, 100, true); + inserter.accept(sampleRecord(1)); + inserter.accept(sampleRecord(2)); + MatcherAssert.assertThat( + "Before close, records should still be buffered", + inserter.getInsertedCount(), + Matchers.is(0L) + ); + inserter.close(); + MatcherAssert.assertThat( + "After close, remaining records should be flushed", + inserter.getInsertedCount(), + Matchers.is(2L) + ); + } + + /** + * Verify that multiple flushes accumulate the inserted count. + */ + @Test + void multipleFlushesAccumulateCount() { + try (BatchInserter inserter = new BatchInserter(null, 2, true)) { + inserter.accept(sampleRecord(1)); + inserter.accept(sampleRecord(2)); + MatcherAssert.assertThat( + "First flush should count 2", + inserter.getInsertedCount(), + Matchers.is(2L) + ); + inserter.accept(sampleRecord(3)); + inserter.accept(sampleRecord(4)); + MatcherAssert.assertThat( + "Second flush should bring total to 4", + inserter.getInsertedCount(), + Matchers.is(4L) + ); + } + } + + /** + * Verify that flushing an empty buffer does nothing. + */ + @Test + void flushEmptyBufferIsNoop() { + try (BatchInserter inserter = new BatchInserter(null, 10, true)) { + inserter.flush(); + MatcherAssert.assertThat( + "Flushing empty buffer should leave count at 0", + inserter.getInsertedCount(), + Matchers.is(0L) + ); + } + } + + /** + * Verify that in dry-run mode, DataSource is never touched (null is + * safe). + */ + @Test + void dryRunAcceptsNullDataSource() { + try (BatchInserter inserter = new BatchInserter(null, 5, true)) { + for (int idx = 0; idx < 12; idx++) { + inserter.accept(sampleRecord(idx)); + } + } + } + + /** + * Verify counters start at zero. + */ + @Test + void countersStartAtZero() { + try (BatchInserter inserter = new BatchInserter(null, 10, true)) { + MatcherAssert.assertThat( + "Initial insertedCount should be 0", + inserter.getInsertedCount(), + Matchers.is(0L) + ); + MatcherAssert.assertThat( + "Initial skippedCount should be 0", + inserter.getSkippedCount(), + Matchers.is(0L) + ); + } + } + + /** + * Create a sample ArtifactRecord for testing. + * + * @param idx Unique index to distinguish records + * @return Sample record + */ + private static ArtifactRecord sampleRecord(final int idx) { + return new ArtifactRecord( + "maven", "repo", "art-" + idx, "1.0." + idx, + 1024L, 1700000000L + idx, null, "system", null + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/BulkBackfillRunnerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/BulkBackfillRunnerTest.java new file mode 100644 index 000000000..6ea248353 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/BulkBackfillRunnerTest.java @@ -0,0 +1,354 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link BulkBackfillRunner}. + * + *

All tests use {@code dryRun=true} and a null datasource unless testing + * the FAILED path, which deliberately uses {@code dryRun=false} and a null + * datasource to trigger a NullPointerException in BatchInserter.

+ * + * @since 1.20.13 + */ +final class BulkBackfillRunnerTest { + + /** + * Null print stream for suppressing summary output during tests. + */ + private static final PrintStream DEV_NULL = + new PrintStream(OutputStream.nullOutputStream()); + + // ── Happy path ─────────────────────────────────────────────────────────── + + /** + * Empty config dir → exit code 0, zero repos processed. + * + * @param tmp JUnit temp directory + * @throws IOException if directory setup fails + */ + @Test + void emptyConfigDirSucceeds(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + Files.createDirectories(storageRoot); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "Empty config dir should return exit code 0", + code, + Matchers.is(0) + ); + } + + /** + * Two valid repos with file scanner → both succeed, exit code 0. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void twoValidReposSucceed(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + // Repo 1: "myfiles" type file + Files.writeString(configDir.resolve("myfiles.yaml"), "repo:\n type: file\n"); + final Path repo1 = storageRoot.resolve("myfiles"); + Files.createDirectories(repo1); + Files.writeString(repo1.resolve("artifact.txt"), "content"); + // Repo 2: "otherfiles" type file + Files.writeString(configDir.resolve("otherfiles.yaml"), "repo:\n type: file\n"); + final Path repo2 = storageRoot.resolve("otherfiles"); + Files.createDirectories(repo2); + Files.writeString(repo2.resolve("pkg.dat"), "data"); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "Two valid repos should return exit code 0", + code, + Matchers.is(0) + ); + } + + // ── SKIPPED paths ──────────────────────────────────────────────────────── + + /** + * Repo with unknown type → SKIPPED, rest continue, exit code 0. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void unknownTypeIsSkipped(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + // Unknown type + Files.writeString(configDir.resolve("weird.yaml"), "repo:\n type: weird-hosted\n"); + // Valid repo that should still run + Files.writeString(configDir.resolve("myfiles.yaml"), "repo:\n type: file\n"); + Files.createDirectories(storageRoot.resolve("myfiles")); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "Unknown type should be SKIPPED, run exits 0", + code, + Matchers.is(0) + ); + } + + /** + * Repo with missing storage path → SKIPPED, rest continue, exit code 0. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void missingStoragePathIsSkipped(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + Files.createDirectories(storageRoot); + // This repo has a valid YAML but no matching storage directory + Files.writeString(configDir.resolve("ghost.yaml"), "repo:\n type: file\n"); + // Valid repo + Files.writeString(configDir.resolve("real.yaml"), "repo:\n type: file\n"); + Files.createDirectories(storageRoot.resolve("real")); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "Missing storage path should be SKIPPED, run exits 0", + code, + Matchers.is(0) + ); + } + + /** + * Proxy type is normalised before lookup: docker-proxy → docker scanner is used. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void proxyTypeIsNormalised(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + // docker-proxy should normalise to docker + Files.writeString( + configDir.resolve("docker_cache.yaml"), + "repo:\n type: docker-proxy\n" + ); + // Create minimal docker v2 storage layout so DockerScanner doesn't fail on missing dirs + final Path dockerRepo = storageRoot.resolve("docker_cache"); + Files.createDirectories(dockerRepo.resolve("repositories")); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "docker-proxy should normalise to docker scanner, exit 0", + code, + Matchers.is(0) + ); + } + + // ── PARSE_ERROR paths ──────────────────────────────────────────────────── + + /** + * Malformed YAML → PARSE_ERROR, rest continue, exit code 0. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void parseErrorContinuesRun(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + Files.writeString(configDir.resolve("bad.yaml"), "repo: [\nunclosed\n"); + Files.writeString(configDir.resolve("good.yaml"), "repo:\n type: file\n"); + Files.createDirectories(storageRoot.resolve("good")); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "PARSE_ERROR should not set exit code to 1", + code, + Matchers.is(0) + ); + } + + /** + * PARSE_ERROR only run → exit code 0. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void parseErrorOnlyExitsZero(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + Files.createDirectories(configDir); + Files.writeString(configDir.resolve("bad.yaml"), "not: valid: yaml: content\n broken"); + final int code = runner(configDir, tmp, true).run(); + MatcherAssert.assertThat( + "PARSE_ERROR only should exit 0", + code, + Matchers.is(0) + ); + } + + // ── FAILED paths ───────────────────────────────────────────────────────── + + /** + * Scanner throws (triggered by null datasource + dryRun=false) → FAILED, + * rest continue, exit code 1. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void failedRepoExitsOne(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + // This repo will FAIL: dryRun=false, dataSource=null → NPE in BatchInserter + Files.writeString(configDir.resolve("willbreak.yaml"), "repo:\n type: file\n"); + final Path breakRepo = storageRoot.resolve("willbreak"); + Files.createDirectories(breakRepo); + Files.writeString(breakRepo.resolve("a.txt"), "x"); + // dryRun=false, dataSource=null triggers failure + final int code = new BulkBackfillRunner( + configDir, storageRoot, null, "system", 100, false, 10000, DEV_NULL + ).run(); + MatcherAssert.assertThat( + "FAILED repo should set exit code to 1", + code, + Matchers.is(1) + ); + } + + /** + * PARSE_ERROR + FAILED in same run → exit code 1 (FAILED dominates). + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void parseErrorPlusFailed(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + Files.writeString(configDir.resolve("bad.yaml"), "not: valid\n broken: ["); + Files.writeString(configDir.resolve("willbreak.yaml"), "repo:\n type: file\n"); + final Path breakRepo = storageRoot.resolve("willbreak"); + Files.createDirectories(breakRepo); + Files.writeString(breakRepo.resolve("a.txt"), "x"); + final int code = new BulkBackfillRunner( + configDir, storageRoot, null, "system", 100, false, 10000, DEV_NULL + ).run(); + MatcherAssert.assertThat( + "PARSE_ERROR + FAILED should exit 1", + code, + Matchers.is(1) + ); + } + + // ── Edge cases ─────────────────────────────────────────────────────────── + + /** + * Subdirectories in config dir are ignored (non-recursive). + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void subdirectoriesAreIgnored(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + // Subdirectory with a yaml inside — should not be processed + final Path subdir = configDir.resolve("subgroup"); + Files.createDirectories(subdir); + Files.writeString(subdir.resolve("inner.yaml"), "repo:\n type: file\n"); + // Valid top-level repo + Files.writeString(configDir.resolve("top.yaml"), "repo:\n type: file\n"); + Files.createDirectories(storageRoot.resolve("top")); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "Subdirectories should be ignored, run exits 0", + code, + Matchers.is(0) + ); + } + + /** + * A .yml file (wrong extension) is skipped — not processed, run still succeeds. + * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void ymlExtensionIsSkipped(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + // .yml file should be silently skipped + Files.writeString(configDir.resolve("repo.yml"), "repo:\n type: file\n"); + // Valid .yaml file + Files.writeString(configDir.resolve("valid.yaml"), "repo:\n type: file\n"); + Files.createDirectories(storageRoot.resolve("valid")); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + ".yml file should be skipped, run exits 0", + code, + Matchers.is(0) + ); + } + + /** + * Two repos with different names both succeed — verifies the seenNames set + * does not produce false-positive duplicate collisions. + * + *

Note: the filesystem guarantees unique filenames within a directory, + * so a true stem collision (two files producing the same stem) cannot + * occur in practice. The {@code seenNames} guard is a defensive measure. + * This test verifies the guard does not interfere with normal operation.

+ * + * @param tmp JUnit temp directory + * @throws IOException if file setup fails + */ + @Test + void twoDistinctReposDoNotCollide(@TempDir final Path tmp) throws IOException { + final Path configDir = tmp.resolve("configs"); + final Path storageRoot = tmp.resolve("data"); + Files.createDirectories(configDir); + Files.writeString(configDir.resolve("alpha.yaml"), "repo:\n type: file\n"); + Files.writeString(configDir.resolve("beta.yaml"), "repo:\n type: file\n"); + Files.createDirectories(storageRoot.resolve("alpha")); + Files.createDirectories(storageRoot.resolve("beta")); + final int code = runner(configDir, storageRoot, true).run(); + MatcherAssert.assertThat( + "Two repos with distinct names should both succeed, exit 0", + code, + Matchers.is(0) + ); + } + + // ── Helper ─────────────────────────────────────────────────────────────── + + private static BulkBackfillRunner runner( + final Path configDir, + final Path storageRoot, + final boolean dryRun + ) { + return new BulkBackfillRunner( + configDir, storageRoot, null, "system", 1000, dryRun, 10000, DEV_NULL + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/ComposerScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/ComposerScannerTest.java new file mode 100644 index 000000000..a20a0f5d6 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/ComposerScannerTest.java @@ -0,0 +1,481 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link ComposerScanner}. + * + * @since 1.20.13 + */ +final class ComposerScannerTest { + + @Test + void scansP2Layout(@TempDir final Path temp) throws IOException { + final Path vendorDir = temp.resolve("p2").resolve("vendor"); + Files.createDirectories(vendorDir); + Files.writeString( + vendorDir.resolve("package.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"vendor/package\": {", + " \"1.0.0\": {", + " \"name\": \"vendor/package\",", + " \"version\": \"1.0.0\",", + " \"dist\": {", + " \"url\": \"https://example.com/vendor/package-1.0.0.zip\",", + " \"type\": \"zip\"", + " }", + " },", + " \"2.0.0\": {", + " \"name\": \"vendor/package\",", + " \"version\": \"2.0.0\",", + " \"dist\": {", + " \"url\": \"https://example.com/vendor/package-2.0.0.zip\",", + " \"type\": \"zip\"", + " }", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "composer-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 2 records for 2 versions", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "All records should have name vendor/package", + records.stream().allMatch( + r -> "vendor/package".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 1.0.0", + records.stream().anyMatch(r -> "1.0.0".equals(r.version())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 2.0.0", + records.stream().anyMatch(r -> "2.0.0".equals(r.version())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Repo type should be composer", + records.get(0).repoType(), + Matchers.is("composer") + ); + } + + @Test + void scansPackagesJsonLayout(@TempDir final Path temp) throws IOException { + Files.writeString( + temp.resolve("packages.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"acme/foo\": {", + " \"1.0.0\": {", + " \"name\": \"acme/foo\",", + " \"version\": \"1.0.0\",", + " \"dist\": {", + " \"url\": \"https://example.com/acme/foo-1.0.0.zip\",", + " \"type\": \"zip\"", + " }", + " }", + " },", + " \"acme/bar\": {", + " \"2.0.0\": {", + " \"name\": \"acme/bar\",", + " \"version\": \"2.0.0\",", + " \"dist\": {", + " \"url\": \"https://example.com/acme/bar-2.0.0.zip\",", + " \"type\": \"zip\"", + " }", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "composer-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 2 records for 2 packages", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain acme/foo", + records.stream().anyMatch(r -> "acme/foo".equals(r.name())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain acme/bar", + records.stream().anyMatch(r -> "acme/bar".equals(r.name())), + Matchers.is(true) + ); + } + + @Test + void prefersP2OverPackagesJson(@TempDir final Path temp) + throws IOException { + final Path vendorDir = temp.resolve("p2").resolve("vendor"); + Files.createDirectories(vendorDir); + Files.writeString( + vendorDir.resolve("lib.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"vendor/lib\": {", + " \"1.0.0\": {", + " \"name\": \"vendor/lib\",", + " \"version\": \"1.0.0\"", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + Files.writeString( + temp.resolve("packages.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"other/pkg\": {", + " \"3.0.0\": {", + " \"name\": \"other/pkg\",", + " \"version\": \"3.0.0\"", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "composer-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record from p2 only", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Should contain vendor/lib from p2 layout", + records.get(0).name(), + Matchers.is("vendor/lib") + ); + MatcherAssert.assertThat( + "Should NOT contain other/pkg from packages.json", + records.stream().noneMatch(r -> "other/pkg".equals(r.name())), + Matchers.is(true) + ); + } + + @Test + void handlesMissingPackagesKey(@TempDir final Path temp) + throws IOException { + final Path vendorDir = temp.resolve("p2").resolve("vendor"); + Files.createDirectories(vendorDir); + Files.writeString( + vendorDir.resolve("nopackages.json"), + String.join( + "\n", + "{", + " \"minified\": \"provider/latest\"", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "composer-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records when packages key is missing", + records, + Matchers.empty() + ); + } + + @Test + void skipsDevJsonFiles(@TempDir final Path temp) throws IOException { + final Path vendorDir = temp.resolve("p2").resolve("vendor"); + Files.createDirectories(vendorDir); + Files.writeString( + vendorDir.resolve("pkg~dev.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"vendor/pkg\": {", + " \"dev-master\": {", + " \"name\": \"vendor/pkg\",", + " \"version\": \"dev-master\"", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "composer-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records when only ~dev.json files exist", + records, + Matchers.empty() + ); + } + + @Test + void handlesEmptyRoot(@TempDir final Path temp) throws IOException { + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "composer-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records for empty root", + records, + Matchers.empty() + ); + } + + @Test + void skipsEmptyPackagesJsonAndScansVendorDirs(@TempDir final Path temp) + throws IOException { + // packages.json exists but is 0 bytes (common in Artipie proxy repos) + Files.createFile(temp.resolve("packages.json")); + // vendor-dir layout files exist with real content + final Path vendorDir = temp.resolve("psr"); + Files.createDirectories(vendorDir); + Files.writeString( + vendorDir.resolve("log.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"psr/log\": {", + " \"1.0.0\": {", + " \"name\": \"psr/log\",", + " \"version\": \"1.0.0\"", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "php-proxy") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find 1 record from vendor-dir layout despite empty packages.json", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Record name should be psr/log", + records.get(0).name(), + Matchers.is("psr/log") + ); + MatcherAssert.assertThat( + "Record version should be 1.0.0", + records.get(0).version(), + Matchers.is("1.0.0") + ); + } + + @Test + void scansVendorDirLayout(@TempDir final Path temp) throws IOException { + // Two vendor directories, multiple packages + final Path psr = temp.resolve("psr"); + Files.createDirectories(psr); + Files.writeString( + psr.resolve("log.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"psr/log\": {", + " \"1.0.0\": { \"name\": \"psr/log\", \"version\": \"1.0.0\" },", + " \"2.0.0\": { \"name\": \"psr/log\", \"version\": \"2.0.0\" }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + Files.writeString( + psr.resolve("http-message.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"psr/http-message\": {", + " \"1.1.0\": { \"name\": \"psr/http-message\", \"version\": \"1.1.0\" }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final Path symfony = temp.resolve("symfony"); + Files.createDirectories(symfony); + Files.writeString( + symfony.resolve("http-client.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"symfony/http-client\": {", + " \"6.4.0\": { \"name\": \"symfony/http-client\", \"version\": \"6.4.0\" }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "php-proxy") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 4 records total (2 psr/log + 1 psr/http-message + 1 symfony/http-client)", + records, + Matchers.hasSize(4) + ); + MatcherAssert.assertThat( + "Should contain psr/log", + records.stream().anyMatch(r -> "psr/log".equals(r.name())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain psr/http-message", + records.stream().anyMatch(r -> "psr/http-message".equals(r.name())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain symfony/http-client", + records.stream().anyMatch(r -> "symfony/http-client".equals(r.name())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "All records should have composer repo type", + records.stream().allMatch(r -> "composer".equals(r.repoType())), + Matchers.is(true) + ); + } + + @Test + void skipsEmptyFilesInVendorDirLayout(@TempDir final Path temp) + throws IOException { + final Path psr = temp.resolve("psr"); + Files.createDirectories(psr); + // One empty file (0 bytes) — should be skipped silently + Files.createFile(psr.resolve("log.json")); + // One non-empty file — should be scanned + Files.writeString( + psr.resolve("container.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"psr/container\": {", + " \"2.0.0\": { \"name\": \"psr/container\", \"version\": \"2.0.0\" }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "php-proxy") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record — empty file skipped, non-empty file scanned", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Record should be from psr/container (non-empty file)", + records.get(0).name(), + Matchers.is("psr/container") + ); + } + + @Test + void skipsDevJsonFilesInVendorDirLayout(@TempDir final Path temp) + throws IOException { + final Path openTelemetry = temp.resolve("open-telemetry"); + Files.createDirectories(openTelemetry); + // dev file — should be skipped + Files.writeString( + openTelemetry.resolve("sem-conv~dev.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"open-telemetry/sem-conv\": {", + " \"dev-main\": { \"name\": \"open-telemetry/sem-conv\", \"version\": \"dev-main\" }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + // stable file — should be scanned + Files.writeString( + openTelemetry.resolve("sem-conv.json"), + String.join( + "\n", + "{", + " \"packages\": {", + " \"open-telemetry/sem-conv\": {", + " \"1.0.0\": { \"name\": \"open-telemetry/sem-conv\", \"version\": \"1.0.0\" }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final ComposerScanner scanner = new ComposerScanner(); + final List records = scanner.scan(temp, "php-proxy") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record — ~dev.json file skipped", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Record version should be 1.0.0 (from stable file, not dev)", + records.get(0).version(), + Matchers.is("1.0.0") + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/DebianScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/DebianScannerTest.java new file mode 100644 index 000000000..6fa7aebad --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/DebianScannerTest.java @@ -0,0 +1,313 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link DebianScanner}. + * + * @since 1.20.13 + */ +final class DebianScannerTest { + + @Test + void parsesUncompressedPackagesFile(@TempDir final Path temp) + throws IOException { + final Path dir = temp.resolve("dists/focal/main/binary-amd64"); + Files.createDirectories(dir); + Files.writeString( + dir.resolve("Packages"), + String.join( + "\n", + "Package: curl", + "Version: 7.68.0-1ubuntu2.6", + "Architecture: amd64", + "Size: 161672", + "Filename: pool/main/c/curl/curl_7.68.0-1ubuntu2.6_amd64.deb", + "", + "Package: wget", + "Version: 1.20.3-1ubuntu2", + "Architecture: amd64", + "Size: 345678", + "Filename: pool/main/w/wget/wget_1.20.3-1ubuntu2_amd64.deb", + "" + ) + ); + final DebianScanner scanner = new DebianScanner(); + final List records = scanner.scan(temp, "deb-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 2 records", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "First record name should be curl_amd64", + records.stream().anyMatch( + r -> "curl_amd64".equals(r.name()) + && "7.68.0-1ubuntu2.6".equals(r.version()) + && r.size() == 161672L + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Second record name should be wget_amd64", + records.stream().anyMatch( + r -> "wget_amd64".equals(r.name()) + && "1.20.3-1ubuntu2".equals(r.version()) + && r.size() == 345678L + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Repo type should be deb", + records.get(0).repoType(), + Matchers.is("deb") + ); + MatcherAssert.assertThat( + "Owner should be system", + records.get(0).owner(), + Matchers.is("system") + ); + } + + @Test + void parsesGzipCompressedPackagesFile(@TempDir final Path temp) + throws IOException { + final Path dir = temp.resolve("dists/focal/main/binary-amd64"); + Files.createDirectories(dir); + final String content = String.join( + "\n", + "Package: nginx", + "Version: 1.18.0-0ubuntu1", + "Architecture: amd64", + "Size: 543210", + "", + "Package: apache2", + "Version: 2.4.41-4ubuntu3", + "Architecture: amd64", + "Size: 987654", + "" + ); + final Path gzPath = dir.resolve("Packages.gz"); + try (OutputStream fos = Files.newOutputStream(gzPath); + GZIPOutputStream gzos = new GZIPOutputStream(fos)) { + gzos.write(content.getBytes(StandardCharsets.UTF_8)); + } + final DebianScanner scanner = new DebianScanner(); + final List records = scanner.scan(temp, "deb-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 2 records from gzip file", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain nginx_amd64 record", + records.stream().anyMatch( + r -> "nginx_amd64".equals(r.name()) + && "1.18.0-0ubuntu1".equals(r.version()) + && r.size() == 543210L + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain apache2_amd64 record", + records.stream().anyMatch( + r -> "apache2_amd64".equals(r.name()) + && "2.4.41-4ubuntu3".equals(r.version()) + && r.size() == 987654L + ), + Matchers.is(true) + ); + } + + @Test + void defaultsSizeToZeroWhenMissing(@TempDir final Path temp) + throws IOException { + final Path dir = temp.resolve("dists/focal/main/binary-amd64"); + Files.createDirectories(dir); + Files.writeString( + dir.resolve("Packages"), + String.join( + "\n", + "Package: nano", + "Version: 4.8-1ubuntu1", + "Architecture: amd64", + "" + ) + ); + final DebianScanner scanner = new DebianScanner(); + final List records = scanner.scan(temp, "deb-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Size should default to 0 when missing", + records.get(0).size(), + Matchers.is(0L) + ); + } + + @Test + void skipsStanzasMissingPackageOrVersion(@TempDir final Path temp) + throws IOException { + final Path dir = temp.resolve("dists/focal/main/binary-amd64"); + Files.createDirectories(dir); + Files.writeString( + dir.resolve("Packages"), + String.join( + "\n", + "Package: valid-pkg", + "Version: 1.0", + "Size: 100", + "", + "Version: 2.0", + "Size: 200", + "", + "Package: no-version", + "Size: 300", + "", + "Package: another-valid", + "Version: 3.0", + "Size: 400", + "" + ) + ); + final DebianScanner scanner = new DebianScanner(); + final List records = scanner.scan(temp, "deb-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 2 records, skipping incomplete stanzas", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain valid-pkg", + records.stream().anyMatch( + r -> "valid-pkg".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain another-valid", + records.stream().anyMatch( + r -> "another-valid".equals(r.name()) + ), + Matchers.is(true) + ); + } + + @Test + void handlesMultipleDistributionsAndComponents(@TempDir final Path temp) + throws IOException { + final Path focal = temp.resolve("dists/focal/main/binary-amd64"); + Files.createDirectories(focal); + Files.writeString( + focal.resolve("Packages"), + String.join( + "\n", + "Package: focal-pkg", + "Version: 1.0", + "Size: 100", + "" + ) + ); + final Path bionic = temp.resolve("dists/bionic/contrib/binary-i386"); + Files.createDirectories(bionic); + Files.writeString( + bionic.resolve("Packages"), + String.join( + "\n", + "Package: bionic-pkg", + "Version: 2.0", + "Size: 200", + "" + ) + ); + final DebianScanner scanner = new DebianScanner(); + final List records = scanner.scan(temp, "deb-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce records from both distributions", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain focal-pkg", + records.stream().anyMatch( + r -> "focal-pkg".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain bionic-pkg", + records.stream().anyMatch( + r -> "bionic-pkg".equals(r.name()) + ), + Matchers.is(true) + ); + } + + @Test + void returnsEmptyForEmptyDirectory(@TempDir final Path temp) + throws IOException { + final DebianScanner scanner = new DebianScanner(); + final List records = scanner.scan(temp, "deb-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should return empty stream for empty directory", + records, + Matchers.empty() + ); + } + + @Test + void prefersPackagesGzOverPackages(@TempDir final Path temp) + throws IOException { + final Path dir = temp.resolve("dists/focal/main/binary-amd64"); + Files.createDirectories(dir); + final String content = String.join( + "\n", + "Package: curl", + "Version: 7.68.0", + "Size: 100", + "", + "Package: wget", + "Version: 1.20.3", + "Size: 200", + "" + ); + Files.writeString(dir.resolve("Packages"), content); + final Path gzPath = dir.resolve("Packages.gz"); + try (OutputStream fos = Files.newOutputStream(gzPath); + GZIPOutputStream gzos = new GZIPOutputStream(fos)) { + gzos.write(content.getBytes(StandardCharsets.UTF_8)); + } + final DebianScanner scanner = new DebianScanner(); + final List records = scanner.scan(temp, "deb-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should not double-count when both Packages and Packages.gz exist", + records, + Matchers.hasSize(2) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/DockerScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/DockerScannerTest.java new file mode 100644 index 000000000..a6fc1203f --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/DockerScannerTest.java @@ -0,0 +1,381 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link DockerScanner}. + * + * @since 1.20.13 + */ +final class DockerScannerTest { + + @Test + void scansImageWithTag(@TempDir final Path temp) throws IOException { + final String digest = "sha256:abc123def456"; + DockerScannerTest.createTagLink(temp, "nginx", "latest", digest); + final String manifest = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"config\": { \"size\": 7023, \"digest\": \"sha256:config1\" },", + " \"layers\": [", + " { \"size\": 32654, \"digest\": \"sha256:layer1\" },", + " { \"size\": 73109, \"digest\": \"sha256:layer2\" }", + " ]", + "}" + ); + DockerScannerTest.createBlob(temp, digest, manifest); + final DockerScanner scanner = new DockerScanner(true); + final List records = scanner.scan(temp, "docker-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + final ArtifactRecord record = records.get(0); + MatcherAssert.assertThat( + "Image name should be nginx", + record.name(), + Matchers.is("nginx") + ); + MatcherAssert.assertThat( + "Version should be the tag name", + record.version(), + Matchers.is("latest") + ); + MatcherAssert.assertThat( + "Size should be config + layers sum", + record.size(), + Matchers.is(7023L + 32654L + 73109L) + ); + MatcherAssert.assertThat( + "Repo type should be docker-proxy", + record.repoType(), + Matchers.is("docker-proxy") + ); + MatcherAssert.assertThat( + "Repo name should be docker-repo", + record.repoName(), + Matchers.is("docker-repo") + ); + } + + @Test + void scansMultipleTagsForImage(@TempDir final Path temp) + throws IOException { + final String digest1 = "sha256:aaa111bbb222"; + final String digest2 = "sha256:ccc333ddd444"; + DockerScannerTest.createTagLink(temp, "nginx", "latest", digest1); + DockerScannerTest.createTagLink(temp, "nginx", "1.25", digest2); + final String manifest1 = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"config\": { \"size\": 1000, \"digest\": \"sha256:cfg1\" },", + " \"layers\": [", + " { \"size\": 2000, \"digest\": \"sha256:l1\" }", + " ]", + "}" + ); + final String manifest2 = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"config\": { \"size\": 500, \"digest\": \"sha256:cfg2\" },", + " \"layers\": [", + " { \"size\": 1500, \"digest\": \"sha256:l2\" }", + " ]", + "}" + ); + DockerScannerTest.createBlob(temp, digest1, manifest1); + DockerScannerTest.createBlob(temp, digest2, manifest2); + final DockerScanner scanner = new DockerScanner(); + final List records = scanner.scan(temp, "docker-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 2 records", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain 'latest' as version", + records.stream().anyMatch( + r -> "latest".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain '1.25' as version", + records.stream().anyMatch( + r -> "1.25".equals(r.version()) + ), + Matchers.is(true) + ); + final ArtifactRecord first = records.stream() + .filter(r -> "latest".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "latest tag size should be 3000", + first.size(), + Matchers.is(3000L) + ); + final ArtifactRecord second = records.stream() + .filter(r -> "1.25".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "1.25 tag size should be 2000", + second.size(), + Matchers.is(2000L) + ); + } + + @Test + void handlesMissingBlob(@TempDir final Path temp) throws IOException { + final String digest = "sha256:deadbeef0000"; + DockerScannerTest.createTagLink(temp, "alpine", "3.18", digest); + final DockerScanner scanner = new DockerScanner(); + final List records = scanner.scan(temp, "docker-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record even with missing blob", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Size should be 0 when blob is missing", + records.get(0).size(), + Matchers.is(0L) + ); + } + + @Test + void handlesManifestList(@TempDir final Path temp) throws IOException { + final String childDigest = "sha256:child111222333"; + final String childManifest = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",", + " \"config\": { \"size\": 1504, \"digest\": \"sha256:cfgchild\" },", + " \"layers\": [", + " { \"size\": 28865120, \"digest\": \"sha256:layerchild\" }", + " ]", + "}" + ); + DockerScannerTest.createBlob(temp, childDigest, childManifest); + final String attestDigest = "sha256:attest999888777"; + final String attestManifest = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",", + " \"config\": { \"size\": 167, \"digest\": \"sha256:cfgattest\" },", + " \"layers\": [", + " { \"size\": 1331, \"digest\": \"sha256:layerattest\",", + " \"mediaType\": \"application/vnd.in-toto+json\" }", + " ]", + "}" + ); + DockerScannerTest.createBlob(temp, attestDigest, attestManifest); + final String listDigest = "sha256:ffee00112233"; + final String manifestList = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\",", + " \"manifests\": [", + " { \"digest\": \"" + childDigest + "\", \"size\": 482,", + " \"platform\": { \"architecture\": \"amd64\", \"os\": \"linux\" } },", + " { \"digest\": \"" + attestDigest + "\", \"size\": 566,", + " \"platform\": { \"architecture\": \"unknown\", \"os\": \"unknown\" } }", + " ]", + "}" + ); + DockerScannerTest.createTagLink(temp, "ubuntu", "22.04", listDigest); + DockerScannerTest.createBlob(temp, listDigest, manifestList); + final DockerScanner scanner = new DockerScanner(); + final List records = scanner.scan(temp, "docker-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record", + records, + Matchers.hasSize(1) + ); + final ArtifactRecord record = records.get(0); + MatcherAssert.assertThat( + "Size should be sum of ALL child manifests' layers and configs", + record.size(), + Matchers.is(1504L + 28865120L + 167L + 1331L) + ); + } + + @Test + void handlesNestedImageName(@TempDir final Path temp) throws IOException { + final String digest = "sha256:1122334455aa"; + DockerScannerTest.createTagLink(temp, "library/redis", "7.0", digest); + final String manifest = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"config\": { \"size\": 500, \"digest\": \"sha256:rcfg\" },", + " \"layers\": [", + " { \"size\": 10000, \"digest\": \"sha256:rl1\" }", + " ]", + "}" + ); + DockerScannerTest.createBlob(temp, digest, manifest); + final DockerScanner scanner = new DockerScanner(true); + final List records = scanner.scan(temp, "docker-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Image name should include nested path", + records.get(0).name(), + Matchers.is("library/redis") + ); + MatcherAssert.assertThat( + "Version should be the tag name", + records.get(0).version(), + Matchers.is("7.0") + ); + } + + @Test + void scansDockerRegistryV2Layout(@TempDir final Path temp) + throws IOException { + final String digest = "sha256:abcdef123456"; + final Path v2 = temp.resolve("docker/registry/v2"); + final Path linkDir = v2 + .resolve("repositories/ubuntu/_manifests/tags/latest/current"); + Files.createDirectories(linkDir); + Files.writeString( + linkDir.resolve("link"), digest, StandardCharsets.UTF_8 + ); + final String manifest = String.join( + "\n", + "{", + " \"schemaVersion\": 2,", + " \"config\": { \"size\": 2000, \"digest\": \"sha256:c1\" },", + " \"layers\": [", + " { \"size\": 50000, \"digest\": \"sha256:l1\" }", + " ]", + "}" + ); + final String[] parts = digest.split(":", 2); + final Path blobDir = v2.resolve("blobs") + .resolve(parts[0]) + .resolve(parts[1].substring(0, 2)) + .resolve(parts[1]); + Files.createDirectories(blobDir); + Files.writeString( + blobDir.resolve("data"), manifest, StandardCharsets.UTF_8 + ); + final DockerScanner scanner = new DockerScanner(true); + final List records = scanner.scan(temp, "docker-cache") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find image in docker/registry/v2 layout", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Image name should be ubuntu", + records.get(0).name(), + Matchers.is("ubuntu") + ); + MatcherAssert.assertThat( + "Size should be config + layer", + records.get(0).size(), + Matchers.is(52000L) + ); + } + + @Test + void handlesMissingRepositoriesDir(@TempDir final Path temp) + throws IOException { + final DockerScanner scanner = new DockerScanner(); + final List records = scanner.scan(temp, "docker-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records when repositories dir is missing", + records, + Matchers.empty() + ); + } + + /** + * Create a tag link file in the Docker registry layout. + * + * @param root Root directory (contains repositories/ and blobs/) + * @param imageName Image name (e.g., "nginx" or "library/redis") + * @param tag Tag name (e.g., "latest") + * @param digest Digest string (e.g., "sha256:abc123") + * @throws IOException If an I/O error occurs + */ + private static void createTagLink(final Path root, + final String imageName, final String tag, final String digest) + throws IOException { + final Path linkDir = root + .resolve("repositories") + .resolve(imageName) + .resolve("_manifests") + .resolve("tags") + .resolve(tag) + .resolve("current"); + Files.createDirectories(linkDir); + Files.writeString( + linkDir.resolve("link"), digest, StandardCharsets.UTF_8 + ); + } + + /** + * Create a blob data file for a given digest. + * + * @param root Root directory (contains repositories/ and blobs/) + * @param digest Digest string (e.g., "sha256:abc123def456") + * @param content Blob content (manifest JSON) + * @throws IOException If an I/O error occurs + */ + private static void createBlob(final Path root, final String digest, + final String content) throws IOException { + final Path dataPath = DockerScannerTest.blobDataPath(root, digest); + Files.createDirectories(dataPath.getParent()); + Files.writeString(dataPath, content, StandardCharsets.UTF_8); + } + + /** + * Compute the blob data path for a given digest. + * + * @param root Root directory + * @param digest Digest string + * @return Path to the data file + */ + private static Path blobDataPath(final Path root, final String digest) { + final String[] parts = digest.split(":", 2); + final String algorithm = parts[0]; + final String hex = parts[1]; + return root.resolve("blobs") + .resolve(algorithm) + .resolve(hex.substring(0, 2)) + .resolve(hex) + .resolve("data"); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/GemScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/GemScannerTest.java new file mode 100644 index 000000000..57962e484 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/GemScannerTest.java @@ -0,0 +1,244 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link GemScanner}. + * + * @since 1.20.13 + */ +final class GemScannerTest { + + @Test + void parsesSimpleGemFilename(@TempDir final Path temp) throws IOException { + final Path gems = temp.resolve("gems"); + Files.createDirectories(gems); + Files.write(gems.resolve("rake-13.0.6.gem"), new byte[100]); + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + final ArtifactRecord record = records.get(0); + MatcherAssert.assertThat( + "Name should be rake", + record.name(), + Matchers.is("rake") + ); + MatcherAssert.assertThat( + "Version should be 13.0.6", + record.version(), + Matchers.is("13.0.6") + ); + MatcherAssert.assertThat( + "Size should be 100", + record.size(), + Matchers.is(100L) + ); + MatcherAssert.assertThat( + "Repo type should be gem", + record.repoType(), + Matchers.is("gem") + ); + MatcherAssert.assertThat( + "Owner should be system", + record.owner(), + Matchers.is("system") + ); + } + + @Test + void parsesGemWithHyphenatedName(@TempDir final Path temp) + throws IOException { + final Path gems = temp.resolve("gems"); + Files.createDirectories(gems); + Files.write(gems.resolve("net-http-0.3.2.gem"), new byte[80]); + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be net-http", + records.get(0).name(), + Matchers.is("net-http") + ); + MatcherAssert.assertThat( + "Version should be 0.3.2", + records.get(0).version(), + Matchers.is("0.3.2") + ); + } + + @Test + void parsesGemWithPlatform(@TempDir final Path temp) + throws IOException { + final Path gems = temp.resolve("gems"); + Files.createDirectories(gems); + Files.write( + gems.resolve("nokogiri-1.13.8-x86_64-linux.gem"), + new byte[200] + ); + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be nokogiri", + records.get(0).name(), + Matchers.is("nokogiri") + ); + MatcherAssert.assertThat( + "Version should be 1.13.8", + records.get(0).version(), + Matchers.is("1.13.8") + ); + } + + @Test + void parsesGemWithMultipleHyphensInName(@TempDir final Path temp) + throws IOException { + final Path gems = temp.resolve("gems"); + Files.createDirectories(gems); + Files.write(gems.resolve("ruby-ole-1.2.12.7.gem"), new byte[150]); + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be ruby-ole", + records.get(0).name(), + Matchers.is("ruby-ole") + ); + MatcherAssert.assertThat( + "Version should be 1.2.12.7", + records.get(0).version(), + Matchers.is("1.2.12.7") + ); + } + + @Test + void handlesMultipleGems(@TempDir final Path temp) throws IOException { + final Path gems = temp.resolve("gems"); + Files.createDirectories(gems); + Files.write(gems.resolve("rails-7.0.4.gem"), new byte[300]); + Files.write(gems.resolve("rake-13.0.6.gem"), new byte[100]); + Files.write( + gems.resolve("activerecord-7.0.4.gem"), new byte[250] + ); + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 3 records", + records, + Matchers.hasSize(3) + ); + MatcherAssert.assertThat( + "Should contain rails", + records.stream().anyMatch( + r -> "rails".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain rake", + records.stream().anyMatch( + r -> "rake".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain activerecord", + records.stream().anyMatch( + r -> "activerecord".equals(r.name()) + ), + Matchers.is(true) + ); + } + + @Test + void skipsNonGemFiles(@TempDir final Path temp) throws IOException { + final Path gems = temp.resolve("gems"); + Files.createDirectories(gems); + Files.writeString(gems.resolve("readme.txt"), "hello"); + Files.writeString(gems.resolve("notes.md"), "notes"); + Files.write(gems.resolve("data.tar.gz"), new byte[50]); + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records for non-gem files", + records, + Matchers.empty() + ); + } + + @Test + void returnsEmptyForEmptyDirectory(@TempDir final Path temp) + throws IOException { + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records for empty directory", + records, + Matchers.empty() + ); + } + + @Test + void handlesGemsInRootDirectly(@TempDir final Path temp) + throws IOException { + Files.write(temp.resolve("rake-13.0.6.gem"), new byte[100]); + Files.write(temp.resolve("rails-7.0.4.gem"), new byte[200]); + final GemScanner scanner = new GemScanner(); + final List records = scanner.scan(temp, "gem-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 2 records from root-level gems", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain rake", + records.stream().anyMatch( + r -> "rake".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain rails", + records.stream().anyMatch( + r -> "rails".equals(r.name()) + ), + Matchers.is(true) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/GoScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/GoScannerTest.java new file mode 100644 index 000000000..bb33611c4 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/GoScannerTest.java @@ -0,0 +1,327 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link GoScanner}. + * + * @since 1.20.13 + */ +final class GoScannerTest { + + @Test + void scansModuleWithVersions(@TempDir final Path temp) throws IOException { + final Path atv = temp.resolve("example.com/foo/bar/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "v1.0.0\nv1.1.0\n", + StandardCharsets.UTF_8 + ); + Files.writeString( + atv.resolve("v1.0.0.info"), + "{\"Version\":\"v1.0.0\",\"Time\":\"2024-01-15T10:30:00Z\"}", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v1.0.0.zip"), new byte[200]); + Files.writeString( + atv.resolve("v1.1.0.info"), + "{\"Version\":\"v1.1.0\",\"Time\":\"2024-02-20T14:00:00Z\"}", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v1.1.0.zip"), new byte[350]); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 2 records for 2 versions", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "All records should have module path example.com/foo/bar", + records.stream().allMatch( + r -> "example.com/foo/bar".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "All records should have repoType go", + records.stream().allMatch(r -> "go".equals(r.repoType())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 1.0.0", + records.stream().anyMatch(r -> "1.0.0".equals(r.version())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 1.1.0", + records.stream().anyMatch(r -> "1.1.0".equals(r.version())), + Matchers.is(true) + ); + final ArtifactRecord first = records.stream() + .filter(r -> "1.0.0".equals(r.version())) + .findFirst() + .orElseThrow(); + MatcherAssert.assertThat( + "v1.0.0 zip size should be 200", + first.size(), + Matchers.is(200L) + ); + final ArtifactRecord second = records.stream() + .filter(r -> "1.1.0".equals(r.version())) + .findFirst() + .orElseThrow(); + MatcherAssert.assertThat( + "v1.1.0 zip size should be 350", + second.size(), + Matchers.is(350L) + ); + } + + @Test + void handlesMissingZipFile(@TempDir final Path temp) throws IOException { + final Path atv = temp.resolve("example.com/lib/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "v2.0.0\n", + StandardCharsets.UTF_8 + ); + Files.writeString( + atv.resolve("v2.0.0.info"), + "{\"Version\":\"v2.0.0\",\"Time\":\"2024-03-10T08:00:00Z\"}", + StandardCharsets.UTF_8 + ); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records when zip is not cached", + records, + Matchers.empty() + ); + } + + @Test + void handlesMissingInfoFile(@TempDir final Path temp) throws IOException { + final Path atv = temp.resolve("example.com/noinfo/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "v3.0.0\n", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v3.0.0.zip"), new byte[100]); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should still produce 1 record", + records, + Matchers.hasSize(1) + ); + final long listMtime = Files.readAttributes( + atv.resolve("list"), BasicFileAttributes.class + ).lastModifiedTime().toMillis(); + MatcherAssert.assertThat( + "CreatedDate should fall back to list file mtime", + records.get(0).createdDate(), + Matchers.is(listMtime) + ); + } + + @Test + void parsesTimestampFromInfoFile(@TempDir final Path temp) + throws IOException { + final String timestamp = "2024-01-15T10:30:00Z"; + final long expected = Instant.parse(timestamp).toEpochMilli(); + final Path atv = temp.resolve("example.com/timed/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "v1.0.0\n", + StandardCharsets.UTF_8 + ); + Files.writeString( + atv.resolve("v1.0.0.info"), + "{\"Version\":\"v1.0.0\",\"Time\":\"" + timestamp + "\"}", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v1.0.0.zip"), new byte[50]); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "CreatedDate should match the parsed Time field", + records.get(0).createdDate(), + Matchers.is(expected) + ); + } + + @Test + void skipsUncachedVersionsInListFile(@TempDir final Path temp) + throws IOException { + // List has v1.0.1–v1.0.4 but only v1.0.4 was actually downloaded + final Path atv = temp.resolve("gopkg.in/example/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "v1.0.1\nv1.0.2\nv1.0.3\nv1.0.4\n", + StandardCharsets.UTF_8 + ); + for (final String ver : new String[]{"v1.0.1", "v1.0.2", "v1.0.3", "v1.0.4"}) { + Files.writeString( + atv.resolve(ver + ".info"), + "{\"Version\":\"" + ver + "\",\"Time\":\"2024-01-01T00:00:00Z\"}", + StandardCharsets.UTF_8 + ); + } + // Only v1.0.4 has a zip (actually cached) + Files.write(atv.resolve("v1.0.4.zip"), new byte[12345]); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-proxy") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should only index the one version that has a zip", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Only cached version 1.0.4 should be indexed", + records.get(0).version(), + Matchers.is("1.0.4") + ); + MatcherAssert.assertThat( + "Size should reflect the zip file", + records.get(0).size(), + Matchers.is(12345L) + ); + } + + @Test + void handlesEmptyListFile(@TempDir final Path temp) throws IOException { + final Path atv = temp.resolve("example.com/empty/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "", + StandardCharsets.UTF_8 + ); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Empty list file should produce 0 records", + records, + Matchers.empty() + ); + } + + @Test + void scansByInfoFilesWhenNoListFile(@TempDir final Path temp) + throws IOException { + // Proxy layout: only .info and .zip files, no list file + final Path atv = temp.resolve("example.com/proxy-mod/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("v1.0.0.info"), + "{\"Version\":\"v1.0.0\",\"Time\":\"2024-06-01T12:00:00Z\"}", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v1.0.0.zip"), new byte[300]); + Files.writeString( + atv.resolve("v1.1.0.info"), + "{\"Version\":\"v1.1.0\",\"Time\":\"2024-07-01T12:00:00Z\"}", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v1.1.0.zip"), new byte[400]); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-proxy") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find 2 versions via .info files", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "All records should have module path example.com/proxy-mod", + records.stream().allMatch( + r -> "example.com/proxy-mod".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 1.0.0", + records.stream().anyMatch(r -> "1.0.0".equals(r.version())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 1.1.0", + records.stream().anyMatch(r -> "1.1.0".equals(r.version())), + Matchers.is(true) + ); + } + + @Test + void handlesNestedModulePaths(@TempDir final Path temp) + throws IOException { + final Path atv = temp.resolve("github.com/org/project/v2/@v"); + Files.createDirectories(atv); + Files.writeString( + atv.resolve("list"), + "v2.0.0\n", + StandardCharsets.UTF_8 + ); + Files.writeString( + atv.resolve("v2.0.0.info"), + "{\"Version\":\"v2.0.0\",\"Time\":\"2024-05-01T00:00:00Z\"}", + StandardCharsets.UTF_8 + ); + Files.write(atv.resolve("v2.0.0.zip"), new byte[500]); + final GoScanner scanner = new GoScanner(); + final List records = scanner.scan(temp, "go-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record for nested module", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Module path should be github.com/org/project/v2", + records.get(0).name(), + Matchers.is("github.com/org/project/v2") + ); + MatcherAssert.assertThat( + "Version should be 2.0.0 (v prefix stripped)", + records.get(0).version(), + Matchers.is("2.0.0") + ); + MatcherAssert.assertThat( + "Size should be 500", + records.get(0).size(), + Matchers.is(500L) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/HelmScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/HelmScannerTest.java new file mode 100644 index 000000000..413838a2f --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/HelmScannerTest.java @@ -0,0 +1,219 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link HelmScanner}. + * + * @since 1.20.13 + */ +final class HelmScannerTest { + + @Test + void scansMultipleChartsWithVersions(@TempDir final Path temp) + throws IOException { + Files.writeString( + temp.resolve("index.yaml"), + String.join( + "\n", + "apiVersion: v1", + "entries:", + " tomcat:", + " - name: tomcat", + " version: 0.4.1", + " urls:", + " - tomcat-0.4.1.tgz", + " created: '2021-01-11T16:21:01.376598500+03:00'", + " redis:", + " - name: redis", + " version: 7.0.0", + " urls:", + " - redis-7.0.0.tgz", + " created: '2023-05-01T10:00:00+00:00'", + " - name: redis", + " version: 6.2.0", + " urls:", + " - redis-6.2.0.tgz", + " created: '2022-03-15T08:30:00+00:00'" + ), + StandardCharsets.UTF_8 + ); + Files.write(temp.resolve("tomcat-0.4.1.tgz"), new byte[1024]); + Files.write(temp.resolve("redis-7.0.0.tgz"), new byte[2048]); + Files.write(temp.resolve("redis-6.2.0.tgz"), new byte[512]); + final HelmScanner scanner = new HelmScanner(); + final List records = scanner.scan(temp, "helm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 3 records total", + records, + Matchers.hasSize(3) + ); + MatcherAssert.assertThat( + "Should contain tomcat 0.4.1", + records.stream().anyMatch( + r -> "tomcat".equals(r.name()) && "0.4.1".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain redis 7.0.0", + records.stream().anyMatch( + r -> "redis".equals(r.name()) && "7.0.0".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain redis 6.2.0", + records.stream().anyMatch( + r -> "redis".equals(r.name()) && "6.2.0".equals(r.version()) + ), + Matchers.is(true) + ); + final ArtifactRecord tomcat = records.stream() + .filter(r -> "tomcat".equals(r.name())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "Tomcat size should be 1024", + tomcat.size(), + Matchers.is(1024L) + ); + final ArtifactRecord redis7 = records.stream() + .filter(r -> "7.0.0".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "Redis 7.0.0 size should be 2048", + redis7.size(), + Matchers.is(2048L) + ); + final ArtifactRecord redis6 = records.stream() + .filter(r -> "6.2.0".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "Redis 6.2.0 size should be 512", + redis6.size(), + Matchers.is(512L) + ); + MatcherAssert.assertThat( + "Repo type should be helm", + tomcat.repoType(), + Matchers.is("helm") + ); + } + + @Test + void handlesMissingTgzFile(@TempDir final Path temp) + throws IOException { + Files.writeString( + temp.resolve("index.yaml"), + String.join( + "\n", + "apiVersion: v1", + "entries:", + " nginx:", + " - name: nginx", + " version: 1.0.0", + " urls:", + " - nginx-1.0.0.tgz", + " created: '2023-01-01T00:00:00+00:00'" + ), + StandardCharsets.UTF_8 + ); + final HelmScanner scanner = new HelmScanner(); + final List records = scanner.scan(temp, "helm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should still produce 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Size should be 0 when tgz is missing", + records.get(0).size(), + Matchers.is(0L) + ); + } + + @Test + void handlesMissingIndexYaml(@TempDir final Path temp) + throws IOException { + final HelmScanner scanner = new HelmScanner(); + final List records = scanner.scan(temp, "helm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records when index.yaml is missing", + records, + Matchers.empty() + ); + } + + @Test + void handlesMissingEntriesKey(@TempDir final Path temp) + throws IOException { + Files.writeString( + temp.resolve("index.yaml"), + "apiVersion: v1\n", + StandardCharsets.UTF_8 + ); + final HelmScanner scanner = new HelmScanner(); + final List records = scanner.scan(temp, "helm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records when entries is missing", + records, + Matchers.empty() + ); + } + + @Test + void parsesCreatedTimestamp(@TempDir final Path temp) + throws IOException { + final String timestamp = "2021-01-11T16:21:01.376598500+03:00"; + final long expected = OffsetDateTime.parse( + timestamp, DateTimeFormatter.ISO_OFFSET_DATE_TIME + ).toInstant().toEpochMilli(); + Files.writeString( + temp.resolve("index.yaml"), + String.join( + "\n", + "apiVersion: v1", + "entries:", + " mychart:", + " - name: mychart", + " version: 1.0.0", + " urls:", + " - mychart-1.0.0.tgz", + " created: '" + timestamp + "'" + ), + StandardCharsets.UTF_8 + ); + final HelmScanner scanner = new HelmScanner(); + final List records = scanner.scan(temp, "helm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "CreatedDate should match the parsed timestamp", + records.get(0).createdDate(), + Matchers.is(expected) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/MavenScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/MavenScannerTest.java new file mode 100644 index 000000000..647a61a26 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/MavenScannerTest.java @@ -0,0 +1,357 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link MavenScanner}. + * + * @since 1.20.13 + */ +final class MavenScannerTest { + + @Test + void scansMultipleVersions(@TempDir final Path temp) throws IOException { + final Path v1 = temp.resolve("com/test/logger/1.0"); + final Path v2 = temp.resolve("com/test/logger/2.0"); + Files.createDirectories(v1); + Files.createDirectories(v2); + Files.write(v1.resolve("logger-1.0.jar"), new byte[100]); + Files.write(v1.resolve("logger-1.0.pom"), new byte[20]); + Files.write(v2.resolve("logger-2.0.jar"), new byte[200]); + Files.write(v2.resolve("logger-2.0.pom"), new byte[25]); + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "my-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 2 records", + records, + Matchers.hasSize(2) + ); + final ArtifactRecord first = records.stream() + .filter(r -> "1.0".equals(r.version())) + .findFirst() + .orElseThrow(); + MatcherAssert.assertThat( + "Name should be groupId.artifactId", + first.name(), + Matchers.is("com.test.logger") + ); + MatcherAssert.assertThat( + "Size should be JAR size (100), not POM", + first.size(), + Matchers.is(100L) + ); + MatcherAssert.assertThat( + "Repo type should be maven", + first.repoType(), + Matchers.is("maven") + ); + final ArtifactRecord second = records.stream() + .filter(r -> "2.0".equals(r.version())) + .findFirst() + .orElseThrow(); + MatcherAssert.assertThat( + "Size of version 2.0 jar should be 200", + second.size(), + Matchers.is(200L) + ); + } + + @Test + void handlesMultipleArtifacts(@TempDir final Path temp) + throws IOException { + final Path commonsDir = temp.resolve( + "org/apache/commons/commons-lang3/3.12.0" + ); + Files.createDirectories(commonsDir); + Files.write( + commonsDir.resolve("commons-lang3-3.12.0.jar"), new byte[50] + ); + Files.write( + commonsDir.resolve("commons-lang3-3.12.0.pom"), new byte[10] + ); + final Path guavaDir = temp.resolve("com/google/guava/guava/31.0"); + Files.createDirectories(guavaDir); + Files.write(guavaDir.resolve("guava-31.0.jar"), new byte[75]); + Files.write(guavaDir.resolve("guava-31.0.pom"), new byte[15]); + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "central") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find records from both artifacts", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain commons-lang3", + records.stream() + .anyMatch(r -> "org.apache.commons.commons-lang3".equals(r.name())), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain guava", + records.stream() + .anyMatch(r -> "com.google.guava.guava".equals(r.name())), + Matchers.is(true) + ); + } + + @Test + void handlesWarFile(@TempDir final Path temp) throws IOException { + final Path ver = temp.resolve("com/test/webapp/1.0"); + Files.createDirectories(ver); + Files.write(ver.resolve("webapp-1.0.war"), new byte[300]); + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find the war artifact", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "War file size should be 300", + records.get(0).size(), + Matchers.is(300L) + ); + } + + @Test + void gradleUsesCorrectRepoType(@TempDir final Path temp) + throws IOException { + final Path ver = temp.resolve("com/test/gradlelib/1.0"); + Files.createDirectories(ver); + Files.write(ver.resolve("gradlelib-1.0.jar"), new byte[50]); + final MavenScanner scanner = new MavenScanner("gradle"); + final List records = scanner.scan(temp, "repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce a record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Repo type should be gradle", + records.get(0).repoType(), + Matchers.is("gradle") + ); + } + + @Test + void skipsSidecarFiles(@TempDir final Path temp) throws IOException { + final Path dir = temp.resolve("uk/co/datumedge/hamcrest-json/0.2"); + Files.createDirectories(dir); + Files.write(dir.resolve("hamcrest-json-0.2.jar"), new byte[200]); + Files.write(dir.resolve("hamcrest-json-0.2.pom"), new byte[30]); + Files.writeString(dir.resolve("hamcrest-json-0.2.jar.sha1"), "hash"); + Files.writeString(dir.resolve("hamcrest-json-0.2.jar.sha256"), "hash"); + Files.writeString(dir.resolve("hamcrest-json-0.2.jar.md5"), "hash"); + Files.writeString( + dir.resolve("hamcrest-json-0.2.jar.artipie-meta.json"), "{}" + ); + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "proxy-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 deduplicated record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be uk.co.datumedge.hamcrest-json", + records.get(0).name(), + Matchers.is("uk.co.datumedge.hamcrest-json") + ); + MatcherAssert.assertThat( + "Size should be JAR size (200), not POM (30)", + records.get(0).size(), + Matchers.is(200L) + ); + } + + @Test + void handlesPomOnlyArtifacts(@TempDir final Path temp) + throws IOException { + final Path dir = temp.resolve( + "com/fasterxml/jackson/jackson-bom/3.0.1" + ); + Files.createDirectories(dir); + Files.write(dir.resolve("jackson-bom-3.0.1.pom"), new byte[80]); + Files.writeString(dir.resolve("jackson-bom-3.0.1.pom.sha1"), "hash"); + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "proxy-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find the POM-only artifact", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be com.fasterxml.jackson.jackson-bom", + records.get(0).name(), + Matchers.is("com.fasterxml.jackson.jackson-bom") + ); + MatcherAssert.assertThat( + "Size should be the POM size", + records.get(0).size(), + Matchers.is(80L) + ); + } + + @Test + void skipsMetadataXmlFiles(@TempDir final Path temp) + throws IOException { + // Plugin-level or artifact-level metadata files should not + // be indexed as artifacts themselves + final Path pluginDir = temp.resolve("com/example/maven/plugins"); + Files.createDirectories(pluginDir); + Files.writeString( + pluginDir.resolve("maven-metadata.xml"), + "" + ); + final Path artifactDir = temp.resolve( + "com/example/maven/plugins/my-plugin/1.0" + ); + Files.createDirectories(artifactDir); + Files.write(artifactDir.resolve("my-plugin-1.0.jar"), new byte[150]); + Files.write(artifactDir.resolve("my-plugin-1.0.pom"), new byte[20]); + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should only find the actual artifact, not the metadata XML", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Should be the plugin JAR", + records.get(0).name(), + Matchers.is("com.example.maven.plugins.my-plugin") + ); + } + + @Test + void handlesZipArtifactsWithSidecars(@TempDir final Path temp) + throws IOException { + final Path v1 = temp.resolve( + "com/auto1/aws/lambda/rackspace_swift_uploader_lambda/1.2.10" + ); + final Path v2 = temp.resolve( + "com/auto1/aws/lambda/rackspace_swift_uploader_lambda/1.2.10-beta" + ); + Files.createDirectories(v1); + Files.createDirectories(v2); + Files.write( + v1.resolve( + "rackspace_swift_uploader_lambda_1.2.10.zip" + ), new byte[400] + ); + Files.writeString( + v1.resolve( + "rackspace_swift_uploader_lambda_1.2.10.zip.md5" + ), "hash" + ); + Files.writeString( + v1.resolve( + "rackspace_swift_uploader_lambda_1.2.10.zip.sha1" + ), "hash" + ); + Files.writeString( + v1.resolve( + "rackspace_swift_uploader_lambda_1.2.10.zip.sha256" + ), "hash" + ); + Files.write( + v2.resolve( + "rackspace_swift_uploader_lambda_1.2.10-beta.zip" + ), new byte[350] + ); + Files.writeString( + v2.resolve( + "rackspace_swift_uploader_lambda_1.2.10-beta.zip.md5" + ), "hash" + ); + final MavenScanner scanner = new MavenScanner("gradle"); + final List records = scanner.scan(temp, "ops") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find 2 zip versions", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Name should be fully qualified", + records.get(0).name(), + Matchers.is( + "com.auto1.aws.lambda:rackspace_swift_uploader_lambda" + ) + ); + MatcherAssert.assertThat( + "Repo type should be gradle", + records.get(0).repoType(), + Matchers.is("gradle") + ); + final ArtifactRecord release = records.stream() + .filter(r -> "1.2.10".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "Release zip size should be 400", + release.size(), + Matchers.is(400L) + ); + final ArtifactRecord beta = records.stream() + .filter(r -> "1.2.10-beta".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "Beta zip size should be 350", + beta.size(), + Matchers.is(350L) + ); + } + + @Test + void returnsEmptyForEmptyDirectory(@TempDir final Path temp) + throws IOException { + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "empty") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Empty directory should produce no records", + records, + Matchers.empty() + ); + } + + @Test + void deduplicatesJarAndPom(@TempDir final Path temp) throws IOException { + final Path dir = temp.resolve("com/test/lib/1.0"); + Files.createDirectories(dir); + Files.write(dir.resolve("lib-1.0.jar"), new byte[500]); + Files.write(dir.resolve("lib-1.0.pom"), new byte[50]); + final MavenScanner scanner = new MavenScanner("maven"); + final List records = scanner.scan(temp, "repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "JAR + POM should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Size should be JAR (500), not POM (50)", + records.get(0).size(), + Matchers.is(500L) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/NpmScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/NpmScannerTest.java new file mode 100644 index 000000000..da41b7fc6 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/NpmScannerTest.java @@ -0,0 +1,475 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link NpmScanner}. + * + * @since 1.20.13 + */ +final class NpmScannerTest { + + @Test + void scansUnscopedPackageWithVersionsDir(@TempDir final Path temp) + throws IOException { + final Path pkg = temp.resolve("simple-modal-window"); + final Path versions = pkg.resolve(".versions"); + final Path tgzDir = pkg.resolve("-/@platform"); + Files.createDirectories(versions); + Files.createDirectories(tgzDir); + Files.writeString( + versions.resolve("0.0.2.json"), "{}", StandardCharsets.UTF_8 + ); + Files.writeString( + versions.resolve("0.0.3.json"), "{}", StandardCharsets.UTF_8 + ); + Files.write( + tgzDir.resolve("simple-modal-window-0.0.2.tgz"), + new byte[100] + ); + Files.write( + tgzDir.resolve("simple-modal-window-0.0.3.tgz"), + new byte[200] + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 2 records", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "All records should have name simple-modal-window", + records.stream().allMatch( + r -> "simple-modal-window".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 0.0.2", + records.stream().anyMatch( + r -> "0.0.2".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 0.0.3", + records.stream().anyMatch( + r -> "0.0.3".equals(r.version()) + ), + Matchers.is(true) + ); + final ArtifactRecord v2 = records.stream() + .filter(r -> "0.0.2".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "Size of 0.0.2 should be 100", + v2.size(), + Matchers.is(100L) + ); + final ArtifactRecord v3 = records.stream() + .filter(r -> "0.0.3".equals(r.version())) + .findFirst().orElseThrow(); + MatcherAssert.assertThat( + "Size of 0.0.3 should be 200", + v3.size(), + Matchers.is(200L) + ); + MatcherAssert.assertThat( + "Repo type should be npm", + v2.repoType(), + Matchers.is("npm") + ); + } + + @Test + void scansScopedPackageWithVersionsDir(@TempDir final Path temp) + throws IOException { + final Path pkg = temp.resolve("@ui-components/button"); + final Path versions = pkg.resolve(".versions"); + final Path tgzDir = pkg.resolve("-/@ui-components"); + Files.createDirectories(versions); + Files.createDirectories(tgzDir); + Files.writeString( + versions.resolve("0.1.8.json"), "{}", StandardCharsets.UTF_8 + ); + Files.write( + tgzDir.resolve("button-0.1.8.tgz"), new byte[50] + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record for scoped package", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be @ui-components/button", + records.get(0).name(), + Matchers.is("@ui-components/button") + ); + MatcherAssert.assertThat( + "Version should be 0.1.8", + records.get(0).version(), + Matchers.is("0.1.8") + ); + MatcherAssert.assertThat( + "Size should be 50", + records.get(0).size(), + Matchers.is(50L) + ); + } + + @Test + void handlesPreReleaseVersions(@TempDir final Path temp) + throws IOException { + final Path pkg = temp.resolve("ssu-popup"); + final Path versions = pkg.resolve(".versions"); + final Path tgzDir = pkg.resolve("-/@platform"); + Files.createDirectories(versions); + Files.createDirectories(tgzDir); + Files.writeString( + versions.resolve("0.0.1-dev.0.json"), "{}", + StandardCharsets.UTF_8 + ); + Files.writeString( + versions.resolve("0.0.1.json"), "{}", StandardCharsets.UTF_8 + ); + Files.writeString( + versions.resolve("1.0.1-dev.2.json"), "{}", + StandardCharsets.UTF_8 + ); + Files.write( + tgzDir.resolve("ssu-popup-0.0.1-dev.0.tgz"), new byte[30] + ); + Files.write( + tgzDir.resolve("ssu-popup-0.0.1.tgz"), new byte[40] + ); + Files.write( + tgzDir.resolve("ssu-popup-1.0.1-dev.2.tgz"), new byte[60] + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 3 records (including pre-release)", + records, + Matchers.hasSize(3) + ); + MatcherAssert.assertThat( + "Should contain 0.0.1-dev.0", + records.stream().anyMatch( + r -> "0.0.1-dev.0".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain 1.0.1-dev.2", + records.stream().anyMatch( + r -> "1.0.1-dev.2".equals(r.version()) + ), + Matchers.is(true) + ); + } + + @Test + void handlesMultiplePackages(@TempDir final Path temp) + throws IOException { + final Path pkg1 = temp.resolve("tracking"); + final Path pkg2 = temp.resolve("str-formatter"); + Files.createDirectories(pkg1.resolve(".versions")); + Files.createDirectories(pkg1.resolve("-/@platform")); + Files.createDirectories(pkg2.resolve(".versions")); + Files.createDirectories(pkg2.resolve("-/@platform")); + Files.writeString( + pkg1.resolve(".versions/0.0.1.json"), "{}", + StandardCharsets.UTF_8 + ); + Files.write( + pkg1.resolve("-/@platform/tracking-0.0.1.tgz"), new byte[80] + ); + Files.writeString( + pkg2.resolve(".versions/0.0.2.json"), "{}", + StandardCharsets.UTF_8 + ); + Files.write( + pkg2.resolve("-/@platform/str-formatter-0.0.2.tgz"), + new byte[90] + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find records from both packages", + records, + Matchers.hasSize(2) + ); + MatcherAssert.assertThat( + "Should contain tracking", + records.stream().anyMatch( + r -> "tracking".equals(r.name()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain str-formatter", + records.stream().anyMatch( + r -> "str-formatter".equals(r.name()) + ), + Matchers.is(true) + ); + } + + @Test + void handlesMissingTgzInVersionsMode(@TempDir final Path temp) + throws IOException { + final Path pkg = temp.resolve("no-tgz"); + Files.createDirectories(pkg.resolve(".versions")); + Files.writeString( + pkg.resolve(".versions/1.0.0.json"), "{}", + StandardCharsets.UTF_8 + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should still produce 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Size should be 0 when tgz is missing", + records.get(0).size(), + Matchers.is(0L) + ); + } + + @Test + void fallsBackToMetaJson(@TempDir final Path temp) throws IOException { + final Path pkgDir = temp.resolve("lodash"); + final Path tgzDir = temp.resolve("lodash/-"); + Files.createDirectories(tgzDir); + Files.write(tgzDir.resolve("lodash-4.17.21.tgz"), new byte[12345]); + Files.writeString( + pkgDir.resolve("meta.json"), + String.join( + "\n", + "{", + " \"name\": \"lodash\",", + " \"versions\": {", + " \"4.17.21\": {", + " \"name\": \"lodash\",", + " \"version\": \"4.17.21\",", + " \"dist\": {", + " \"tarball\": \"/lodash/-/lodash-4.17.21.tgz\"", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record via meta.json fallback", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be lodash", + records.get(0).name(), + Matchers.is("lodash") + ); + MatcherAssert.assertThat( + "Version should be 4.17.21", + records.get(0).version(), + Matchers.is("4.17.21") + ); + MatcherAssert.assertThat( + "Size should reflect the tarball", + records.get(0).size(), + Matchers.is(12345L) + ); + } + + @Test + void skipsUncachedVersionsInMetaJson(@TempDir final Path temp) + throws IOException { + // meta.json lists 3 versions but only 1.0.11 tarball is on disk + final Path pkgDir = temp.resolve("pako"); + final Path tgzDir = temp.resolve("pako/-"); + Files.createDirectories(tgzDir); + Files.write(tgzDir.resolve("pako-1.0.11.tgz"), new byte[98765]); + Files.writeString( + pkgDir.resolve("meta.json"), + String.join( + "\n", + "{", + " \"name\": \"pako\",", + " \"versions\": {", + " \"1.0.9\": {\"dist\":{\"tarball\":\"/pako/-/pako-1.0.9.tgz\"}},", + " \"1.0.10\": {\"dist\":{\"tarball\":\"/pako/-/pako-1.0.10.tgz\"}},", + " \"1.0.11\": {\"dist\":{\"tarball\":\"/pako/-/pako-1.0.11.tgz\"}}", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-proxy") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should only index the one cached version", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Only version 1.0.11 should be indexed", + records.get(0).version(), + Matchers.is("1.0.11") + ); + MatcherAssert.assertThat( + "Size should reflect the cached tarball", + records.get(0).size(), + Matchers.is(98765L) + ); + } + + @Test + void skipsMalformedMetaJson(@TempDir final Path temp) + throws IOException { + final Path pkgDir = temp.resolve("broken"); + Files.createDirectories(pkgDir); + Files.writeString( + pkgDir.resolve("meta.json"), + "<<>>", + StandardCharsets.UTF_8 + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Malformed JSON should produce 0 records", + records, + Matchers.empty() + ); + } + + @Test + void metaJsonUsesTimeField(@TempDir final Path temp) + throws IOException { + final String timestamp = "2023-06-15T12:30:00.000Z"; + final long expected = Instant.parse(timestamp).toEpochMilli(); + final Path pkgDir = temp.resolve("timed"); + final Path tgzDir = temp.resolve("timed/-"); + Files.createDirectories(tgzDir); + Files.write(tgzDir.resolve("timed-1.0.0.tgz"), new byte[100]); + Files.writeString( + pkgDir.resolve("meta.json"), + String.join( + "\n", + "{", + " \"name\": \"timed\",", + " \"versions\": {", + " \"1.0.0\": {", + " \"name\": \"timed\",", + " \"version\": \"1.0.0\",", + " \"dist\": {", + " \"tarball\": \"/timed/-/timed-1.0.0.tgz\"", + " }", + " }", + " },", + " \"time\": {", + " \"1.0.0\": \"" + timestamp + "\"", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "CreatedDate should match the parsed time field", + records.get(0).createdDate(), + Matchers.is(expected) + ); + } + + @Test + void scansScopedPackageWithMetaJson(@TempDir final Path temp) + throws IOException { + final Path pkgDir = temp.resolve("@hello/simple"); + final Path tgzDir = temp.resolve("@hello/simple/-"); + Files.createDirectories(tgzDir); + Files.write(tgzDir.resolve("simple-1.0.1.tgz"), new byte[200]); + Files.writeString( + pkgDir.resolve("meta.json"), + String.join( + "\n", + "{", + " \"name\": \"@hello/simple\",", + " \"versions\": {", + " \"1.0.1\": {", + " \"name\": \"@hello/simple\",", + " \"version\": \"1.0.1\",", + " \"dist\": {", + " \"tarball\": \"/@hello/simple/-/simple-1.0.1.tgz\"", + " }", + " }", + " }", + "}" + ), + StandardCharsets.UTF_8 + ); + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 1 record for scoped package via meta.json", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be @hello/simple", + records.get(0).name(), + Matchers.is("@hello/simple") + ); + } + + @Test + void returnsEmptyForEmptyDirectory(@TempDir final Path temp) + throws IOException { + final NpmScanner scanner = new NpmScanner(); + final List records = scanner.scan(temp, "npm-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Empty directory should produce no records", + records, + Matchers.empty() + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/ProgressReporterTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/ProgressReporterTest.java new file mode 100644 index 000000000..da4d2c196 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/ProgressReporterTest.java @@ -0,0 +1,95 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link ProgressReporter}. + * + * @since 1.20.13 + */ +final class ProgressReporterTest { + + @Test + void incrementIncrementsScannedCount() { + final ProgressReporter reporter = new ProgressReporter(1000); + reporter.increment(); + reporter.increment(); + reporter.increment(); + MatcherAssert.assertThat( + "Scanned count should reflect three increments", + reporter.getScanned(), + Matchers.is(3L) + ); + } + + @Test + void getScannedReturnsZeroInitially() { + final ProgressReporter reporter = new ProgressReporter(100); + MatcherAssert.assertThat( + "Initial scanned count should be zero", + reporter.getScanned(), + Matchers.is(0L) + ); + } + + @Test + void recordErrorIncrementsErrorCount() { + final ProgressReporter reporter = new ProgressReporter(100); + reporter.recordError(); + reporter.recordError(); + MatcherAssert.assertThat( + "Error count should reflect two errors", + reporter.getErrors(), + Matchers.is(2L) + ); + } + + @Test + void errorsStartAtZero() { + final ProgressReporter reporter = new ProgressReporter(100); + MatcherAssert.assertThat( + "Initial error count should be zero", + reporter.getErrors(), + Matchers.is(0L) + ); + } + + @Test + void incrementAndErrorsAreIndependent() { + final ProgressReporter reporter = new ProgressReporter(100); + reporter.increment(); + reporter.increment(); + reporter.recordError(); + MatcherAssert.assertThat( + "Scanned should be 2", + reporter.getScanned(), + Matchers.is(2L) + ); + MatcherAssert.assertThat( + "Errors should be 1", + reporter.getErrors(), + Matchers.is(1L) + ); + } + + @Test + void printFinalSummaryDoesNotThrow() { + final ProgressReporter reporter = new ProgressReporter(10); + for (int idx = 0; idx < 25; idx++) { + reporter.increment(); + } + reporter.recordError(); + reporter.printFinalSummary(); + MatcherAssert.assertThat( + "Scanned should be 25 after summary", + reporter.getScanned(), + Matchers.is(25L) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/PypiScannerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/PypiScannerTest.java new file mode 100644 index 000000000..a454cb494 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/PypiScannerTest.java @@ -0,0 +1,317 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link PypiScanner}. + * + * @since 1.20.13 + */ +final class PypiScannerTest { + + @Test + void parsesWheelFilename(@TempDir final Path temp) throws IOException { + final Path pkgDir = temp.resolve("my-package"); + Files.createDirectories(pkgDir); + Files.write( + pkgDir.resolve("my_package-1.0.0-py3-none-any.whl"), + new byte[50] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + final ArtifactRecord record = records.get(0); + MatcherAssert.assertThat( + "Name should be normalized to my-package", + record.name(), + Matchers.is("my-package") + ); + MatcherAssert.assertThat( + "Version should be 1.0.0", + record.version(), + Matchers.is("1.0.0") + ); + MatcherAssert.assertThat( + "Size should be 50", + record.size(), + Matchers.is(50L) + ); + MatcherAssert.assertThat( + "Repo type should be pypi", + record.repoType(), + Matchers.is("pypi") + ); + } + + @Test + void parsesSdistTarGz(@TempDir final Path temp) throws IOException { + final Path pkgDir = temp.resolve("requests"); + Files.createDirectories(pkgDir); + Files.write( + pkgDir.resolve("requests-2.28.0.tar.gz"), + new byte[100] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + final ArtifactRecord record = records.get(0); + MatcherAssert.assertThat( + "Name should be requests", + record.name(), + Matchers.is("requests") + ); + MatcherAssert.assertThat( + "Version should be 2.28.0", + record.version(), + Matchers.is("2.28.0") + ); + } + + @Test + void parsesSdistZip(@TempDir final Path temp) throws IOException { + final Path pkgDir = temp.resolve("foo"); + Files.createDirectories(pkgDir); + Files.write( + pkgDir.resolve("foo-3.0.zip"), + new byte[75] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + final ArtifactRecord record = records.get(0); + MatcherAssert.assertThat( + "Name should be foo", + record.name(), + Matchers.is("foo") + ); + MatcherAssert.assertThat( + "Version should be 3.0", + record.version(), + Matchers.is("3.0") + ); + MatcherAssert.assertThat( + "Size should be 75", + record.size(), + Matchers.is(75L) + ); + } + + @Test + void normalizesPackageName(@TempDir final Path temp) + throws IOException { + final Path pkgDir = temp.resolve("My_Package"); + Files.createDirectories(pkgDir); + Files.write( + pkgDir.resolve("My_Package-2.0.0-py3-none-any.whl"), + new byte[30] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce exactly 1 record", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be normalized to my-package", + records.get(0).name(), + Matchers.is("my-package") + ); + } + + @Test + void skipsNonConformingFilenames(@TempDir final Path temp) + throws IOException { + final Path dataDir = temp.resolve("data"); + Files.createDirectories(dataDir); + Files.writeString(dataDir.resolve("readme.txt"), "hello"); + Files.writeString(dataDir.resolve("notes.md"), "notes"); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 0 records for non-conforming files", + records, + Matchers.empty() + ); + } + + @Test + void handlesMultipleVersions(@TempDir final Path temp) + throws IOException { + final Path pkgDir = temp.resolve("flask"); + Files.createDirectories(pkgDir); + Files.write( + pkgDir.resolve("flask-2.0.0-py3-none-any.whl"), + new byte[40] + ); + Files.write( + pkgDir.resolve("flask-2.1.0.tar.gz"), + new byte[60] + ); + Files.write( + pkgDir.resolve("flask-2.2.0.zip"), + new byte[80] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should produce 3 records for multiple versions", + records, + Matchers.hasSize(3) + ); + MatcherAssert.assertThat( + "Should contain version 2.0.0", + records.stream().anyMatch( + r -> "2.0.0".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 2.1.0", + records.stream().anyMatch( + r -> "2.1.0".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 2.2.0", + records.stream().anyMatch( + r -> "2.2.0".equals(r.version()) + ), + Matchers.is(true) + ); + } + + @Test + void skipsHiddenFiles(@TempDir final Path temp) throws IOException { + final Path pkgDir = temp.resolve("hidden"); + Files.createDirectories(pkgDir); + Files.write( + pkgDir.resolve(".hidden-1.0.0.tar.gz"), + new byte[20] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should skip hidden files", + records, + Matchers.empty() + ); + } + + @Test + void scansVersionedSubdirectoryLayout(@TempDir final Path temp) + throws IOException { + // Real Artipie PyPI layout: package-name/version/file + final Path v100 = temp.resolve("dnssec-validator/1.0.0"); + final Path v101 = temp.resolve("dnssec-validator/1.0.1"); + Files.createDirectories(v100); + Files.createDirectories(v101); + Files.write( + v100.resolve("dnssec_validator-1.0.0-py3-none-any.whl"), + new byte[30] + ); + Files.write( + v100.resolve("dnssec_validator-1.0.0.tar.gz"), + new byte[40] + ); + Files.write( + v101.resolve("dnssec_validator-1.0.1-py3-none-any.whl"), + new byte[50] + ); + Files.write( + v101.resolve("dnssec_validator-1.0.1.tar.gz"), + new byte[60] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should find all 4 files in versioned subdirs", + records, + Matchers.hasSize(4) + ); + MatcherAssert.assertThat( + "Should contain version 1.0.0", + records.stream().anyMatch( + r -> "1.0.0".equals(r.version()) + ), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Should contain version 1.0.1", + records.stream().anyMatch( + r -> "1.0.1".equals(r.version()) + ), + Matchers.is(true) + ); + } + + @Test + void skipsHiddenDirectories(@TempDir final Path temp) + throws IOException { + // Real Artipie layout has .meta and .pypi hidden dirs + final Path metaDir = temp.resolve(".meta/pypi/shards/pkg/1.0.0"); + final Path pypiDir = temp.resolve(".pypi/pkg"); + final Path realDir = temp.resolve("pkg/1.0.0"); + Files.createDirectories(metaDir); + Files.createDirectories(pypiDir); + Files.createDirectories(realDir); + Files.write( + metaDir.resolve("pkg-1.0.0-py3-none-any.whl.json"), + "{}".getBytes() + ); + Files.writeString( + pypiDir.resolve("pkg.html"), + "" + ); + Files.write( + realDir.resolve("pkg-1.0.0-py3-none-any.whl"), + new byte[25] + ); + final PypiScanner scanner = new PypiScanner(); + final List records = scanner.scan(temp, "pypi-repo") + .collect(Collectors.toList()); + MatcherAssert.assertThat( + "Should only find the real whl, not files in hidden dirs", + records, + Matchers.hasSize(1) + ); + MatcherAssert.assertThat( + "Name should be pkg", + records.get(0).name(), + Matchers.is("pkg") + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/RepoConfigYamlTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/RepoConfigYamlTest.java new file mode 100644 index 000000000..4134cf7f2 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/RepoConfigYamlTest.java @@ -0,0 +1,137 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link RepoConfigYaml}. + * + * @since 1.20.13 + */ +final class RepoConfigYamlTest { + + /** + * Happy path: a well-formed config file is parsed correctly. + * Repo name is derived from the filename stem; rawType from repo.type. + * + * @param tmp JUnit temp directory + * @throws IOException if file creation fails + */ + @Test + void parsesValidConfig(@TempDir final Path tmp) throws IOException { + final Path file = tmp.resolve("go.yaml"); + Files.writeString(file, "repo:\n type: go\n"); + final RepoEntry entry = RepoConfigYaml.parse(file); + MatcherAssert.assertThat( + "repoName should be the filename stem", + entry.repoName(), + Matchers.is("go") + ); + MatcherAssert.assertThat( + "rawType should match repo.type in YAML", + entry.rawType(), + Matchers.is("go") + ); + } + + /** + * Proxy type is preserved as-is (normalisation is done by RepoTypeNormalizer). + * + * @param tmp JUnit temp directory + * @throws IOException if file creation fails + */ + @Test + void parsesProxyType(@TempDir final Path tmp) throws IOException { + final Path file = tmp.resolve("docker_proxy.yaml"); + Files.writeString(file, "repo:\n type: docker-proxy\n"); + final RepoEntry entry = RepoConfigYaml.parse(file); + MatcherAssert.assertThat( + "rawType should be preserved without normalisation", + entry.rawType(), + Matchers.is("docker-proxy") + ); + MatcherAssert.assertThat( + "repoName should match filename stem", + entry.repoName(), + Matchers.is("docker_proxy") + ); + } + + /** + * Missing {@code repo.type} key must throw {@link IOException}. + * + * @param tmp JUnit temp directory + * @throws IOException if file creation fails + */ + @Test + void throwsWhenRepoTypeMissing(@TempDir final Path tmp) throws IOException { + final Path file = tmp.resolve("bad.yaml"); + Files.writeString(file, "repo:\n storage:\n type: fs\n"); + Assertions.assertThrows( + IOException.class, + () -> RepoConfigYaml.parse(file), + "Missing repo.type should throw IOException" + ); + } + + /** + * Malformed YAML (not parseable) must throw {@link IOException}. + * + * @param tmp JUnit temp directory + * @throws IOException if file creation fails + */ + @Test + void throwsOnMalformedYaml(@TempDir final Path tmp) throws IOException { + final Path file = tmp.resolve("broken.yaml"); + Files.writeString(file, "repo: [\nunclosed bracket\n"); + Assertions.assertThrows( + IOException.class, + () -> RepoConfigYaml.parse(file), + "Malformed YAML should throw IOException" + ); + } + + /** + * Empty YAML file must throw {@link IOException}. + * + * @param tmp JUnit temp directory + * @throws IOException if file creation fails + */ + @Test + void throwsOnEmptyFile(@TempDir final Path tmp) throws IOException { + final Path file = tmp.resolve("empty.yaml"); + Files.writeString(file, ""); + Assertions.assertThrows( + IOException.class, + () -> RepoConfigYaml.parse(file), + "Empty YAML should throw IOException" + ); + } + + /** + * YAML with additional fields alongside repo.type parses without error. + * + * @param tmp JUnit temp directory + * @throws IOException if file creation fails + */ + @Test + void toleratesExtraFields(@TempDir final Path tmp) throws IOException { + final Path file = tmp.resolve("npm.yaml"); + Files.writeString( + file, + "repo:\n type: npm\n url: http://example.com\n storage:\n type: fs\n path: /data\n" + ); + final RepoEntry entry = RepoConfigYaml.parse(file); + MatcherAssert.assertThat(entry.rawType(), Matchers.is("npm")); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/RepoTypeNormalizerTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/RepoTypeNormalizerTest.java new file mode 100644 index 000000000..bcde8c80f --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/RepoTypeNormalizerTest.java @@ -0,0 +1,37 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +/** + * Tests for {@link RepoTypeNormalizer}. + * + * @since 1.20.13 + */ +final class RepoTypeNormalizerTest { + + @ParameterizedTest + @CsvSource({ + "docker-proxy, docker", + "npm-proxy, npm", + "maven-proxy, maven", + "go-proxy, go", + "maven, maven", + "docker, docker", + "file, file", + "go, go" + }) + void normalizesType(final String raw, final String expected) { + MatcherAssert.assertThat( + String.format("normalize('%s') should return '%s'", raw, expected), + RepoTypeNormalizer.normalize(raw), + Matchers.is(expected.trim()) + ); + } +} diff --git a/artipie-backfill/src/test/java/com/artipie/backfill/ScannerFactoryTest.java b/artipie-backfill/src/test/java/com/artipie/backfill/ScannerFactoryTest.java new file mode 100644 index 000000000..c72475f91 --- /dev/null +++ b/artipie-backfill/src/test/java/com/artipie/backfill/ScannerFactoryTest.java @@ -0,0 +1,99 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.backfill; + +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Tests for {@link ScannerFactory}. + * + * @since 1.20.13 + */ +final class ScannerFactoryTest { + + @ParameterizedTest + @ValueSource(strings = { + "maven", "gradle", "docker", "npm", "pypi", + "go", "helm", "composer", "php", "file", + "deb", "debian", "gem", "gems", + "maven-proxy", "gradle-proxy", "docker-proxy", + "npm-proxy", "pypi-proxy", "go-proxy", + "helm-proxy", "php-proxy", "file-proxy", + "deb-proxy", "debian-proxy", "gem-proxy" + }) + void createsNonNullScannerForKnownTypes(final String type) { + MatcherAssert.assertThat( + String.format("Scanner for type '%s' must not be null", type), + ScannerFactory.create(type), + Matchers.notNullValue() + ); + } + + @ParameterizedTest + @ValueSource(strings = { + "MAVEN", "Docker", "NPM", "PyPi", "HELM" + }) + void handlesUpperCaseTypes(final String type) { + MatcherAssert.assertThat( + String.format("Scanner for type '%s' (case-insensitive) must not be null", type), + ScannerFactory.create(type), + Matchers.notNullValue() + ); + } + + @ParameterizedTest + @ValueSource(strings = {"unknown", "svn", ""}) + void throwsForUnknownType(final String type) { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ScannerFactory.create(type), + String.format("Expected IllegalArgumentException for unknown type '%s'", type) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"maven", "gradle"}) + void mavenAndGradleReturnMavenScanner(final String type) { + MatcherAssert.assertThat( + String.format("Type '%s' should produce a MavenScanner", type), + ScannerFactory.create(type), + Matchers.instanceOf(MavenScanner.class) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"composer", "php"}) + void composerAndPhpReturnComposerScanner(final String type) { + MatcherAssert.assertThat( + String.format("Type '%s' should produce a ComposerScanner", type), + ScannerFactory.create(type), + Matchers.instanceOf(ComposerScanner.class) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"deb", "debian"}) + void debAndDebianReturnDebianScanner(final String type) { + MatcherAssert.assertThat( + String.format("Type '%s' should produce a DebianScanner", type), + ScannerFactory.create(type), + Matchers.instanceOf(DebianScanner.class) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"gem", "gems"}) + void gemAndGemsReturnGemScanner(final String type) { + MatcherAssert.assertThat( + String.format("Type '%s' should produce a GemScanner", type), + ScannerFactory.create(type), + Matchers.instanceOf(GemScanner.class) + ); + } +} diff --git a/artipie-core/pom.xml b/artipie-core/pom.xml index b110aca61..44d0e2a5d 100644 --- a/artipie-core/pom.xml +++ b/artipie-core/pom.xml @@ -6,11 +6,11 @@ com.artipie artipie - 1.20.12 + 1.22.0 artipie-core - 1.20.12 + 1.22.0 jar ${project.basedir}/../LICENSE.header @@ -19,7 +19,7 @@ com.artipie asto-core - 1.20.12 + 1.22.0 compile @@ -94,6 +94,13 @@ 6.4.0.RELEASE compile + + + org.apache.commons + commons-pool2 + 2.12.0 + compile + com.fasterxml.jackson.core diff --git a/artipie-core/src/main/java/com/artipie/asto/dedup/ContentAddressableStorage.java b/artipie-core/src/main/java/com/artipie/asto/dedup/ContentAddressableStorage.java new file mode 100644 index 000000000..3769a1a9d --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/asto/dedup/ContentAddressableStorage.java @@ -0,0 +1,132 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.asto.dedup; + +import com.artipie.asto.Key; +import com.artipie.asto.Storage; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Content-addressable storage layer for deduplication. + * Stores blobs by SHA-256 hash with reference counting. + * + * @since 1.20.13 + */ +public final class ContentAddressableStorage { + + /** + * Blob reference counts: sha256 -> ref count. + */ + private final ConcurrentMap refCounts; + + /** + * Artifact-to-blob mapping: "repoName::path" -> sha256. + */ + private final ConcurrentMap artifactBlobs; + + /** + * Underlying storage for actual blob content. + */ + private final Storage storage; + + /** + * Ctor. + * @param storage Underlying storage for blobs + */ + public ContentAddressableStorage(final Storage storage) { + this.storage = Objects.requireNonNull(storage, "storage"); + this.refCounts = new ConcurrentHashMap<>(); + this.artifactBlobs = new ConcurrentHashMap<>(); + } + + /** + * Save content with deduplication. + * If the same SHA-256 already exists, increment ref count instead of storing again. + * + * @param repoName Repository name + * @param path Artifact path + * @param sha256 SHA-256 hash of the content + * @param content Content bytes + * @return Future completing when saved + */ + public CompletableFuture save( + final String repoName, final String path, + final String sha256, final byte[] content + ) { + final String artKey = artKey(repoName, path); + // Remove old mapping if exists + final String oldSha = this.artifactBlobs.put(artKey, sha256); + if (oldSha != null && !oldSha.equals(sha256)) { + this.decrementRef(oldSha); + } + // Increment ref count + this.refCounts.computeIfAbsent(sha256, k -> new AtomicLong(0)).incrementAndGet(); + // Store blob if new + final Key blobKey = blobKey(sha256); + return this.storage.exists(blobKey).thenCompose(exists -> { + if (exists) { + return CompletableFuture.completedFuture(null); + } + return this.storage.save(blobKey, new com.artipie.asto.Content.From(content)) + .toCompletableFuture(); + }); + } + + /** + * Delete an artifact reference. + * Decrements ref count and removes blob if zero. + * + * @param repoName Repository name + * @param path Artifact path + * @return Future completing when deleted + */ + public CompletableFuture delete(final String repoName, final String path) { + final String sha = this.artifactBlobs.remove(artKey(repoName, path)); + if (sha == null) { + return CompletableFuture.completedFuture(null); + } + return this.decrementRef(sha); + } + + /** + * Get the ref count for a blob. + * @param sha256 SHA-256 hash + * @return Reference count, 0 if not found + */ + public long refCount(final String sha256) { + final AtomicLong count = this.refCounts.get(sha256); + return count != null ? count.get() : 0; + } + + /** + * Decrement ref count, delete blob if zero. + */ + private CompletableFuture decrementRef(final String sha256) { + final AtomicLong count = this.refCounts.get(sha256); + if (count != null && count.decrementAndGet() <= 0) { + this.refCounts.remove(sha256); + return this.storage.delete(blobKey(sha256)).toCompletableFuture(); + } + return CompletableFuture.completedFuture(null); + } + + private static String artKey(final String repoName, final String path) { + return repoName + "::" + path; + } + + private static Key blobKey(final String sha256) { + return new Key.From( + ".cas", + sha256.substring(0, 2), + sha256.substring(2, 4), + sha256 + ); + } +} diff --git a/artipie-core/src/main/java/com/artipie/cache/CacheInvalidationPubSub.java b/artipie-core/src/main/java/com/artipie/cache/CacheInvalidationPubSub.java new file mode 100644 index 000000000..2a62834a0 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/cache/CacheInvalidationPubSub.java @@ -0,0 +1,184 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cache; + +import com.artipie.asto.misc.Cleanable; +import com.artipie.http.log.EcsLogger; +import io.lettuce.core.pubsub.RedisPubSubAdapter; +import io.lettuce.core.pubsub.StatefulRedisPubSubConnection; +import io.lettuce.core.pubsub.api.async.RedisPubSubAsyncCommands; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Redis/Valkey pub/sub channel for cross-instance cache invalidation. + *

+ * When multiple Artipie instances share a Valkey/Redis server, local + * Caffeine caches can become stale when another instance modifies data. + * This class uses Redis pub/sub to broadcast invalidation messages so + * all instances stay in sync. + *

+ * Each instance generates a unique {@code instanceId} on startup. + * Messages published by this instance are ignored on receipt to avoid + * invalidating caches that were already updated locally. + *

+ * Message format: {@code instanceId|cacheType|key} + *
+ * For invalidateAll: {@code instanceId|cacheType|*} + * + * @since 1.20.13 + */ +public final class CacheInvalidationPubSub implements AutoCloseable { + + /** + * Redis channel name for cache invalidation messages. + */ + static final String CHANNEL = "artipie:cache:invalidate"; + + /** + * Wildcard key used for invalidateAll messages. + */ + private static final String ALL = "*"; + + /** + * Message field separator. + */ + private static final String SEP = "|"; + + /** + * Unique instance identifier to filter out self-messages. + */ + private final String instanceId; + + /** + * Connection for subscribing (receiving messages). + */ + private final StatefulRedisPubSubConnection subConn; + + /** + * Connection for publishing (sending messages). + * Pub/sub spec requires separate connections for sub and pub. + */ + private final StatefulRedisPubSubConnection pubConn; + + /** + * Async publish commands. + */ + private final RedisPubSubAsyncCommands pubCommands; + + /** + * Registered cache handlers keyed by cache type name. + */ + private final Map> caches; + + /** + * Ctor. + * @param valkey Valkey connection to create pub/sub connections from + */ + public CacheInvalidationPubSub(final ValkeyConnection valkey) { + this.instanceId = UUID.randomUUID().toString(); + this.subConn = valkey.connectPubSub(); + this.pubConn = valkey.connectPubSub(); + this.pubCommands = this.pubConn.async(); + this.caches = new ConcurrentHashMap<>(); + this.subConn.addListener(new Listener()); + this.subConn.async().subscribe(CacheInvalidationPubSub.CHANNEL); + EcsLogger.info("com.artipie.cache") + .message("Cache invalidation pub/sub started (instance: " + + this.instanceId.substring(0, 8) + ")") + .eventCategory("cache") + .eventAction("pubsub_start") + .eventOutcome("success") + .log(); + } + + /** + * Register a cache for remote invalidation. + * @param name Cache type name (e.g. "auth", "filters", "policy") + * @param cache Cache instance to invalidate on remote messages + */ + public void register(final String name, final Cleanable cache) { + this.caches.put(name, cache); + } + + /** + * Publish an invalidation message for a specific key. + * Other instances will call {@code cache.invalidate(key)} on receipt. + * @param cacheType Cache type name + * @param key Cache key to invalidate + */ + public void publish(final String cacheType, final String key) { + final String msg = String.join( + CacheInvalidationPubSub.SEP, this.instanceId, cacheType, key + ); + this.pubCommands.publish(CacheInvalidationPubSub.CHANNEL, msg); + } + + /** + * Publish an invalidateAll message. + * Other instances will call {@code cache.invalidateAll()} on receipt. + * @param cacheType Cache type name + */ + public void publishAll(final String cacheType) { + final String msg = String.join( + CacheInvalidationPubSub.SEP, this.instanceId, cacheType, + CacheInvalidationPubSub.ALL + ); + this.pubCommands.publish(CacheInvalidationPubSub.CHANNEL, msg); + } + + @Override + public void close() { + this.subConn.close(); + this.pubConn.close(); + EcsLogger.info("com.artipie.cache") + .message("Cache invalidation pub/sub closed") + .eventCategory("cache") + .eventAction("pubsub_stop") + .eventOutcome("success") + .log(); + } + + /** + * Listener that receives pub/sub messages and dispatches to caches. + */ + private final class Listener extends RedisPubSubAdapter { + @Override + public void message(final String channel, final String message) { + if (!CacheInvalidationPubSub.CHANNEL.equals(channel)) { + return; + } + final String[] parts = message.split( + "\\" + CacheInvalidationPubSub.SEP, 3 + ); + if (parts.length < 3) { + return; + } + final String sender = parts[0]; + if (CacheInvalidationPubSub.this.instanceId.equals(sender)) { + return; + } + final String cacheType = parts[1]; + final String key = parts[2]; + final Cleanable cache = + CacheInvalidationPubSub.this.caches.get(cacheType); + if (cache == null) { + return; + } + if (CacheInvalidationPubSub.ALL.equals(key)) { + cache.invalidateAll(); + } else { + cache.invalidate(key); + } + EcsLogger.debug("com.artipie.cache") + .message("Remote cache invalidation: " + cacheType + ":" + key) + .eventCategory("cache") + .eventAction("remote_invalidate") + .eventOutcome("success") + .log(); + } + } +} diff --git a/artipie-core/src/main/java/com/artipie/cache/PublishingCleanable.java b/artipie-core/src/main/java/com/artipie/cache/PublishingCleanable.java new file mode 100644 index 000000000..0e1346ea8 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/cache/PublishingCleanable.java @@ -0,0 +1,67 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cache; + +import com.artipie.asto.misc.Cleanable; + +/** + * Decorator that broadcasts cache invalidation to other Artipie instances + * via Redis pub/sub, in addition to performing the local invalidation. + *

+ * When {@link #invalidate(String)} or {@link #invalidateAll()} is called, + * this wrapper: + *

    + *
  1. Invalidates the local cache (delegates to wrapped instance)
    2. + *
  2. Publishes a message to Redis so other instances invalidate too
    3. + *
+ *

+ * The {@link CacheInvalidationPubSub} subscriber registers the inner + * (unwrapped) cache, so remote messages bypass this decorator and don't + * re-publish — preventing infinite loops. + * + * @since 1.20.13 + */ +public final class PublishingCleanable implements Cleanable { + + /** + * Inner cache to delegate to. + */ + private final Cleanable inner; + + /** + * Pub/sub channel to publish invalidation messages. + */ + private final CacheInvalidationPubSub pubsub; + + /** + * Cache type name (e.g. "auth", "filters", "policy"). + */ + private final String cacheType; + + /** + * Ctor. + * @param inner Local cache to wrap + * @param pubsub Redis pub/sub channel + * @param cacheType Cache type identifier + */ + public PublishingCleanable(final Cleanable inner, + final CacheInvalidationPubSub pubsub, final String cacheType) { + this.inner = inner; + this.pubsub = pubsub; + this.cacheType = cacheType; + } + + @Override + public void invalidate(final String key) { + this.inner.invalidate(key); + this.pubsub.publish(this.cacheType, key); + } + + @Override + public void invalidateAll() { + this.inner.invalidateAll(); + this.pubsub.publishAll(this.cacheType); + } +} diff --git a/artipie-core/src/main/java/com/artipie/cache/StoragesCache.java b/artipie-core/src/main/java/com/artipie/cache/StoragesCache.java index 83a107055..1b6621776 100644 --- a/artipie-core/src/main/java/com/artipie/cache/StoragesCache.java +++ b/artipie-core/src/main/java/com/artipie/cache/StoragesCache.java @@ -18,6 +18,7 @@ import com.google.common.base.Strings; import org.apache.commons.lang3.NotImplementedException; import com.artipie.http.log.EcsLogger; +import com.artipie.http.misc.DispatchedStorage; import java.time.Duration; @@ -117,8 +118,10 @@ public Storage storage(final YamlMapping yaml) { // Direct storage without JfrStorage wrapper // JFR profiling removed - adds 2-10% overhead and bypassed by optimized slices // Request-level metrics still active via Vert.x HTTP - return StoragesLoader.STORAGES - .newObject(type, new Config.YamlStorageConfig(key)); + return new DispatchedStorage( + StoragesLoader.STORAGES + .newObject(type, new Config.YamlStorageConfig(key)) + ); } ); diff --git a/artipie-core/src/main/java/com/artipie/cache/ValkeyConnection.java b/artipie-core/src/main/java/com/artipie/cache/ValkeyConnection.java index 8c94c41e7..e2708fbb5 100644 --- a/artipie-core/src/main/java/com/artipie/cache/ValkeyConnection.java +++ b/artipie-core/src/main/java/com/artipie/cache/ValkeyConnection.java @@ -4,6 +4,7 @@ */ package com.artipie.cache; +import com.artipie.http.log.EcsLogger; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisURI; import io.lettuce.core.api.StatefulRedisConnection; @@ -11,34 +12,70 @@ import io.lettuce.core.codec.ByteArrayCodec; import io.lettuce.core.codec.RedisCodec; import io.lettuce.core.codec.StringCodec; +import io.lettuce.core.pubsub.StatefulRedisPubSubConnection; +import io.lettuce.core.support.ConnectionPoolSupport; import java.time.Duration; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; /** - * Valkey/Redis connection for L2 cache across Artipie. - * Shared connection used by all two-tier caches. - * Thread-safe, async operations. + * Valkey/Redis connection pool for L2 cache across Artipie. + * Uses Lettuce's built-in connection pooling backed by Apache Commons Pool2. + * Thread-safe, async operations with round-robin connection selection. * * @since 1.0 */ public final class ValkeyConnection implements AutoCloseable { + /** + * Default maximum total connections in the pool. + */ + private static final int DEFAULT_MAX_TOTAL = 8; + + /** + * Default maximum idle connections. + */ + private static final int DEFAULT_MAX_IDLE = 4; + + /** + * Default minimum idle connections. + */ + private static final int DEFAULT_MIN_IDLE = 2; + /** * Redis client. */ private final RedisClient client; /** - * Stateful connection. + * Connection pool. + */ + private final GenericObjectPool> pool; + + /** + * Pre-borrowed connections for round-robin async access. + * These connections stay borrowed for the lifetime of ValkeyConnection. */ - private final StatefulRedisConnection connection; + private final StatefulRedisConnection[] connections; /** - * Async commands interface. + * Async command interfaces corresponding to each connection. */ - private final RedisAsyncCommands async; + private final RedisAsyncCommands[] asyncCommands; + + /** + * Round-robin index for connection selection. + */ + private final AtomicInteger index; + + /** + * Number of active (pre-borrowed) connections. + */ + private final int poolSize; /** * Constructor from configuration. @@ -54,7 +91,7 @@ public ValkeyConnection(final CacheConfig config) { } /** - * Constructor with explicit parameters. + * Constructor with explicit parameters and default pool size. * * @param host Valkey/Redis host * @param port Valkey/Redis port @@ -64,6 +101,24 @@ public ValkeyConnection( final String host, final int port, final Duration timeout + ) { + this(host, port, timeout, ValkeyConnection.DEFAULT_MAX_TOTAL); + } + + /** + * Constructor with explicit parameters and custom pool size. + * + * @param host Valkey/Redis host + * @param port Valkey/Redis port + * @param timeout Request timeout + * @param size Number of connections in the pool + */ + @SuppressWarnings("unchecked") + public ValkeyConnection( + final String host, + final int port, + final Duration timeout, + final int size ) { this.client = RedisClient.create( RedisURI.builder() @@ -72,24 +127,37 @@ public ValkeyConnection( .withTimeout(timeout) .build() ); - // Use String keys and byte[] values final RedisCodec codec = RedisCodec.of( StringCodec.UTF8, ByteArrayCodec.INSTANCE ); - this.connection = this.client.connect(codec); - this.async = this.connection.async(); - // Enable pipelining for better throughput - this.async.setAutoFlushCommands(true); + final GenericObjectPoolConfig> config = + new GenericObjectPoolConfig<>(); + config.setMaxTotal(Math.max(size, ValkeyConnection.DEFAULT_MIN_IDLE)); + config.setMaxIdle(Math.min(ValkeyConnection.DEFAULT_MAX_IDLE, size)); + config.setMinIdle(ValkeyConnection.DEFAULT_MIN_IDLE); + config.setTestOnBorrow(true); + this.pool = ConnectionPoolSupport.createGenericObjectPool( + () -> this.client.connect(codec), + config + ); + this.poolSize = Math.max(size, ValkeyConnection.DEFAULT_MIN_IDLE); + this.connections = new StatefulRedisConnection[this.poolSize]; + this.asyncCommands = new RedisAsyncCommands[this.poolSize]; + this.index = new AtomicInteger(0); + this.initConnections(); } /** * Get async commands interface. + * Returns commands from a pool connection using round-robin selection, + * distributing load across multiple connections. * * @return Redis async commands */ public RedisAsyncCommands async() { - return this.async; + final int idx = Math.abs(this.index.getAndIncrement() % this.poolSize); + return this.asyncCommands[idx]; } /** @@ -102,12 +170,12 @@ public RedisAsyncCommands async() { @Deprecated public boolean ping() { try { - return "PONG".equals(this.async.ping().get()); - } catch (Exception e) { + return "PONG".equals(this.async().ping().get()); + } catch (final Exception ex) { return false; } } - + /** * Async ping to check connectivity (non-blocking). * Preferred over blocking ping() method. @@ -115,16 +183,71 @@ public boolean ping() { * @return Future with true if connected, false on timeout or error */ public CompletableFuture pingAsync() { - return this.async.ping() + return this.async().ping() .toCompletableFuture() .orTimeout(1000, TimeUnit.MILLISECONDS) .thenApply(pong -> "PONG".equals(pong)) .exceptionally(err -> false); } + /** + * Returns the number of connections in the pool. + * + * @return Pool size + */ + public int poolSize() { + return this.poolSize; + } + + /** + * Create a new pub/sub connection for subscribe/publish operations. + * Uses String codec for both keys and values (pub/sub channels are text). + *

+ * The caller is responsible for closing the returned connection. + * + * @return New pub/sub connection + * @since 1.20.13 + */ + public StatefulRedisPubSubConnection connectPubSub() { + return this.client.connectPubSub(); + } + @Override public void close() { - this.connection.close(); + for (int idx = 0; idx < this.poolSize; idx += 1) { + if (this.connections[idx] != null) { + try { + this.pool.returnObject(this.connections[idx]); + } catch (final Exception ex) { + EcsLogger.debug("com.artipie.cache") + .message("Failed to return connection to pool during close") + .error(ex) + .log(); + } + } + } + this.pool.close(); this.client.shutdown(); } + + /** + * Pre-borrow connections from the pool and set up async command interfaces. + */ + private void initConnections() { + for (int idx = 0; idx < this.poolSize; idx += 1) { + try { + this.connections[idx] = this.pool.borrowObject(); + this.asyncCommands[idx] = this.connections[idx].async(); + this.asyncCommands[idx].setAutoFlushCommands(true); + } catch (final Exception ex) { + throw new IllegalStateException( + String.format( + "Failed to initialize connection %d of %d in Valkey pool", + idx + 1, this.poolSize + ), + ex + ); + } + } + } } diff --git a/artipie-core/src/main/java/com/artipie/cluster/ClusterEventBus.java b/artipie-core/src/main/java/com/artipie/cluster/ClusterEventBus.java new file mode 100644 index 000000000..783be1cf7 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/cluster/ClusterEventBus.java @@ -0,0 +1,237 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cluster; + +import com.artipie.cache.ValkeyConnection; +import com.artipie.http.log.EcsLogger; +import io.lettuce.core.pubsub.RedisPubSubAdapter; +import io.lettuce.core.pubsub.StatefulRedisPubSubConnection; +import io.lettuce.core.pubsub.api.async.RedisPubSubAsyncCommands; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; + +/** + * Cross-instance event bus using Valkey pub/sub. + * Broadcasts events to all connected Artipie instances for HA clustering. + *

+ * Events are published as strings on Valkey channels with the naming + * convention {@code artipie:events:{topic}}. Each instance subscribes + * to channels of interest and dispatches received messages to all + * registered handlers for that topic. + *

+ * Each instance generates a unique identifier on startup. Messages + * published by the local instance are ignored on receipt to avoid + * double-processing events that were already handled locally. + *

+ * Message format on the wire: {@code instanceId|payload} + *

+ * Thread safety: this class is thread-safe. Handler lists use + * {@link CopyOnWriteArrayList} and topic subscriptions use + * {@link ConcurrentHashMap}. + * + * @since 1.20.13 + */ +public final class ClusterEventBus implements AutoCloseable { + + /** + * Channel prefix for all event bus topics. + */ + static final String CHANNEL_PREFIX = "artipie:events:"; + + /** + * Message field separator between instance ID and payload. + */ + private static final String SEP = "|"; + + /** + * Unique instance identifier to filter out self-published messages. + */ + private final String instanceId; + + /** + * Connection for subscribing (receiving messages). + */ + private final StatefulRedisPubSubConnection subConn; + + /** + * Connection for publishing (sending messages). + * Pub/sub spec requires separate connections for subscribe and publish. + */ + private final StatefulRedisPubSubConnection pubConn; + + /** + * Async publish commands. + */ + private final RedisPubSubAsyncCommands pubCommands; + + /** + * Registered handlers keyed by topic name. + * Each topic can have multiple handlers. + */ + private final Map>> handlers; + + /** + * Constructor. Sets up pub/sub connections and the message listener. + * + * @param valkey Valkey connection to create pub/sub connections from + */ + public ClusterEventBus(final ValkeyConnection valkey) { + this.instanceId = UUID.randomUUID().toString(); + this.subConn = valkey.connectPubSub(); + this.pubConn = valkey.connectPubSub(); + this.pubCommands = this.pubConn.async(); + this.handlers = new ConcurrentHashMap<>(); + this.subConn.addListener(new Dispatcher()); + EcsLogger.info("com.artipie.cluster") + .message( + "Cluster event bus started (instance: " + + this.instanceId.substring(0, 8) + ")" + ) + .eventCategory("cluster") + .eventAction("eventbus_start") + .eventOutcome("success") + .log(); + } + + /** + * Publish an event to a topic. + * The event will be broadcast to all Artipie instances subscribed + * to this topic. The publishing instance will ignore its own message. + * + * @param topic Topic name (e.g. "config.change", "repo.update") + * @param payload Event payload (typically JSON) + */ + public void publish(final String topic, final String payload) { + final String channel = ClusterEventBus.CHANNEL_PREFIX + topic; + final String message = String.join( + ClusterEventBus.SEP, this.instanceId, payload + ); + this.pubCommands.publish(channel, message); + EcsLogger.debug("com.artipie.cluster") + .message("Event published: " + topic) + .eventCategory("cluster") + .eventAction("event_publish") + .field("cluster.topic", topic) + .eventOutcome("success") + .log(); + } + + /** + * Subscribe a handler to a topic. + * The handler will be called with the event payload whenever a + * remote instance publishes to this topic. If this is the first + * handler for the topic, the Valkey channel subscription is created. + * + * @param topic Topic name (e.g. "config.change", "repo.update") + * @param handler Consumer that receives the event payload + */ + public void subscribe(final String topic, final Consumer handler) { + final String channel = ClusterEventBus.CHANNEL_PREFIX + topic; + final boolean firstHandler = !this.handlers.containsKey(topic); + this.handlers + .computeIfAbsent(topic, key -> new CopyOnWriteArrayList<>()) + .add(handler); + if (firstHandler) { + this.subConn.async().subscribe(channel); + EcsLogger.debug("com.artipie.cluster") + .message("Subscribed to topic: " + topic) + .eventCategory("cluster") + .eventAction("topic_subscribe") + .field("cluster.topic", topic) + .eventOutcome("success") + .log(); + } + } + + /** + * Returns the unique instance identifier for this event bus. + * + * @return Instance ID string + */ + public String instanceId() { + return this.instanceId; + } + + /** + * Returns the number of topics with active subscriptions. + * + * @return Number of subscribed topics + */ + public int topicCount() { + return this.handlers.size(); + } + + @Override + public void close() { + this.subConn.close(); + this.pubConn.close(); + EcsLogger.info("com.artipie.cluster") + .message("Cluster event bus closed") + .eventCategory("cluster") + .eventAction("eventbus_stop") + .eventOutcome("success") + .log(); + } + + /** + * Listener that receives Valkey pub/sub messages and dispatches + * them to registered topic handlers. + */ + private final class Dispatcher extends RedisPubSubAdapter { + @Override + public void message(final String channel, final String message) { + if (!channel.startsWith(ClusterEventBus.CHANNEL_PREFIX)) { + return; + } + final int sep = message.indexOf(ClusterEventBus.SEP); + if (sep < 0) { + return; + } + final String sender = message.substring(0, sep); + if (ClusterEventBus.this.instanceId.equals(sender)) { + return; + } + final String payload = message.substring(sep + 1); + final String topic = channel.substring( + ClusterEventBus.CHANNEL_PREFIX.length() + ); + final List> topicHandlers = + ClusterEventBus.this.handlers.get(topic); + if (topicHandlers == null || topicHandlers.isEmpty()) { + return; + } + for (final Consumer handler : topicHandlers) { + try { + handler.accept(payload); + } catch (final Exception ex) { + EcsLogger.error("com.artipie.cluster") + .message( + "Event handler failed for topic: " + topic + ) + .error(ex) + .eventCategory("cluster") + .eventAction("event_dispatch") + .field("cluster.topic", topic) + .eventOutcome("failure") + .log(); + } + } + EcsLogger.debug("com.artipie.cluster") + .message( + "Event dispatched: " + topic + " to " + + topicHandlers.size() + " handler(s)" + ) + .eventCategory("cluster") + .eventAction("event_dispatch") + .field("cluster.topic", topic) + .eventOutcome("success") + .log(); + } + } +} diff --git a/artipie-core/src/main/java/com/artipie/cooldown/CooldownCache.java b/artipie-core/src/main/java/com/artipie/cooldown/CooldownCache.java index 5676568f5..62f957fc1 100644 --- a/artipie-core/src/main/java/com/artipie/cooldown/CooldownCache.java +++ b/artipie-core/src/main/java/com/artipie/cooldown/CooldownCache.java @@ -9,6 +9,8 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Expiry; import io.lettuce.core.RedisFuture; +import io.lettuce.core.ScanArgs; +import io.lettuce.core.ScanCursor; import io.lettuce.core.api.async.RedisAsyncCommands; import java.nio.ByteBuffer; import java.time.Duration; @@ -366,14 +368,7 @@ public void unblockAll(final String repoName) { // L2: Pattern update (SCAN is expensive but unblockAll is rare) if (this.twoTier) { final String pattern = prefix + "*"; - this.l2.keys(pattern).thenAccept(keys -> { - if (keys != null && !keys.isEmpty()) { - for (final String key : keys) { - // Set each key to false with configured TTL - this.l2.setex(key, this.l2AllowedTtlSeconds, "false".getBytes()); - } - } - }); + this.scanAndUpdate(pattern); } } @@ -443,4 +438,30 @@ private void putL2Boolean(final String key, final boolean blocked, final long tt this.l2.setex(key, ttlSeconds, value); } + /** + * Scan and update keys matching pattern using cursor-based SCAN. + * Sets each matched key to "false" (allowed) with configured TTL. + * Avoids blocking KEYS command that freezes Redis on large datasets. + * @param pattern Redis key pattern (glob-style) + */ + private CompletableFuture scanAndUpdate(final String pattern) { + return this.scanAndUpdateStep(ScanCursor.INITIAL, pattern); + } + + private CompletableFuture scanAndUpdateStep( + final ScanCursor cursor, final String pattern + ) { + return this.l2.scan(cursor, ScanArgs.Builder.matches(pattern).limit(100)) + .toCompletableFuture() + .thenCompose(result -> { + for (final String key : result.getKeys()) { + this.l2.setex(key, this.l2AllowedTtlSeconds, "false".getBytes()); + } + if (result.isFinished()) { + return CompletableFuture.completedFuture(null); + } + return this.scanAndUpdateStep(result, pattern); + }); + } + } diff --git a/artipie-core/src/main/java/com/artipie/cooldown/CooldownSettings.java b/artipie-core/src/main/java/com/artipie/cooldown/CooldownSettings.java index 7e242ebdc..2f9586324 100644 --- a/artipie-core/src/main/java/com/artipie/cooldown/CooldownSettings.java +++ b/artipie-core/src/main/java/com/artipie/cooldown/CooldownSettings.java @@ -8,7 +8,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.Optional; /** * Global and per-repo-type cooldown configuration. @@ -23,21 +22,21 @@ public final class CooldownSettings { /** * Whether cooldown logic is enabled globally. */ - private final boolean enabled; + private volatile boolean enabled; /** * Minimum allowed age for an artifact release. If an artifact's release time * is within this window (i.e. too fresh), it will be blocked until it reaches * the minimum allowed age. */ - private final Duration minimumAllowedAge; + private volatile Duration minimumAllowedAge; /** * Per-repo-type overrides. * Key: repository type (maven, npm, docker, etc.) * Value: RepoTypeConfig with enabled flag and minimum age */ - private final Map repoTypeOverrides; + private volatile Map repoTypeOverrides; /** * Ctor with global settings only. @@ -108,6 +107,29 @@ public Duration minimumAllowedAgeFor(final String repoType) { return override != null ? override.minimumAllowedAge() : this.minimumAllowedAge; } + /** + * Get a copy of per-repo-type overrides. + * + * @return Map of repo type to config + */ + public Map repoTypeOverrides() { + return new HashMap<>(this.repoTypeOverrides); + } + + /** + * Update cooldown settings in-place for hot reload. + * + * @param newEnabled Whether cooldown is enabled + * @param newMinAge New global minimum allowed age + * @param overrides New per-repo-type overrides + */ + public void update(final boolean newEnabled, final Duration newMinAge, + final Map overrides) { + this.enabled = newEnabled; + this.minimumAllowedAge = Objects.requireNonNull(newMinAge); + this.repoTypeOverrides = new HashMap<>(Objects.requireNonNull(overrides)); + } + /** * Creates default configuration (enabled, 72 hours minimum allowed age). * diff --git a/artipie-core/src/main/java/com/artipie/cooldown/metadata/CooldownMetadataServiceImpl.java b/artipie-core/src/main/java/com/artipie/cooldown/metadata/CooldownMetadataServiceImpl.java index c30e200e7..64f7fd938 100644 --- a/artipie-core/src/main/java/com/artipie/cooldown/metadata/CooldownMetadataServiceImpl.java +++ b/artipie-core/src/main/java/com/artipie/cooldown/metadata/CooldownMetadataServiceImpl.java @@ -294,13 +294,13 @@ private CompletableFuture computeFilteredM } EcsLogger.debug("com.artipie.cooldown.metadata") - .message("Evaluating cooldown for versions") + .message(String.format( + "Evaluating cooldown for versions: %d total, %d to evaluate", + allVersions.size(), versionsToEvaluate.size())) .eventCategory("cooldown") .eventAction("metadata_filter") .field("repository.type", repoType) .field("package.name", packageName) - .field("versions.total", allVersions.size()) - .field("versions.evaluating", versionsToEvaluate.size()) .log(); return new FilterContext<>( @@ -349,12 +349,12 @@ private CompletableFuture evaluateAndFilte } EcsLogger.debug("com.artipie.cooldown.metadata") - .message("Cooldown evaluation complete") + .message(String.format( + "Cooldown evaluation complete: %d versions blocked", blockedVersions.size())) .eventCategory("cooldown") .eventAction("metadata_filter") .field("repository.type", ctx.repoType) .field("package.name", ctx.packageName) - .field("versions.blocked", blockedVersions.size()) .log(); // Note: Blocked versions gauge is updated by JdbcCooldownService on block/unblock @@ -383,12 +383,12 @@ private CompletableFuture evaluateAndFilte if (newLatest.isPresent()) { filtered = ctx.filter.updateLatest(filtered, newLatest.get()); EcsLogger.debug("com.artipie.cooldown.metadata") - .message("Updated latest version (by release date)") + .message(String.format( + "Updated latest version (by release date): %s -> %s", + currentLatest.get(), newLatest.get())) .eventCategory("cooldown") .eventAction("metadata_filter") .field("package.name", ctx.packageName) - .field("latest.old", currentLatest.get()) - .field("latest.new", newLatest.get()) .log(); } } @@ -399,15 +399,15 @@ private CompletableFuture evaluateAndFilte // Log performance final long durationMs = (System.nanoTime() - ctx.startTime) / 1_000_000; EcsLogger.info("com.artipie.cooldown.metadata") - .message("Metadata filtering complete") + .message(String.format( + "Metadata filtering complete: %d total versions, %d blocked", + ctx.allVersions.size(), blockedVersions.size())) .eventCategory("cooldown") .eventAction("metadata_filter") .eventOutcome("success") .field("repository.type", ctx.repoType) .field("package.name", ctx.packageName) - .field("versions.total", ctx.allVersions.size()) - .field("versions.blocked", blockedVersions.size()) - .field("duration_ms", durationMs) + .field("event.duration", durationMs * 1_000_000L) .log(); // Record metrics via CooldownMetrics @@ -501,10 +501,10 @@ private void preloadReleaseDates( if (!releaseDates.isEmpty()) { ((MetadataAwareInspector) inspector).preloadReleaseDates(releaseDates); EcsLogger.debug("com.artipie.cooldown.metadata") - .message("Preloaded release dates from metadata") + .message(String.format( + "Preloaded %d release dates from metadata", releaseDates.size())) .eventCategory("cooldown") .eventAction("metadata_filter") - .field("dates.count", releaseDates.size()) .log(); } } diff --git a/artipie-core/src/main/java/com/artipie/http/ResponseBuilder.java b/artipie-core/src/main/java/com/artipie/http/ResponseBuilder.java index 1aad820ad..dcd024617 100644 --- a/artipie-core/src/main/java/com/artipie/http/ResponseBuilder.java +++ b/artipie-core/src/main/java/com/artipie/http/ResponseBuilder.java @@ -100,6 +100,10 @@ public static ResponseBuilder rangeNotSatisfiable() { return new ResponseBuilder(RsStatus.REQUESTED_RANGE_NOT_SATISFIABLE); } + public static ResponseBuilder badGateway() { + return new ResponseBuilder(RsStatus.BAD_GATEWAY); + } + public static ResponseBuilder gatewayTimeout() { return new ResponseBuilder(RsStatus.GATEWAY_TIMEOUT); } @@ -245,6 +249,7 @@ public Response build() { case TOO_MANY_REQUESTS -> RSP_TOO_MANY_REQUESTS; case INTERNAL_ERROR -> RSP_INTERNAL_ERROR; case NOT_IMPLEMENTED -> RSP_NOT_IMPLEMENTED; + case BAD_GATEWAY -> RSP_BAD_GATEWAY; case SERVICE_UNAVAILABLE -> RSP_SERVICE_UNAVAILABLE; case PARTIAL_CONTENT -> RSP_PARTIAL_CONTENT; case GATEWAY_TIMEOUT -> RSP_GATEWAY_TIMEOUT; @@ -284,6 +289,7 @@ public CompletableFuture completedFuture() { private final static Response RSP_TOO_MANY_REQUESTS = new Response(RsStatus.TOO_MANY_REQUESTS, Headers.EMPTY, Content.EMPTY); private final static Response RSP_INTERNAL_ERROR = new Response(RsStatus.INTERNAL_ERROR, Headers.EMPTY, Content.EMPTY); private final static Response RSP_NOT_IMPLEMENTED = new Response(RsStatus.NOT_IMPLEMENTED, Headers.EMPTY, Content.EMPTY); + private final static Response RSP_BAD_GATEWAY = new Response(RsStatus.BAD_GATEWAY, Headers.EMPTY, Content.EMPTY); private final static Response RSP_SERVICE_UNAVAILABLE = new Response(RsStatus.SERVICE_UNAVAILABLE, Headers.EMPTY, Content.EMPTY); private final static Response RSP_PARTIAL_CONTENT = new Response(RsStatus.PARTIAL_CONTENT, Headers.EMPTY, Content.EMPTY); private final static Response RSP_GATEWAY_TIMEOUT = new Response(RsStatus.GATEWAY_TIMEOUT, Headers.EMPTY, Content.EMPTY); diff --git a/artipie-core/src/main/java/com/artipie/http/RsStatus.java b/artipie-core/src/main/java/com/artipie/http/RsStatus.java index edfff24e5..e27105dc8 100644 --- a/artipie-core/src/main/java/com/artipie/http/RsStatus.java +++ b/artipie-core/src/main/java/com/artipie/http/RsStatus.java @@ -120,6 +120,10 @@ public enum RsStatus { * Not Implemented. */ NOT_IMPLEMENTED(HttpStatus.SC_NOT_IMPLEMENTED), + /** + * Bad Gateway (502). + */ + BAD_GATEWAY(HttpStatus.SC_BAD_GATEWAY), /** * Service Unavailable. */ diff --git a/artipie-core/src/main/java/com/artipie/http/auth/AuthzSlice.java b/artipie-core/src/main/java/com/artipie/http/auth/AuthzSlice.java index 775761d31..e0aaf3424 100644 --- a/artipie-core/src/main/java/com/artipie/http/auth/AuthzSlice.java +++ b/artipie-core/src/main/java/com/artipie/http/auth/AuthzSlice.java @@ -11,6 +11,7 @@ import com.artipie.http.Slice; import com.artipie.http.headers.Header; import com.artipie.http.headers.WwwAuthenticate; +import com.artipie.http.log.EcsLogger; import com.artipie.http.rq.RequestLine; import org.slf4j.MDC; @@ -87,8 +88,11 @@ public CompletableFuture response( .header(new WwwAuthenticate(challenge)) .completedFuture(); } - } catch (final UnsupportedOperationException ignored) { - // fall through when scheme does not provide challenge + } catch (final UnsupportedOperationException ex) { + EcsLogger.debug("com.artipie.http.auth") + .message("Auth scheme does not provide challenge") + .error(ex) + .log(); } if (this.control.allowed(result.user())) { return this.origin.response( diff --git a/artipie-core/src/main/java/com/artipie/http/auth/CombinedAuthzSlice.java b/artipie-core/src/main/java/com/artipie/http/auth/CombinedAuthzSlice.java index 2a1d55b40..060dbe095 100644 --- a/artipie-core/src/main/java/com/artipie/http/auth/CombinedAuthzSlice.java +++ b/artipie-core/src/main/java/com/artipie/http/auth/CombinedAuthzSlice.java @@ -11,6 +11,7 @@ import com.artipie.http.headers.Authorization; import com.artipie.http.headers.Header; import com.artipie.http.headers.WwwAuthenticate; +import com.artipie.http.log.EcsLogger; import com.artipie.http.rq.RequestLine; import com.artipie.http.rq.RqHeaders; import com.artipie.http.trace.TraceContextExecutor; @@ -137,8 +138,11 @@ public CompletableFuture response( .header(new WwwAuthenticate(challenge)) .completedFuture(); } - } catch (final UnsupportedOperationException ignored) { - // fall through when scheme does not provide challenge + } catch (final UnsupportedOperationException ex) { + EcsLogger.debug("com.artipie.http.auth") + .message("Auth scheme does not provide challenge") + .error(ex) + .log(); } if (this.control.allowed(result.user())) { return this.origin.response( diff --git a/artipie-core/src/main/java/com/artipie/http/cache/BaseCachedProxySlice.java b/artipie-core/src/main/java/com/artipie/http/cache/BaseCachedProxySlice.java new file mode 100644 index 000000000..c6e5e28b9 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/cache/BaseCachedProxySlice.java @@ -0,0 +1,999 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import com.artipie.asto.Content; +import com.artipie.asto.Key; +import com.artipie.asto.Storage; +import com.artipie.asto.cache.Cache; +import com.artipie.asto.cache.CacheControl; +import com.artipie.asto.cache.Remote; +import com.artipie.cooldown.CooldownInspector; +import com.artipie.cooldown.CooldownRequest; +import com.artipie.cooldown.CooldownResponses; +import com.artipie.cooldown.CooldownResult; +import com.artipie.cooldown.CooldownService; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.RsStatus; +import com.artipie.http.Slice; +import com.artipie.http.headers.Header; +import com.artipie.http.headers.Login; +import com.artipie.http.log.EcsLogger; +import com.artipie.http.rq.RequestLine; +import com.artipie.http.slice.KeyFromPath; +import com.artipie.scheduling.ProxyArtifactEvent; + +import io.reactivex.Flowable; +import java.io.IOException; +import java.net.ConnectException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Abstract base class for all proxy adapter cache slices. + * + *

Implements the shared proxy flow via template method pattern: + *

    + *
  1. Check negative cache - fast-fail on known 404s
    2. + *
  2. Check local cache (offline-safe) - serve if fresh hit
    3. + *
  3. Evaluate cooldown - block if in cooldown period
    4. + *
  4. Deduplicate concurrent requests for same path
    5. + *
  5. Fetch from upstream
    6. + *
  6. On 200: cache content, compute digests, generate sidecars, enqueue event
    7. + *
  7. On 404: update negative cache
    8. + *
  8. Record metrics
    9. + *
+ * + *

Adapters override only the hooks they need: + * {@link #isCacheable(String)}, {@link #buildCooldownRequest(String, Headers)}, + * {@link #digestAlgorithms()}, {@link #buildArtifactEvent(Key, Headers, long, String)}, + * {@link #postProcess(Response, RequestLine)}, {@link #generateSidecars(String, Map)}. + * + * @since 1.20.13 + */ +@SuppressWarnings({"PMD.GodClass", "PMD.ExcessiveImports"}) +public abstract class BaseCachedProxySlice implements Slice { + + /** + * Upstream remote slice. + */ + private final Slice client; + + /** + * Asto cache for artifact storage. + */ + private final Cache cache; + + /** + * Repository name. + */ + private final String repoName; + + /** + * Repository type (e.g., "maven", "npm", "pypi"). + */ + private final String repoType; + + /** + * Upstream base URL for metrics. + */ + private final String upstreamUrl; + + /** + * Optional local storage for metadata and sidecars. + */ + private final Optional metadataStore; + + /** + * Whether cache is backed by persistent storage. + */ + private final boolean storageBacked; + + /** + * Event queue for proxy artifact events. + */ + private final Optional> events; + + /** + * Unified proxy configuration. + */ + private final ProxyCacheConfig config; + + /** + * Negative cache for 404 responses. + */ + private final NegativeCache negativeCache; + + /** + * Cooldown service (null if cooldown disabled). + */ + private final CooldownService cooldownService; + + /** + * Cooldown inspector (null if cooldown disabled). + */ + private final CooldownInspector cooldownInspector; + + /** + * Request deduplicator. + */ + private final RequestDeduplicator deduplicator; + + /** + * Raw storage for direct saves (bypasses FromStorageCache lazy tee-content). + */ + private final Optional storage; + + /** + * Constructor. + * + * @param client Upstream remote slice + * @param cache Asto cache for artifact storage + * @param repoName Repository name + * @param repoType Repository type + * @param upstreamUrl Upstream base URL + * @param storage Optional local storage + * @param events Event queue for proxy artifacts + * @param config Unified proxy configuration + * @param cooldownService Cooldown service (nullable, required if cooldown enabled) + * @param cooldownInspector Cooldown inspector (nullable, required if cooldown enabled) + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + protected BaseCachedProxySlice( + final Slice client, + final Cache cache, + final String repoName, + final String repoType, + final String upstreamUrl, + final Optional storage, + final Optional> events, + final ProxyCacheConfig config, + final CooldownService cooldownService, + final CooldownInspector cooldownInspector + ) { + this.client = Objects.requireNonNull(client, "client"); + this.cache = Objects.requireNonNull(cache, "cache"); + this.repoName = Objects.requireNonNull(repoName, "repoName"); + this.repoType = Objects.requireNonNull(repoType, "repoType"); + this.upstreamUrl = Objects.requireNonNull(upstreamUrl, "upstreamUrl"); + this.events = Objects.requireNonNull(events, "events"); + this.config = Objects.requireNonNull(config, "config"); + this.storage = storage; + this.metadataStore = storage.map(CachedArtifactMetadataStore::new); + this.storageBacked = this.metadataStore.isPresent() + && !Objects.equals(this.cache, Cache.NOP); + this.negativeCache = config.negativeCacheEnabled() + ? new NegativeCache(repoType, repoName) : null; + this.cooldownService = cooldownService; + this.cooldownInspector = cooldownInspector; + this.deduplicator = new RequestDeduplicator(config.dedupStrategy()); + } + + /** + * Convenience constructor without cooldown (for adapters that don't use it). + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + protected BaseCachedProxySlice( + final Slice client, + final Cache cache, + final String repoName, + final String repoType, + final String upstreamUrl, + final Optional storage, + final Optional> events, + final ProxyCacheConfig config + ) { + this(client, cache, repoName, repoType, upstreamUrl, + storage, events, config, null, null); + } + + @Override + public final CompletableFuture response( + final RequestLine line, final Headers headers, final Content body + ) { + final String path = line.uri().getPath(); + if ("/".equals(path) || path.isEmpty()) { + return this.handleRootPath(line); + } + final Key key = new KeyFromPath(path); + // Step 1: Negative cache fast-fail + if (this.negativeCache != null && this.negativeCache.isNotFound(key)) { + this.logDebug("Negative cache hit", path); + return CompletableFuture.completedFuture(ResponseBuilder.notFound().build()); + } + // Step 2: Pre-process hook (adapter-specific short-circuit) + final Optional> pre = + this.preProcess(line, headers, key, path); + if (pre.isPresent()) { + return pre.get(); + } + // Step 3: Check if path is cacheable at all + if (!this.isCacheable(path)) { + return this.fetchDirect(line, key, new Login(headers).getValue()); + } + // Step 4: Cache-first (offline-safe) — check cache before any network calls + if (this.storageBacked) { + return this.cacheFirstFlow(line, headers, key, path); + } + // No persistent storage — go directly to upstream + return this.fetchDirect(line, key, new Login(headers).getValue()); + } + + // ===== Abstract hooks — adapters override these ===== + + /** + * Determine if a request path is cacheable. + * @param path Request path (e.g., "/com/example/foo/1.0/foo-1.0.jar") + * @return True if this path should be cached + */ + protected abstract boolean isCacheable(String path); + + // ===== Overridable hooks with defaults ===== + + /** + * Build a cooldown request from the path. + * Return empty to skip cooldown for this path. + * @param path Request path + * @param headers Request headers + * @return Cooldown request or empty + */ + protected Optional buildCooldownRequest( + final String path, final Headers headers + ) { + return Optional.empty(); + } + + /** + * Return the set of digest algorithms to compute during cache streaming. + * Return empty set to skip digest computation. + * Override in adapters to enable digest computation (e.g., SHA-256, MD5). + * @return Set of algorithm names (e.g., "SHA-256", "MD5") + */ + protected java.util.Set digestAlgorithms() { + return Collections.emptySet(); + } + + /** + * Build a proxy artifact event for the event queue. + * Return empty to skip event emission. + * @param key Artifact cache key + * @param responseHeaders Upstream response headers + * @param size Artifact size in bytes + * @param owner Authenticated user login + * @return Proxy artifact event or empty + */ + protected Optional buildArtifactEvent( + final Key key, final Headers responseHeaders, final long size, + final String owner + ) { + return Optional.empty(); + } + + /** + * Post-process response before returning to caller. + * Default: identity (no transformation). + * @param response The response to post-process + * @param line Original request line + * @return Post-processed response + */ + protected Response postProcess(final Response response, final RequestLine line) { + return response; + } + + /** + * Generate sidecar files from computed digests. + * Default: empty list (no sidecars). + * @param path Original artifact path + * @param digests Computed digests map (algorithm -> hex value) + * @return List of sidecar files to store alongside the artifact + */ + protected List generateSidecars( + final String path, final Map digests + ) { + return Collections.emptyList(); + } + + /** + * Check if path is a sidecar checksum file that should be served from cache. + * Default: false. Override in adapters that generate checksum sidecars. + * @param path Request path + * @return True if this is a checksum sidecar file + */ + protected boolean isChecksumSidecar(final String path) { + return false; + } + + /** + * Pre-process a request before the standard flow. + * If non-empty, the returned response short-circuits the standard flow. + * Use for adapter-specific handling (e.g., Maven metadata cache). + * Default: empty (use standard flow for all paths). + * @param line Request line + * @param headers Request headers + * @param key Cache key + * @param path Request path + * @return Optional future response to short-circuit, or empty for standard flow + */ + protected Optional> preProcess( + final RequestLine line, final Headers headers, final Key key, final String path + ) { + return Optional.empty(); + } + + // ===== Protected accessors for subclass use ===== + + /** + * @return Repository name + */ + protected final String repoName() { + return this.repoName; + } + + /** + * @return Repository type + */ + protected final String repoType() { + return this.repoType; + } + + /** + * @return Upstream URL + */ + protected final String upstreamUrl() { + return this.upstreamUrl; + } + + /** + * @return The upstream client slice + */ + protected final Slice client() { + return this.client; + } + + /** + * @return The asto cache + */ + protected final Cache cache() { + return this.cache; + } + + /** + * @return Proxy cache config + */ + protected final ProxyCacheConfig config() { + return this.config; + } + + /** + * @return Metadata store if storage-backed + */ + protected final Optional metadataStore() { + return this.metadataStore; + } + + // ===== Internal flow implementation ===== + + /** + * Cache-first flow: check cache, then evaluate cooldown, then fetch. + */ + private CompletableFuture cacheFirstFlow( + final RequestLine line, + final Headers headers, + final Key key, + final String path + ) { + // Checksum sidecars: serve from storage if present, else try upstream + if (this.isChecksumSidecar(path)) { + return this.serveChecksumFromStorage(line, key, new Login(headers).getValue()); + } + final CachedArtifactMetadataStore store = this.metadataStore.orElseThrow(); + return this.cache.load(key, Remote.EMPTY, CacheControl.Standard.ALWAYS) + .thenCompose(cached -> { + if (cached.isPresent()) { + this.logDebug("Cache hit", path); + // Fast path: serve from cache with async metadata + return store.load(key).thenApply(meta -> { + final ResponseBuilder builder = ResponseBuilder.ok() + .body(cached.get()); + meta.ifPresent(m -> builder.headers(stripContentEncoding(m.headers()))); + return this.postProcess(builder.build(), line); + }); + } + // Cache miss: evaluate cooldown then fetch + return this.evaluateCooldownAndFetch(line, headers, key, path, store); + }).toCompletableFuture(); + } + + /** + * Evaluate cooldown, then fetch from upstream if allowed. + */ + private CompletableFuture evaluateCooldownAndFetch( + final RequestLine line, + final Headers headers, + final Key key, + final String path, + final CachedArtifactMetadataStore store + ) { + if (this.config.cooldownEnabled() + && this.cooldownService != null + && this.cooldownInspector != null) { + final Optional request = + this.buildCooldownRequest(path, headers); + if (request.isPresent()) { + return this.cooldownService.evaluate(request.get(), this.cooldownInspector) + .thenCompose(result -> { + if (result.blocked()) { + return CompletableFuture.completedFuture( + CooldownResponses.forbidden(result.block().orElseThrow()) + ); + } + return this.fetchAndCache(line, key, headers, store); + }); + } + } + return this.fetchAndCache(line, key, headers, store); + } + + /** + * Fetch from upstream and cache the result, with request deduplication. + * Uses NIO temp file streaming to avoid buffering full artifacts on heap. + */ + private CompletableFuture fetchAndCache( + final RequestLine line, + final Key key, + final Headers headers, + final CachedArtifactMetadataStore store + ) { + final String owner = new Login(headers).getValue(); + final long startTime = System.currentTimeMillis(); + return this.client.response(line, Headers.EMPTY, Content.EMPTY) + .thenCompose(resp -> { + final long duration = System.currentTimeMillis() - startTime; + if (resp.status().code() == 404) { + return this.handle404(resp, key, duration) + .thenCompose(signal -> + this.signalToResponse(signal, line, key, headers, store)); + } + if (!resp.status().success()) { + return this.handleNonSuccess(resp, key, duration) + .thenCompose(signal -> + this.signalToResponse(signal, line, key, headers, store)); + } + this.recordProxyMetric("success", duration); + return this.deduplicator.deduplicate(key, () -> { + return this.cacheResponse(resp, key, owner, store) + .thenApply(r -> RequestDeduplicator.FetchSignal.SUCCESS); + }).thenCompose(signal -> + this.signalToResponse(signal, line, key, headers, store)); + }) + .exceptionally(error -> { + final long duration = System.currentTimeMillis() - startTime; + this.trackUpstreamFailure(error); + this.recordProxyMetric("exception", duration); + EcsLogger.warn("com.artipie." + this.repoType) + .message("Upstream request failed with exception") + .eventCategory("repository") + .eventAction("proxy_upstream") + .eventOutcome("failure") + .field("repository.name", this.repoName) + .field("event.duration", duration) + .error(error) + .log(); + return ResponseBuilder.unavailable() + .textBody("Upstream temporarily unavailable") + .build(); + }); + } + + /** + * Convert a dedup signal into an HTTP response. + */ + private CompletableFuture signalToResponse( + final RequestDeduplicator.FetchSignal signal, + final RequestLine line, + final Key key, + final Headers headers, + final CachedArtifactMetadataStore store + ) { + switch (signal) { + case SUCCESS: + // Read from cache (populated by the winning fetch) + return this.cache.load(key, Remote.EMPTY, CacheControl.Standard.ALWAYS) + .thenCompose(cached -> { + if (cached.isPresent()) { + return store.load(key).thenApply(meta -> { + final ResponseBuilder builder = ResponseBuilder.ok() + .body(cached.get()); + meta.ifPresent(m -> builder.headers(stripContentEncoding(m.headers()))); + return this.postProcess(builder.build(), line); + }); + } + return CompletableFuture.completedFuture( + ResponseBuilder.notFound().build() + ); + }).toCompletableFuture(); + case NOT_FOUND: + return CompletableFuture.completedFuture( + ResponseBuilder.notFound().build() + ); + case ERROR: + default: + return CompletableFuture.completedFuture( + ResponseBuilder.unavailable() + .textBody("Upstream temporarily unavailable") + .build() + ); + } + } + + /** + * Cache a successful upstream response using NIO temp file streaming. + * Streams body to a temp file while computing digests incrementally, + * then saves from temp file to cache. Never buffers the full artifact on heap. + */ + @SuppressWarnings("PMD.AvoidCatchingGenericException") + private CompletableFuture cacheResponse( + final Response resp, + final Key key, + final String owner, + final CachedArtifactMetadataStore store + ) { + final Path tempFile; + final FileChannel channel; + try { + tempFile = Files.createTempFile("artipie-cache-", ".tmp"); + tempFile.toFile().deleteOnExit(); + channel = FileChannel.open( + tempFile, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING + ); + } catch (final IOException ex) { + EcsLogger.warn("com.artipie." + this.repoType) + .message("Failed to create temp file for cache streaming") + .eventCategory("repository") + .eventAction("proxy_cache") + .eventOutcome("failure") + .field("repository.name", this.repoName) + .field("file.path", key.string()) + .error(ex) + .log(); + return CompletableFuture.completedFuture( + RequestDeduplicator.FetchSignal.ERROR + ); + } + final Map digests = + DigestComputer.createDigests(this.digestAlgorithms()); + final AtomicLong totalSize = new AtomicLong(0); + final CompletableFuture streamDone = new CompletableFuture<>(); + Flowable.fromPublisher(resp.body()) + .doOnNext(buf -> { + final int nbytes = buf.remaining(); + DigestComputer.updateDigests(digests, buf); + final ByteBuffer copy = buf.asReadOnlyBuffer(); + while (copy.hasRemaining()) { + channel.write(copy); + } + totalSize.addAndGet(nbytes); + }) + .doOnComplete(() -> { + channel.force(true); + channel.close(); + }) + .doOnError(err -> { + closeChannelQuietly(channel); + deleteTempQuietly(tempFile); + }) + .subscribe( + item -> { }, + streamDone::completeExceptionally, + () -> streamDone.complete(null) + ); + return streamDone.thenCompose(v -> { + final Map digestResults = + DigestComputer.finalizeDigests(digests); + final long size = totalSize.get(); + return this.saveFromTempFile(key, tempFile, size) + .thenCompose(loaded -> { + final Map digestsCopy = + new java.util.HashMap<>(digestResults); + final CachedArtifactMetadataStore.ComputedDigests computed = + new CachedArtifactMetadataStore.ComputedDigests( + size, digestsCopy + ); + return store.save(key, stripContentEncoding(resp.headers()), computed); + }).thenCompose(savedHeaders -> { + final List sidecars = + this.generateSidecars(key.string(), digestResults); + if (sidecars.isEmpty()) { + return CompletableFuture.completedFuture( + (Void) null + ); + } + final CompletableFuture[] writes; + if (this.storage.isPresent()) { + // Save sidecars directly to storage (avoids lazy tee-content) + writes = sidecars.stream() + .map(sc -> this.storage.get().save( + new Key.From(sc.path()), + new Content.From(sc.content()) + )) + .toArray(CompletableFuture[]::new); + } else { + writes = sidecars.stream() + .map(sc -> this.cache.load( + new Key.From(sc.path()), + () -> CompletableFuture.completedFuture( + Optional.of(new Content.From(sc.content())) + ), + CacheControl.Standard.ALWAYS + )) + .toArray(CompletableFuture[]::new); + } + return CompletableFuture.allOf(writes); + }).thenApply(ignored -> { + this.enqueueEvent(key, resp.headers(), size, owner); + deleteTempQuietly(tempFile); + return RequestDeduplicator.FetchSignal.SUCCESS; + }); + }).exceptionally(err -> { + deleteTempQuietly(tempFile); + EcsLogger.warn("com.artipie." + this.repoType) + .message("Failed to cache upstream response") + .eventCategory("repository") + .eventAction("proxy_cache") + .eventOutcome("failure") + .field("repository.name", this.repoName) + .field("file.path", key.string()) + .error(err) + .log(); + return RequestDeduplicator.FetchSignal.ERROR; + }); + } + + /** + * Save content to cache from a temp file using NIO streaming. + * Saves directly to storage to avoid FromStorageCache's lazy tee-content + * which requires the returned Content to be consumed for the save to happen. + * @param key Cache key + * @param tempFile Temp file with content + * @param size File size in bytes + * @return Save future + */ + @SuppressWarnings("PMD.AvoidCatchingGenericException") + private CompletableFuture saveFromTempFile( + final Key key, final Path tempFile, final long size + ) { + if (this.storage.isPresent()) { + final Flowable flow = Flowable.using( + () -> FileChannel.open(tempFile, StandardOpenOption.READ), + chan -> Flowable.generate(emitter -> { + final ByteBuffer buf = ByteBuffer.allocate(65536); + final int read = chan.read(buf); + if (read < 0) { + emitter.onComplete(); + } else { + buf.flip(); + emitter.onNext(buf); + } + }), + FileChannel::close + ); + final Content content = new Content.From(Optional.of(size), flow); + return this.storage.get().save(key, content); + } + // Fallback: use cache.load (non-storage-backed mode) + final Flowable flow = Flowable.using( + () -> FileChannel.open(tempFile, StandardOpenOption.READ), + chan -> Flowable.generate(emitter -> { + final ByteBuffer buf = ByteBuffer.allocate(65536); + final int read = chan.read(buf); + if (read < 0) { + emitter.onComplete(); + } else { + buf.flip(); + emitter.onNext(buf); + } + }), + FileChannel::close + ); + final Content content = new Content.From(Optional.of(size), flow); + return this.cache.load( + key, + () -> CompletableFuture.completedFuture(Optional.of(content)), + CacheControl.Standard.ALWAYS + ).toCompletableFuture(); + } + + /** + * Close a FileChannel quietly. + * @param channel Channel to close + */ + private static void closeChannelQuietly(final FileChannel channel) { + try { + if (channel.isOpen()) { + channel.close(); + } + } catch (final IOException ex) { + EcsLogger.debug("com.artipie.cache") + .message("Failed to close file channel") + .error(ex) + .log(); + } + } + + /** + * Delete a temp file quietly. + * @param path Temp file to delete + */ + private static void deleteTempQuietly(final Path path) { + try { + Files.deleteIfExists(path); + } catch (final IOException ex) { + EcsLogger.debug("com.artipie.cache") + .message("Failed to delete temp file") + .error(ex) + .log(); + } + } + + /** + * Fetch directly from upstream without caching (non-cacheable paths). + */ + private CompletableFuture fetchDirect( + final RequestLine line, final Key key, final String owner + ) { + final long startTime = System.currentTimeMillis(); + return this.client.response(line, Headers.EMPTY, Content.EMPTY) + .thenCompose(resp -> { + final long duration = System.currentTimeMillis() - startTime; + if (!resp.status().success()) { + if (resp.status().code() == 404) { + if (this.negativeCache != null + && !this.isChecksumSidecar(key.string())) { + resp.body().asBytesFuture().thenAccept( + bytes -> this.negativeCache.cacheNotFound(key) + ); + } + this.recordProxyMetric("not_found", duration); + } else if (resp.status().code() >= 500) { + this.trackUpstreamFailure( + new RuntimeException("HTTP " + resp.status().code()) + ); + this.recordProxyMetric("error", duration); + } else { + this.recordProxyMetric("client_error", duration); + } + return resp.body().asBytesFuture() + .thenApply(bytes -> ResponseBuilder.notFound().build()); + } + this.recordProxyMetric("success", duration); + this.enqueueEvent(key, resp.headers(), -1, owner); + return CompletableFuture.completedFuture( + this.postProcess( + ResponseBuilder.ok() + .headers(stripContentEncoding(resp.headers())) + .body(resp.body()) + .build(), + line + ) + ); + }) + .exceptionally(error -> { + final long duration = System.currentTimeMillis() - startTime; + this.trackUpstreamFailure(error); + this.recordProxyMetric("exception", duration); + EcsLogger.warn("com.artipie." + this.repoType) + .message("Direct upstream request failed with exception") + .eventCategory("repository") + .eventAction("proxy_upstream") + .eventOutcome("failure") + .field("repository.name", this.repoName) + .field("event.duration", duration) + .error(error) + .log(); + return ResponseBuilder.unavailable() + .textBody("Upstream error") + .build(); + }); + } + + private CompletableFuture handle404( + final Response resp, final Key key, final long duration + ) { + this.recordProxyMetric("not_found", duration); + return resp.body().asBytesFuture().thenApply(bytes -> { + if (this.negativeCache != null && !this.isChecksumSidecar(key.string())) { + this.negativeCache.cacheNotFound(key); + } + return RequestDeduplicator.FetchSignal.NOT_FOUND; + }); + } + + private CompletableFuture handleNonSuccess( + final Response resp, final Key key, final long duration + ) { + if (resp.status().code() >= 500) { + this.trackUpstreamFailure( + new RuntimeException("HTTP " + resp.status().code()) + ); + this.recordProxyMetric("error", duration); + } else { + this.recordProxyMetric("client_error", duration); + } + return resp.body().asBytesFuture() + .thenApply(bytes -> RequestDeduplicator.FetchSignal.ERROR); + } + + private CompletableFuture serveChecksumFromStorage( + final RequestLine line, final Key key, final String owner + ) { + return this.cache.load(key, Remote.EMPTY, CacheControl.Standard.ALWAYS) + .thenCompose(cached -> { + if (cached.isPresent()) { + return CompletableFuture.completedFuture( + ResponseBuilder.ok() + .header("Content-Type", "text/plain") + .body(cached.get()) + .build() + ); + } + return this.fetchDirect(line, key, owner); + }).toCompletableFuture(); + } + + private CompletableFuture handleRootPath(final RequestLine line) { + return this.client.response(line, Headers.EMPTY, Content.EMPTY) + .thenCompose(resp -> { + if (resp.status().success()) { + return CompletableFuture.completedFuture( + ResponseBuilder.ok() + .headers(stripContentEncoding(resp.headers())) + .body(resp.body()) + .build() + ); + } + return resp.body().asBytesFuture() + .thenApply(ignored -> ResponseBuilder.notFound().build()); + }); + } + + private void enqueueEvent( + final Key key, final Headers headers, final long size, final String owner + ) { + if (this.events.isEmpty()) { + return; + } + final Optional event = + this.buildArtifactEvent(key, headers, size, owner); + event.ifPresent(e -> this.events.get().offer(e)); + } + + private void trackUpstreamFailure(final Throwable error) { + final String errorType; + if (error instanceof TimeoutException) { + errorType = "timeout"; + } else if (error instanceof ConnectException) { + errorType = "connection_refused"; + } else { + errorType = "unknown"; + } + this.recordMetric(() -> + com.artipie.metrics.ArtipieMetrics.instance() + .upstreamFailure(this.repoName, this.upstreamUrl, errorType) + ); + } + + private void recordProxyMetric(final String result, final long duration) { + this.recordMetric(() -> { + if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { + com.artipie.metrics.MicrometerMetrics.getInstance() + .recordProxyRequest(this.repoName, this.upstreamUrl, result, duration); + } + }); + } + + @SuppressWarnings("PMD.AvoidCatchingGenericException") + private void recordMetric(final Runnable metric) { + try { + if (com.artipie.metrics.ArtipieMetrics.isEnabled()) { + metric.run(); + } + } catch (final Exception ex) { + EcsLogger.debug("com.artipie.cache") + .message("Failed to record metric") + .error(ex) + .log(); + } + } + + private void logDebug(final String message, final String path) { + EcsLogger.debug("com.artipie." + this.repoType) + .message(message) + .eventCategory("repository") + .eventAction("proxy_request") + .field("repository.name", this.repoName) + .field("url.path", path) + .log(); + } + + /** + * Strip {@code Content-Encoding} and {@code Content-Length} headers that indicate + * the HTTP client already decoded the response body. + * + *

Jetty's {@code GZIPContentDecoder} (registered by default) auto-decodes gzip, + * deflate and br response bodies but leaves the original {@code Content-Encoding} + * header intact. Passing those headers through to callers creates a header/body + * mismatch: the body is plain bytes while the header still claims it is compressed. + * Any client that trusts the header will fail to inflate the body + * ({@code Z_DATA_ERROR: zlib: incorrect header check}). + * + *

We strip {@code Content-Length} as well because it refers to the compressed + * size, which no longer matches the decoded body length. + * + * @param headers Upstream response headers + * @return Headers without Content-Encoding (gzip/deflate/br) and Content-Length + */ + protected static Headers stripContentEncoding(final Headers headers) { + final boolean hasDecoded = StreamSupport.stream(headers.spliterator(), false) + .filter(h -> "content-encoding".equalsIgnoreCase(h.getKey())) + .map(Header::getValue) + .map(v -> v.toLowerCase(Locale.ROOT).trim()) + .anyMatch(v -> v.contains("gzip") || v.contains("deflate") || v.contains("br")); + if (!hasDecoded) { + return headers; + } + final List

filtered = StreamSupport.stream(headers.spliterator(), false) + .filter(h -> !"content-encoding".equalsIgnoreCase(h.getKey()) + && !"content-length".equalsIgnoreCase(h.getKey())) + .collect(Collectors.toList()); + return new Headers(filtered); + } + + /** + * Extract Last-Modified timestamp from response headers. + * @param headers Response headers + * @return Optional epoch millis + */ + protected static Optional extractLastModified(final Headers headers) { + try { + return StreamSupport.stream(headers.spliterator(), false) + .filter(h -> "Last-Modified".equalsIgnoreCase(h.getKey())) + .findFirst() + .map(Header::getValue) + .map(val -> Instant.from( + DateTimeFormatter.RFC_1123_DATE_TIME.parse(val) + ).toEpochMilli()); + } catch (final DateTimeParseException ex) { + EcsLogger.debug("com.artipie.cache") + .message("Failed to parse Last-Modified header") + .error(ex) + .log(); + return Optional.empty(); + } + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/cache/ConditionalRequest.java b/artipie-core/src/main/java/com/artipie/http/cache/ConditionalRequest.java new file mode 100644 index 000000000..d04a422c8 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/cache/ConditionalRequest.java @@ -0,0 +1,77 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import com.artipie.http.Headers; +import com.artipie.http.headers.Header; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Builds conditional request headers (ETag/If-None-Match, Last-Modified/If-Modified-Since) + * for upstream requests when cached content is available. + * + * @since 1.20.13 + */ +public final class ConditionalRequest { + + /** + * Private ctor — static utility. + */ + private ConditionalRequest() { + } + + /** + * Build conditional headers from cached metadata. + * + * @param cachedEtag ETag from previously cached response (if available) + * @param cachedLastModified Last-Modified header value from cached response (if available) + * @return Headers with conditional request fields, or empty headers if no metadata + */ + public static Headers conditionalHeaders( + final Optional cachedEtag, + final Optional cachedLastModified + ) { + final List
headers = new ArrayList<>(2); + cachedEtag.ifPresent( + etag -> headers.add(new Header("If-None-Match", etag)) + ); + cachedLastModified.ifPresent( + lm -> headers.add(new Header("If-Modified-Since", lm)) + ); + if (headers.isEmpty()) { + return Headers.EMPTY; + } + return new Headers(headers); + } + + /** + * Extract ETag value from response headers. + * + * @param headers Response headers + * @return ETag value if present + */ + public static Optional extractEtag(final Headers headers) { + return headers.stream() + .filter(h -> "ETag".equalsIgnoreCase(h.getKey())) + .findFirst() + .map(com.artipie.http.headers.Header::getValue); + } + + /** + * Extract Last-Modified value from response headers. + * + * @param headers Response headers + * @return Last-Modified value if present + */ + public static Optional extractLastModified(final Headers headers) { + return headers.stream() + .filter(h -> "Last-Modified".equalsIgnoreCase(h.getKey())) + .findFirst() + .map(com.artipie.http.headers.Header::getValue); + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/cache/DedupStrategy.java b/artipie-core/src/main/java/com/artipie/http/cache/DedupStrategy.java new file mode 100644 index 000000000..d8bc535e7 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/cache/DedupStrategy.java @@ -0,0 +1,33 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +/** + * Request deduplication strategy for proxy caches. + * + * @since 1.20.13 + */ +public enum DedupStrategy { + + /** + * No deduplication. Each concurrent request independently fetches from upstream. + */ + NONE, + + /** + * Storage-level deduplication. Uses storage key locking to prevent + * concurrent writes to the same cache key. Second request waits for + * the first to complete and reads from cache. + */ + STORAGE, + + /** + * Signal-based deduplication (zero-copy). First request fetches and caches, + * then signals completion. Waiting requests read from cache on SUCCESS + * signal, or return appropriate error on NOT_FOUND / ERROR signals. + * No response body buffering in memory. + */ + SIGNAL +} diff --git a/artipie-core/src/main/java/com/artipie/http/cache/DigestComputer.java b/artipie-core/src/main/java/com/artipie/http/cache/DigestComputer.java new file mode 100644 index 000000000..a8351b27c --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/cache/DigestComputer.java @@ -0,0 +1,157 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Computes cryptographic digests for artifact content. + * Thread-safe utility — each call allocates fresh MessageDigest instances. + * + *

Supported algorithms: SHA-256, SHA-1, MD5, SHA-512. + * + *

Provides both batch ({@link #compute(byte[], Set)}) and streaming + * ({@link #createDigests}, {@link #updateDigests}, {@link #finalizeDigests}) + * APIs. The streaming API enables incremental digest computation without + * buffering the entire content in memory. + * + * @since 1.20.13 + */ +public final class DigestComputer { + + /** + * SHA-256 algorithm name. + */ + public static final String SHA256 = "SHA-256"; + + /** + * SHA-1 algorithm name. + */ + public static final String SHA1 = "SHA-1"; + + /** + * MD5 algorithm name. + */ + public static final String MD5 = "MD5"; + + /** + * SHA-512 algorithm name. + */ + public static final String SHA512 = "SHA-512"; + + /** + * Maven digest set: SHA-256 + SHA-1 + MD5. + */ + public static final Set MAVEN_DIGESTS = Set.of(SHA256, SHA1, MD5); + + + /** + * Hex formatter for digest output. + */ + private static final HexFormat HEX = HexFormat.of(); + + /** + * Private ctor — static utility class. + */ + private DigestComputer() { + } + + /** + * Compute digests for the given content using specified algorithms. + * + * @param content Raw artifact bytes + * @param algorithms Set of algorithm names (e.g., "SHA-256", "MD5") + * @return Map of algorithm name to lowercase hex-encoded digest string + * @throws IllegalArgumentException If an unsupported algorithm is requested + */ + public static Map compute( + final byte[] content, final Set algorithms + ) { + Objects.requireNonNull(content, "content"); + if (algorithms == null || algorithms.isEmpty()) { + return Collections.emptyMap(); + } + final Map digests = createDigests(algorithms); + for (final MessageDigest digest : digests.values()) { + digest.update(content); + } + return finalizeDigests(digests); + } + + /** + * Create fresh MessageDigest instances for the specified algorithms. + * Use with {@link #updateDigests} and {@link #finalizeDigests} for + * streaming digest computation. + * + * @param algorithms Set of algorithm names (e.g., "SHA-256", "MD5") + * @return Map of algorithm name to MessageDigest instance + * @throws IllegalArgumentException If an unsupported algorithm is requested + */ + public static Map createDigests( + final Set algorithms + ) { + if (algorithms == null || algorithms.isEmpty()) { + return Collections.emptyMap(); + } + final Map digests = new HashMap<>(algorithms.size()); + for (final String algo : algorithms) { + try { + digests.put(algo, MessageDigest.getInstance(algo)); + } catch (final NoSuchAlgorithmException ex) { + throw new IllegalArgumentException( + String.format("Unsupported digest algorithm: %s", algo), ex + ); + } + } + return digests; + } + + /** + * Update all digests with the given chunk of data. + * The ByteBuffer position is advanced to its limit after this call. + * + * @param digests Map of algorithm name to MessageDigest (from {@link #createDigests}) + * @param chunk Data chunk to feed into digests + */ + public static void updateDigests( + final Map digests, final ByteBuffer chunk + ) { + if (digests.isEmpty() || !chunk.hasRemaining()) { + return; + } + for (final MessageDigest digest : digests.values()) { + final ByteBuffer view = chunk.asReadOnlyBuffer(); + digest.update(view); + } + } + + /** + * Finalize all digests and return hex-encoded results. + * After this call the MessageDigest instances are reset. + * + * @param digests Map of algorithm name to MessageDigest + * @return Map of algorithm name to lowercase hex-encoded digest string + */ + public static Map finalizeDigests( + final Map digests + ) { + if (digests.isEmpty()) { + return Collections.emptyMap(); + } + final Map result = new HashMap<>(digests.size()); + for (final Map.Entry entry : digests.entrySet()) { + result.put(entry.getKey(), HEX.formatHex(entry.getValue().digest())); + } + return Collections.unmodifiableMap(result); + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/cache/NegativeCache.java b/artipie-core/src/main/java/com/artipie/http/cache/NegativeCache.java index a9d5ed0f3..90a8115e3 100644 --- a/artipie-core/src/main/java/com/artipie/http/cache/NegativeCache.java +++ b/artipie-core/src/main/java/com/artipie/http/cache/NegativeCache.java @@ -10,6 +10,8 @@ import com.artipie.cache.ValkeyConnection; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import io.lettuce.core.ScanArgs; +import io.lettuce.core.ScanCursor; import io.lettuce.core.api.async.RedisAsyncCommands; import java.time.Duration; import java.util.concurrent.CompletableFuture; @@ -410,11 +412,7 @@ public void invalidatePrefix(final String prefix) { // Invalidate L2 (if enabled) if (this.twoTier) { final String scanPattern = "negative:" + this.repoType + ":" + this.repoName + ":" + prefix + "*"; - this.l2.keys(scanPattern).thenAccept(keys -> { - if (keys != null && !keys.isEmpty()) { - this.l2.del(keys.toArray(new String[0])); - } - }); + this.scanAndDelete(scanPattern); } } @@ -428,14 +426,44 @@ public void clear() { // Clear L2 (if enabled) - scan and delete all negative cache keys if (this.twoTier) { - this.l2.keys("negative:" + this.repoType + ":" + this.repoName + ":*").thenAccept(keys -> { - if (keys != null && !keys.isEmpty()) { - this.l2.del(keys.toArray(new String[0])); - } - }); + this.scanAndDelete("negative:" + this.repoType + ":" + this.repoName + ":*"); } } + /** + * Recursive async scan that collects all matching keys and deletes them in batches. + * Uses SCAN instead of KEYS to avoid blocking the Redis server. + * + * @param pattern Glob pattern to match keys + * @return Future that completes when all matching keys are deleted + */ + private CompletableFuture scanAndDelete(final String pattern) { + return this.scanAndDeleteStep(ScanCursor.INITIAL, pattern); + } + + /** + * Single step of the recursive SCAN-and-delete loop. + * + * @param cursor Current scan cursor + * @param pattern Glob pattern to match keys + * @return Future that completes when this step and all subsequent steps finish + */ + private CompletableFuture scanAndDeleteStep( + final ScanCursor cursor, final String pattern + ) { + return this.l2.scan(cursor, ScanArgs.Builder.matches(pattern).limit(100)) + .toCompletableFuture() + .thenCompose(result -> { + if (!result.getKeys().isEmpty()) { + this.l2.del(result.getKeys().toArray(new String[0])); + } + if (result.isFinished()) { + return CompletableFuture.completedFuture(null); + } + return this.scanAndDeleteStep(result, pattern); + }); + } + /** * Remove expired entries (periodic cleanup). * Caffeine handles expiry automatically, but calling this diff --git a/artipie-core/src/main/java/com/artipie/http/cache/NegativeCacheRegistry.java b/artipie-core/src/main/java/com/artipie/http/cache/NegativeCacheRegistry.java new file mode 100644 index 000000000..b488b9514 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/cache/NegativeCacheRegistry.java @@ -0,0 +1,111 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import com.artipie.asto.Key; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Global registry of all proxy NegativeCache instances. + * Enables cross-adapter cache invalidation when artifacts are published. + * + * @since 1.20.13 + */ +public final class NegativeCacheRegistry { + + /** + * Singleton instance. + */ + private static final NegativeCacheRegistry INSTANCE = new NegativeCacheRegistry(); + + /** + * Registered caches: key = "repoType:repoName". + */ + private final ConcurrentMap caches; + + /** + * Private ctor. + */ + private NegativeCacheRegistry() { + this.caches = new ConcurrentHashMap<>(); + } + + /** + * Get singleton instance. + * @return Registry instance + */ + public static NegativeCacheRegistry instance() { + return INSTANCE; + } + + /** + * Register a negative cache instance. + * @param repoType Repository type + * @param repoName Repository name + * @param cache Negative cache instance + */ + public void register( + final String repoType, final String repoName, final NegativeCache cache + ) { + this.caches.put(key(repoType, repoName), cache); + } + + /** + * Unregister a negative cache instance. + * @param repoType Repository type + * @param repoName Repository name + */ + public void unregister(final String repoType, final String repoName) { + this.caches.remove(key(repoType, repoName)); + } + + /** + * Invalidate a specific artifact path across ALL registered negative caches. + * Called when an artifact is published to ensure stale 404 entries are cleared. + * + * @param artifactPath Artifact path to invalidate + */ + public void invalidateGlobally(final String artifactPath) { + final Key artKey = new Key.From(artifactPath); + this.caches.values().forEach(cache -> cache.invalidate(artKey)); + } + + /** + * Invalidate a specific artifact path in a specific repository's negative cache. + * + * @param repoType Repository type + * @param repoName Repository name + * @param artifactPath Artifact path to invalidate + */ + public void invalidate( + final String repoType, final String repoName, final String artifactPath + ) { + final NegativeCache cache = this.caches.get(key(repoType, repoName)); + if (cache != null) { + cache.invalidate(new Key.From(artifactPath)); + } + } + + /** + * Get the number of registered caches. + * @return Count of registered caches + */ + public int size() { + return this.caches.size(); + } + + /** + * Clear all registrations (for testing). + */ + public void clear() { + this.caches.clear(); + } + + private static String key(final String repoType, final String repoName) { + return repoType + ":" + repoName; + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/cache/ProxyCacheConfig.java b/artipie-core/src/main/java/com/artipie/http/cache/ProxyCacheConfig.java index d7f034503..6d83be60e 100644 --- a/artipie-core/src/main/java/com/artipie/http/cache/ProxyCacheConfig.java +++ b/artipie-core/src/main/java/com/artipie/http/cache/ProxyCacheConfig.java @@ -6,22 +6,36 @@ import com.amihaiemil.eoyaml.YamlMapping; import java.time.Duration; +import java.util.Locale; import java.util.Optional; /** - * Proxy cache configuration parser for YAML. - * Parses cache settings for negative caching and metadata caching. + * Unified proxy cache configuration parsed from YAML. + * Controls negative caching, metadata caching, cooldown toggle, request deduplication, + * conditional requests (ETag), stale-while-revalidate, retry, and metrics. * *

Example YAML: * 

  * cache:
  *   negative:
  *     enabled: true
- *     ttl: PT24H        # ISO-8601 duration (24 hours)
- *     maxSize: 50000    # Maximum entries
+ *     ttl: PT24H
+ *     maxSize: 50000
  *   metadata:
  *     enabled: true
- *     ttl: PT168H       # 7 days
+ *     ttl: PT168H
+ *   cooldown:
+ *     enabled: true
+ *   dedup_strategy: signal          # none | storage | signal
+ *   conditional_requests: true      # ETag / If-None-Match
+ *   stale_while_revalidate:
+ *     enabled: false
+ *     max_age: PT1H
+ *   retry:
+ *     max_retries: 2
+ *     initial_delay: PT0.1S
+ *     backoff_multiplier: 2.0
+ *   metrics: true
  *
* * @since 1.0 @@ -43,6 +57,21 @@ public final class ProxyCacheConfig { */ public static final Duration DEFAULT_METADATA_TTL = Duration.ofDays(7); + /** + * Default stale-while-revalidate max age (1 hour). + */ + public static final Duration DEFAULT_STALE_MAX_AGE = Duration.ofHours(1); + + /** + * Default retry initial delay (100ms). + */ + public static final Duration DEFAULT_RETRY_INITIAL_DELAY = Duration.ofMillis(100); + + /** + * Default retry backoff multiplier. + */ + public static final double DEFAULT_RETRY_BACKOFF_MULTIPLIER = 2.0; + /** * YAML configuration. */ @@ -84,7 +113,7 @@ public int negativeCacheMaxSize() { /** * Check if metadata caching is enabled. - * @return True if enabled (default: false - needs implementation) + * @return True if enabled (default: false) */ public boolean metadataCacheEnabled() { return this.boolValue("cache", "metadata", "enabled").orElse(false); @@ -99,6 +128,84 @@ public Duration metadataCacheTtl() { .orElse(DEFAULT_METADATA_TTL); } + /** + * Check if cooldown is enabled for this adapter. + * @return True if enabled (default: false) + */ + public boolean cooldownEnabled() { + return this.boolValue("cache", "cooldown", "enabled").orElse(false); + } + + /** + * Get request deduplication strategy. + * @return Dedup strategy (default: SIGNAL) + */ + public DedupStrategy dedupStrategy() { + return this.stringValue("cache", "dedup_strategy") + .map(s -> DedupStrategy.valueOf(s.toUpperCase(Locale.ROOT))) + .orElse(DedupStrategy.SIGNAL); + } + + /** + * Check if conditional requests (ETag/If-None-Match) are enabled. + * @return True if enabled (default: true) + */ + public boolean conditionalRequestsEnabled() { + return this.boolValue("cache", "conditional_requests").orElse(true); + } + + /** + * Check if stale-while-revalidate is enabled. + * @return True if enabled (default: false) + */ + public boolean staleWhileRevalidateEnabled() { + return this.boolValue("cache", "stale_while_revalidate", "enabled") + .orElse(false); + } + + /** + * Get stale-while-revalidate max age. + * @return Max age duration (default: 1 hour) + */ + public Duration staleMaxAge() { + return this.durationValue("cache", "stale_while_revalidate", "max_age") + .orElse(DEFAULT_STALE_MAX_AGE); + } + + /** + * Get maximum number of retry attempts for upstream requests. + * @return Max retries (default: 0 = disabled) + */ + public int retryMaxRetries() { + return this.intValue("cache", "retry", "max_retries").orElse(0); + } + + /** + * Get initial delay between retry attempts. + * @return Initial delay duration (default: 100ms) + */ + public Duration retryInitialDelay() { + return this.durationValue("cache", "retry", "initial_delay") + .orElse(DEFAULT_RETRY_INITIAL_DELAY); + } + + /** + * Get backoff multiplier for retry delays. + * @return Backoff multiplier (default: 2.0) + */ + public double retryBackoffMultiplier() { + return this.doubleValue("cache", "retry", "backoff_multiplier") + .orElse(DEFAULT_RETRY_BACKOFF_MULTIPLIER); + } + + /** + * Check if proxy metrics recording is enabled. + * @return True if enabled (default: true) + */ + public boolean metricsEnabled() { + return this.boolValue("cache", "metrics").orElse(true); + } + /** * Check if any caching is configured. * @return True if cache section exists @@ -113,14 +220,7 @@ public boolean hasCacheConfig() { * @return Optional boolean value */ private Optional boolValue(final String... path) { - YamlMapping current = this.yaml; - for (int i = 0; i < path.length - 1; i++) { - current = current.yamlMapping(path[i]); - if (current == null) { - return Optional.empty(); - } - } - final String value = current.string(path[path.length - 1]); + final String value = this.rawValue(path); return value == null ? Optional.empty() : Optional.of(Boolean.parseBoolean(value)); } @@ -130,14 +230,7 @@ private Optional boolValue(final String... path) { * @return Optional integer value */ private Optional intValue(final String... path) { - YamlMapping current = this.yaml; - for (int i = 0; i < path.length - 1; i++) { - current = current.yamlMapping(path[i]); - if (current == null) { - return Optional.empty(); - } - } - final String value = current.string(path[path.length - 1]); + final String value = this.rawValue(path); try { return value == null ? Optional.empty() : Optional.of(Integer.parseInt(value)); } catch (final NumberFormatException ex) { @@ -145,6 +238,20 @@ private Optional intValue(final String... path) { } } + /** + * Get double value from nested YAML path. + * @param path YAML path segments + * @return Optional double value + */ + private Optional doubleValue(final String... path) { + final String value = this.rawValue(path); + try { + return value == null ? Optional.empty() : Optional.of(Double.parseDouble(value)); + } catch (final NumberFormatException ex) { + return Optional.empty(); + } + } + /** * Get duration value from nested YAML path. * Supports ISO-8601 duration format (e.g., PT24H, P1D). @@ -152,14 +259,7 @@ private Optional intValue(final String... path) { * @return Optional duration value */ private Optional durationValue(final String... path) { - YamlMapping current = this.yaml; - for (int i = 0; i < path.length - 1; i++) { - current = current.yamlMapping(path[i]); - if (current == null) { - return Optional.empty(); - } - } - final String value = current.string(path[path.length - 1]); + final String value = this.rawValue(path); try { return value == null ? Optional.empty() : Optional.of(Duration.parse(value)); } catch (final Exception ex) { @@ -167,11 +267,57 @@ private Optional durationValue(final String... path) { } } + /** + * Get string value from nested YAML path. + * @param path YAML path segments + * @return Optional string value + */ + private Optional stringValue(final String... path) { + return Optional.ofNullable(this.rawValue(path)); + } + + /** + * Navigate YAML path and return raw string value at leaf. + * @param path YAML path segments + * @return Raw string value or null + */ + private String rawValue(final String... path) { + YamlMapping current = this.yaml; + for (int idx = 0; idx < path.length - 1; idx++) { + current = current.yamlMapping(path[idx]); + if (current == null) { + return null; + } + } + return current.string(path[path.length - 1]); + } + /** * Create default configuration (all caching enabled with defaults). * @return Default configuration */ public static ProxyCacheConfig defaults() { - return new ProxyCacheConfig(com.amihaiemil.eoyaml.Yaml.createYamlMappingBuilder().build()); + return new ProxyCacheConfig( + com.amihaiemil.eoyaml.Yaml.createYamlMappingBuilder().build() + ); + } + + /** + * Create configuration with cooldown enabled. + * Used by adapters that support cooldown enforcement (e.g., Maven proxy). + * @return Configuration with cooldown enabled + */ + public static ProxyCacheConfig withCooldown() { + return new ProxyCacheConfig( + com.amihaiemil.eoyaml.Yaml.createYamlMappingBuilder() + .add("cache", + com.amihaiemil.eoyaml.Yaml.createYamlMappingBuilder() + .add("cooldown", + com.amihaiemil.eoyaml.Yaml.createYamlMappingBuilder() + .add("enabled", "true") + .build()) + .build()) + .build() + ); } } diff --git a/artipie-core/src/main/java/com/artipie/http/cache/RequestDeduplicator.java b/artipie-core/src/main/java/com/artipie/http/cache/RequestDeduplicator.java new file mode 100644 index 000000000..883cc9971 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/cache/RequestDeduplicator.java @@ -0,0 +1,198 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import com.artipie.asto.Key; +import com.artipie.http.misc.ConfigDefaults; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +/** + * Deduplicates concurrent requests for the same cache key. + * + *

When multiple clients request the same artifact simultaneously, only one + * upstream fetch is performed. Other callers either wait for the signal (SIGNAL + * strategy) or are coalesced at the storage level (STORAGE strategy). + * + *

With SIGNAL strategy (default): + *

    + *
  • First request: executes the supplier, signals result on completion
    • + *
  • Waiting requests: receive the same signal (SUCCESS, NOT_FOUND, ERROR)
    • + *
  • After completion: entry is removed from in-flight map
    • + *
+ * + *

With NONE strategy, every call immediately delegates to the supplier. + * + * @since 1.20.13 + */ +public final class RequestDeduplicator implements AutoCloseable { + + /** + * Maximum age of an in-flight entry before it's considered zombie (5 minutes). + * Configurable via ARTIPIE_DEDUP_MAX_AGE_MS environment variable. + */ + private static final long MAX_AGE_MS = + ConfigDefaults.getLong("ARTIPIE_DEDUP_MAX_AGE_MS", 300_000L); + + /** + * Maps cache key to the in-flight fetch entry (future + creation time). + */ + private final ConcurrentHashMap inFlight; + + /** + * Strategy to use. + */ + private final DedupStrategy strategy; + + /** + * Cleanup scheduler. + */ + private final java.util.concurrent.ScheduledExecutorService cleanup; + + /** + * Ctor. + * @param strategy Dedup strategy + */ + public RequestDeduplicator(final DedupStrategy strategy) { + this.strategy = Objects.requireNonNull(strategy, "strategy"); + this.inFlight = new ConcurrentHashMap<>(); + this.cleanup = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { + final Thread thread = new Thread(r, "dedup-cleanup"); + thread.setDaemon(true); + return thread; + }); + this.cleanup.scheduleAtFixedRate(this::evictStale, 60, 60, java.util.concurrent.TimeUnit.SECONDS); + } + + /** + * Execute a fetch with deduplication. + * + *

If a fetch for the same key is already in progress and strategy is SIGNAL, + * this call returns a future that completes when the existing fetch completes. + * + * @param key Cache key identifying the artifact + * @param fetcher Supplier that performs the actual upstream fetch. + * Must complete the returned future with a FetchSignal. + * @return Future with the fetch signal (SUCCESS, NOT_FOUND, or ERROR) + */ + public CompletableFuture deduplicate( + final Key key, + final Supplier> fetcher + ) { + if (this.strategy == DedupStrategy.NONE || this.strategy == DedupStrategy.STORAGE) { + return fetcher.get(); + } + final CompletableFuture fresh = new CompletableFuture<>(); + final InFlightEntry freshEntry = new InFlightEntry(fresh, System.currentTimeMillis()); + final InFlightEntry existing = this.inFlight.putIfAbsent(key, freshEntry); + if (existing != null) { + return existing.future; + } + fetcher.get().whenComplete((signal, err) -> { + this.inFlight.remove(key); + if (err != null) { + fresh.complete(FetchSignal.ERROR); + } else { + fresh.complete(signal); + } + }); + return fresh; + } + + /** + * Get the number of currently in-flight requests. For monitoring. + * @return Count of in-flight dedup entries + */ + public int inFlightCount() { + return this.inFlight.size(); + } + + /** + * Remove entries that have been in-flight for too long (zombie protection). + */ + private void evictStale() { + final long now = System.currentTimeMillis(); + this.inFlight.entrySet().removeIf(entry -> { + if (now - entry.getValue().createdAt > MAX_AGE_MS) { + entry.getValue().future.complete(FetchSignal.ERROR); + return true; + } + return false; + }); + } + + /** + * Shuts down the cleanup scheduler and completes all in-flight entries with ERROR. + * Should be called when the deduplicator is no longer needed. + */ + @Override + public void close() { + this.cleanup.shutdownNow(); + this.inFlight.values().forEach( + entry -> entry.future.complete(FetchSignal.ERROR) + ); + this.inFlight.clear(); + } + + /** + * Alias for {@link #close()}, for explicit lifecycle management. + */ + public void shutdown() { + this.close(); + } + + /** + * In-flight entry tracking future and creation time. + */ + private static final class InFlightEntry { + /** + * The future for the in-flight fetch. + */ + final CompletableFuture future; + + /** + * Timestamp when this entry was created. + */ + final long createdAt; + + /** + * Ctor. + * @param future The future for the in-flight fetch + * @param createdAt Timestamp when this entry was created + */ + InFlightEntry(final CompletableFuture future, final long createdAt) { + this.future = future; + this.createdAt = createdAt; + } + } + + /** + * Signal indicating the outcome of a deduplicated fetch. + * + * @since 1.20.13 + */ + public enum FetchSignal { + /** + * Upstream returned 200 and content is now cached in storage. + * Waiting callers should read from cache. + */ + SUCCESS, + + /** + * Upstream returned 404. Negative cache has been updated. + * Waiting callers should return 404. + */ + NOT_FOUND, + + /** + * Upstream returned an error (5xx, timeout, exception). + * Waiting callers should return 503 or fall back to stale cache. + */ + ERROR + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/cache/SidecarFile.java b/artipie-core/src/main/java/com/artipie/http/cache/SidecarFile.java new file mode 100644 index 000000000..4551a4521 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/cache/SidecarFile.java @@ -0,0 +1,28 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import java.util.Objects; + +/** + * Checksum sidecar file generated alongside a cached artifact. + * For example, Maven generates .sha1, .sha256, .md5 files next to each artifact. + * + * @param path Sidecar file path (e.g., "com/example/foo/1.0/foo-1.0.jar.sha256") + * @param content Sidecar file content (the hex-encoded checksum string as bytes) + * @since 1.20.13 + */ +public record SidecarFile(String path, byte[] content) { + + /** + * Ctor with validation. + * @param path Sidecar file path + * @param content Sidecar file content + */ + public SidecarFile { + Objects.requireNonNull(path, "path"); + Objects.requireNonNull(content, "content"); + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/headers/Login.java b/artipie-core/src/main/java/com/artipie/http/headers/Login.java index af882d641..29028fdc6 100644 --- a/artipie-core/src/main/java/com/artipie/http/headers/Login.java +++ b/artipie-core/src/main/java/com/artipie/http/headers/Login.java @@ -6,6 +6,7 @@ import com.artipie.http.Headers; import com.artipie.http.auth.AuthzSlice; +import com.artipie.http.log.EcsLogger; import com.artipie.scheduling.ArtifactEvent; import org.slf4j.MDC; @@ -84,7 +85,11 @@ private static Optional decodeAuthorization(final String header) { if (!credentials.isBlank()) { return Optional.of(credentials); } - } catch (final IllegalArgumentException ignored) { + } catch (final IllegalArgumentException ex) { + EcsLogger.debug("com.artipie.http") + .message("Failed to decode Basic auth credentials") + .error(ex) + .log(); return Optional.empty(); } } diff --git a/artipie-core/src/main/java/com/artipie/http/misc/ConfigDefaults.java b/artipie-core/src/main/java/com/artipie/http/misc/ConfigDefaults.java new file mode 100644 index 000000000..9dd715bee --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/misc/ConfigDefaults.java @@ -0,0 +1,63 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +/** + * Centralized configuration defaults with environment variable overrides. + * Values are read with precedence: env var > system property > default. + * + * @since 1.20.13 + */ +public final class ConfigDefaults { + + private ConfigDefaults() { + } + + /** + * Read a configuration value. + * @param envVar Environment variable name + * @param defaultValue Default value if not set + * @return Configured value or default + */ + public static String get(final String envVar, final String defaultValue) { + final String env = System.getenv(envVar); + if (env != null && !env.isEmpty()) { + return env; + } + final String prop = System.getProperty(envVar.toLowerCase().replace('_', '.')); + if (prop != null && !prop.isEmpty()) { + return prop; + } + return defaultValue; + } + + /** + * Read an integer configuration value. + * @param envVar Environment variable name + * @param defaultValue Default value + * @return Configured value or default + */ + public static int getInt(final String envVar, final int defaultValue) { + try { + return Integer.parseInt(get(envVar, String.valueOf(defaultValue))); + } catch (final NumberFormatException e) { + return defaultValue; + } + } + + /** + * Read a long configuration value. + * @param envVar Environment variable name + * @param defaultValue Default value + * @return Configured value or default + */ + public static long getLong(final String envVar, final long defaultValue) { + try { + return Long.parseLong(get(envVar, String.valueOf(defaultValue))); + } catch (final NumberFormatException e) { + return defaultValue; + } + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/misc/DispatchedStorage.java b/artipie-core/src/main/java/com/artipie/http/misc/DispatchedStorage.java new file mode 100644 index 000000000..e8da6cf0e --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/misc/DispatchedStorage.java @@ -0,0 +1,151 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +import com.artipie.asto.Content; +import com.artipie.asto.Key; +import com.artipie.asto.ListResult; +import com.artipie.asto.Meta; +import com.artipie.asto.Storage; + +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.function.Function; + +/** + * Decorator that wraps any {@link Storage} and dispatches completion + * continuations to the named thread pools from {@link StorageExecutors}. + *

+ * Each storage operation category is dispatched to its own pool: + *

    + *
  • READ ops (exists, value, metadata) use {@link StorageExecutors#READ}
    • + *
  • WRITE ops (save, move, delete) use {@link StorageExecutors#WRITE}
    • + *
  • LIST ops (list) use {@link StorageExecutors#LIST}
    • + *
+ *

+ * The {@code exclusively()} method delegates directly without dispatching + * to avoid deadlocks with lock management. The {@code identifier()} method + * also delegates directly as it is synchronous with no I/O. + * + * @since 1.20.13 + */ +public final class DispatchedStorage implements Storage { + + /** + * Delegate storage. + */ + private final Storage delegate; + + /** + * Wraps the given storage with thread pool dispatching. + * @param delegate Storage to wrap + */ + public DispatchedStorage(final Storage delegate) { + this.delegate = delegate; + } + + @Override + public CompletableFuture exists(final Key key) { + return dispatch(this.delegate.exists(key), StorageExecutors.READ); + } + + @Override + public CompletableFuture> list(final Key prefix) { + return dispatch(this.delegate.list(prefix), StorageExecutors.LIST); + } + + @Override + public CompletableFuture list(final Key prefix, final String delimiter) { + return dispatch(this.delegate.list(prefix, delimiter), StorageExecutors.LIST); + } + + @Override + public CompletableFuture save(final Key key, final Content content) { + return dispatch(this.delegate.save(key, content), StorageExecutors.WRITE); + } + + @Override + public CompletableFuture move(final Key source, final Key destination) { + return dispatch(this.delegate.move(source, destination), StorageExecutors.WRITE); + } + + @Override + public CompletableFuture metadata(final Key key) { + return dispatch(this.delegate.metadata(key), StorageExecutors.READ); + } + + @Override + public CompletableFuture value(final Key key) { + return dispatch(this.delegate.value(key), StorageExecutors.READ); + } + + @Override + public CompletableFuture delete(final Key key) { + return dispatch(this.delegate.delete(key), StorageExecutors.WRITE); + } + + @Override + public CompletableFuture deleteAll(final Key prefix) { + return dispatch(this.delegate.deleteAll(prefix), StorageExecutors.WRITE); + } + + @Override + public CompletionStage exclusively( + final Key key, + final Function> operation + ) { + return this.delegate.exclusively(key, operation); + } + + /** + * Returns the underlying delegate storage. + * Useful for inspecting the actual storage type when this decorator wraps it. + * @return The delegate storage + */ + public Storage unwrap() { + return this.delegate; + } + + @Override + public String identifier() { + return this.delegate.identifier(); + } + + /** + * Dispatch a future's completion to the given executor. + * Guarantees the returned future is always completed by a thread + * from the target executor, so downstream {@code thenApply()} / + * {@code thenCompose()} continuations run on that pool. + * + * @param source Source future from the delegate storage + * @param executor Target executor pool + * @param Result type + * @return Future that completes on the target executor + */ + private static CompletableFuture dispatch( + final CompletableFuture source, + final Executor executor + ) { + final CompletableFuture result = new CompletableFuture<>(); + source.whenComplete( + (val, err) -> { + try { + executor.execute(() -> { + if (err != null) { + result.completeExceptionally(err); + } else { + result.complete(val); + } + }); + } catch (final java.util.concurrent.RejectedExecutionException rex) { + result.completeExceptionally(rex); + } + } + ); + return result; + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/misc/RepoNameMeterFilter.java b/artipie-core/src/main/java/com/artipie/http/misc/RepoNameMeterFilter.java new file mode 100644 index 000000000..0517a4555 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/misc/RepoNameMeterFilter.java @@ -0,0 +1,79 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.config.MeterFilter; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** + * Meter filter that caps the cardinality of the "repo_name" tag. + * Only the first N distinct repo names are kept; additional repos are + * replaced with "_other" to prevent unbounded series growth. + * + * @since 1.20.13 + */ +public final class RepoNameMeterFilter implements MeterFilter { + + /** + * Tag name to filter. + */ + private static final String TAG_NAME = "repo_name"; + + /** + * Maximum number of distinct repo_name values. + */ + private final int maxRepos; + + /** + * Known repo names (first N to be seen). + */ + private final Set known; + + /** + * Counter for tracking how many distinct repos we've seen. + */ + private final AtomicInteger count; + + /** + * Constructor. + * @param maxRepos Maximum distinct repo names to track + */ + public RepoNameMeterFilter(final int maxRepos) { + this.maxRepos = maxRepos; + this.known = ConcurrentHashMap.newKeySet(); + this.count = new AtomicInteger(0); + } + + @Override + public Meter.Id map(final Meter.Id id) { + final String repoName = id.getTag(TAG_NAME); + if (repoName == null) { + return id; + } + if (this.known.contains(repoName)) { + return id; + } + if (this.count.get() < this.maxRepos) { + if (this.known.add(repoName)) { + this.count.incrementAndGet(); + } + return id; + } + // Over limit — replace tag with _other + final List newTags = id.getTags().stream() + .map(tag -> TAG_NAME.equals(tag.getKey()) + ? Tag.of(TAG_NAME, "_other") + : tag) + .collect(Collectors.toList()); + return id.replaceTags(newTags); + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/misc/StorageExecutors.java b/artipie-core/src/main/java/com/artipie/http/misc/StorageExecutors.java new file mode 100644 index 000000000..de492c9d5 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/misc/StorageExecutors.java @@ -0,0 +1,146 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Named thread pools for storage operations, separated by operation type. + * Prevents slow writes from starving fast reads by providing independent pools. + * + *

Pool sizing (configurable via environment variables): + *

    + *
  • READ: ARTIPIE_IO_READ_THREADS, default 4x CPUs
    • + *
  • WRITE: ARTIPIE_IO_WRITE_THREADS, default 2x CPUs
    • + *
  • LIST: ARTIPIE_IO_LIST_THREADS, default 1x CPUs
    • + *
+ * + * @since 1.20.13 + */ +public final class StorageExecutors { + + /** + * Thread pool for storage read operations (value, exists, metadata). + */ + public static final ExecutorService READ = Executors.newFixedThreadPool( + ConfigDefaults.getInt( + "ARTIPIE_IO_READ_THREADS", + Runtime.getRuntime().availableProcessors() * 4 + ), + namedThreadFactory("artipie-io-read-%d") + ); + + /** + * Thread pool for storage write operations (save, move, delete). + */ + public static final ExecutorService WRITE = Executors.newFixedThreadPool( + ConfigDefaults.getInt( + "ARTIPIE_IO_WRITE_THREADS", + Runtime.getRuntime().availableProcessors() * 2 + ), + namedThreadFactory("artipie-io-write-%d") + ); + + /** + * Thread pool for storage list operations. + */ + public static final ExecutorService LIST = Executors.newFixedThreadPool( + ConfigDefaults.getInt( + "ARTIPIE_IO_LIST_THREADS", + Runtime.getRuntime().availableProcessors() + ), + namedThreadFactory("artipie-io-list-%d") + ); + + private StorageExecutors() { + // Utility class + } + + /** + * Register pool utilization metrics gauges with the given meter registry. + * Registers active thread count and queue size for each pool (READ, WRITE, LIST). + * @param registry Micrometer meter registry + */ + public static void registerMetrics(final MeterRegistry registry) { + Gauge.builder( + "artipie.pool.read.active", READ, + pool -> ((ThreadPoolExecutor) pool).getActiveCount() + ).description("Active threads in READ pool").register(registry); + Gauge.builder( + "artipie.pool.write.active", WRITE, + pool -> ((ThreadPoolExecutor) pool).getActiveCount() + ).description("Active threads in WRITE pool").register(registry); + Gauge.builder( + "artipie.pool.list.active", LIST, + pool -> ((ThreadPoolExecutor) pool).getActiveCount() + ).description("Active threads in LIST pool").register(registry); + Gauge.builder( + "artipie.pool.read.queue", READ, + pool -> ((ThreadPoolExecutor) pool).getQueue().size() + ).description("Queue size of READ pool").register(registry); + Gauge.builder( + "artipie.pool.write.queue", WRITE, + pool -> ((ThreadPoolExecutor) pool).getQueue().size() + ).description("Queue size of WRITE pool").register(registry); + Gauge.builder( + "artipie.pool.list.queue", LIST, + pool -> ((ThreadPoolExecutor) pool).getQueue().size() + ).description("Queue size of LIST pool").register(registry); + } + + /** + * Shutdown all storage executor pools and await termination. + * Should be called during application shutdown. + */ + @SuppressWarnings("PMD.AvoidCatchingGenericException") + public static void shutdown() { + READ.shutdown(); + WRITE.shutdown(); + LIST.shutdown(); + try { + if (!READ.awaitTermination(5, TimeUnit.SECONDS)) { + READ.shutdownNow(); + } + if (!WRITE.awaitTermination(5, TimeUnit.SECONDS)) { + WRITE.shutdownNow(); + } + if (!LIST.awaitTermination(5, TimeUnit.SECONDS)) { + LIST.shutdownNow(); + } + } catch (final InterruptedException ex) { + Thread.currentThread().interrupt(); + READ.shutdownNow(); + WRITE.shutdownNow(); + LIST.shutdownNow(); + } + } + + /** + * Create a named daemon thread factory. + * @param nameFormat Thread name format with %d placeholder + * @return Thread factory + */ + private static ThreadFactory namedThreadFactory(final String nameFormat) { + return new ThreadFactory() { + private final AtomicInteger counter = new AtomicInteger(0); + @Override + public Thread newThread(final Runnable r) { + final Thread thread = new Thread(r); + thread.setName( + String.format(nameFormat, this.counter.getAndIncrement()) + ); + thread.setDaemon(true); + return thread; + } + }; + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/retry/RetrySlice.java b/artipie-core/src/main/java/com/artipie/http/retry/RetrySlice.java new file mode 100644 index 000000000..1c9f041fc --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/retry/RetrySlice.java @@ -0,0 +1,167 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.retry; + +import com.artipie.asto.Content; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.Slice; +import com.artipie.http.rq.RequestLine; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +/** + * Slice decorator that retries failed requests with exponential backoff. + *

Retries on: 5xx status codes, connection timeouts, exceptions. + * Does NOT retry on: 4xx client errors, successful responses.

+ * + * @since 1.20.13 + */ +public final class RetrySlice implements Slice { + + /** + * Default max retries. + */ + public static final int DEFAULT_MAX_RETRIES = 2; + + /** + * Default initial delay. + */ + public static final Duration DEFAULT_INITIAL_DELAY = Duration.ofMillis(100); + + /** + * Default backoff multiplier. + */ + public static final double DEFAULT_BACKOFF_MULTIPLIER = 2.0; + + /** + * Wrapped slice. + */ + private final Slice origin; + + /** + * Maximum number of retry attempts. + */ + private final int maxRetries; + + /** + * Initial delay before first retry. + */ + private final Duration initialDelay; + + /** + * Backoff multiplier for subsequent retries. + */ + private final double backoffMultiplier; + + /** + * Constructor with defaults. + * @param origin Slice to wrap + */ + public RetrySlice(final Slice origin) { + this(origin, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY, DEFAULT_BACKOFF_MULTIPLIER); + } + + /** + * Constructor with custom configuration. + * @param origin Slice to wrap + * @param maxRetries Maximum retry attempts + * @param initialDelay Initial delay before first retry + * @param backoffMultiplier Multiplier for exponential backoff + */ + public RetrySlice( + final Slice origin, + final int maxRetries, + final Duration initialDelay, + final double backoffMultiplier + ) { + this.origin = Objects.requireNonNull(origin, "origin"); + this.maxRetries = maxRetries; + this.initialDelay = Objects.requireNonNull(initialDelay, "initialDelay"); + this.backoffMultiplier = backoffMultiplier; + } + + @Override + public CompletableFuture response( + final RequestLine line, + final Headers headers, + final Content body + ) { + return this.attempt(line, headers, body, 0, this.initialDelay.toMillis()); + } + + /** + * Attempt a request, retrying on failure with exponential backoff. + * @param line Request line + * @param headers Request headers + * @param body Request body + * @param attempt Current attempt number (0-based) + * @param delayMs Current delay in milliseconds + * @return Response future + */ + private CompletableFuture attempt( + final RequestLine line, + final Headers headers, + final Content body, + final int attempt, + final long delayMs + ) { + return this.origin.response(line, headers, body) + .>handle((response, error) -> { + if (error != null) { + if (attempt < this.maxRetries) { + return this.delayedAttempt( + line, headers, body, attempt + 1, + (long) (delayMs * this.backoffMultiplier) + ); + } + return CompletableFuture.failedFuture(error); + } + if (shouldRetry(response) && attempt < this.maxRetries) { + return this.delayedAttempt( + line, headers, body, attempt + 1, + (long) (delayMs * this.backoffMultiplier) + ); + } + return CompletableFuture.completedFuture(response); + }) + .thenCompose(Function.identity()); + } + + /** + * Schedule a retry attempt after a delay with jitter. + * Jitter prevents thundering herd by adding random 0-50% to the delay. + */ + private CompletableFuture delayedAttempt( + final RequestLine line, + final Headers headers, + final Content body, + final int attempt, + final long delayMs + ) { + // Add jitter: delay * (1.0 + random[0, 0.5)) to prevent thundering herd + final long jitteredDelay = (long) (delayMs + * (1.0 + java.util.concurrent.ThreadLocalRandom.current().nextDouble(0.5))); + final Executor delayed = CompletableFuture.delayedExecutor( + jitteredDelay, TimeUnit.MILLISECONDS + ); + return CompletableFuture.supplyAsync(() -> null, delayed) + .thenCompose(ignored -> this.attempt(line, headers, body, attempt, delayMs)); + } + + /** + * Whether to retry based on response status. + * @param response HTTP response + * @return True if response indicates a retryable server error + */ + private static boolean shouldRetry(final Response response) { + return response.status().code() >= 500; + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/slice/CircuitBreakerSlice.java b/artipie-core/src/main/java/com/artipie/http/slice/CircuitBreakerSlice.java index 50753cab1..f51ebe0c7 100644 --- a/artipie-core/src/main/java/com/artipie/http/slice/CircuitBreakerSlice.java +++ b/artipie-core/src/main/java/com/artipie/http/slice/CircuitBreakerSlice.java @@ -10,250 +10,63 @@ import com.artipie.http.ResponseBuilder; import com.artipie.http.Slice; import com.artipie.http.rq.RequestLine; -import com.artipie.http.log.EcsLogger; +import com.artipie.http.timeout.AutoBlockRegistry; -import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; /** - * Circuit Breaker pattern for upstream repositories. - * Prevents hammering failed upstream by failing fast after threshold. - * - *

States: - *

    - *
  • CLOSED: Normal operation, requests pass through
    • - *
  • OPEN: Too many failures, fail fast without calling upstream
    • - *
  • HALF_OPEN: Testing if upstream recovered, single request allowed
    • - *
- * + * Circuit breaker slice delegating to {@link AutoBlockRegistry}. + * Fails fast with 503 when the remote is auto-blocked. + * Records success/failure to the registry after each request. + * * @since 1.0 */ public final class CircuitBreakerSlice implements Slice { - /** - * Circuit breaker state. - */ - enum State { - /** - * Normal operation. - */ - CLOSED, - - /** - * Failing fast. - */ - OPEN, - - /** - * Testing recovery. - */ - HALF_OPEN - } - - /** - * Default failure threshold before opening circuit. - */ - private static final int DEFAULT_FAILURE_THRESHOLD = 5; - - /** - * Default timeout before trying again. - */ - private static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(1); - - /** - * Origin slice (upstream). - */ private final Slice origin; + private final AutoBlockRegistry registry; + private final String remoteId; /** - * Current circuit state. - */ - private final AtomicReference state; - - /** - * Consecutive failure count. - */ - private final AtomicInteger failureCount; - - /** - * Timestamp of last failure. - */ - private final AtomicLong lastFailureTime; - - /** - * Failure threshold before opening circuit. - */ - private final int failureThreshold; - - /** - * Timeout before retrying (millis). - */ - private final long timeoutMillis; - - /** - * Constructor with defaults. - * @param origin Origin slice - */ - public CircuitBreakerSlice(final Slice origin) { - this(origin, DEFAULT_FAILURE_THRESHOLD, DEFAULT_TIMEOUT); - } - - /** - * Constructor with custom settings. - * @param origin Origin slice - * @param failureThreshold Failures before opening circuit - * @param timeout Timeout before retrying + * Constructor. + * @param origin Origin slice (upstream) + * @param registry Shared auto-block registry + * @param remoteId Unique identifier for this remote */ public CircuitBreakerSlice( final Slice origin, - final int failureThreshold, - final Duration timeout + final AutoBlockRegistry registry, + final String remoteId ) { this.origin = origin; - this.state = new AtomicReference<>(State.CLOSED); - this.failureCount = new AtomicInteger(0); - this.lastFailureTime = new AtomicLong(0); - this.failureThreshold = failureThreshold; - this.timeoutMillis = timeout.toMillis(); + this.registry = registry; + this.remoteId = remoteId; } @Override public CompletableFuture response( - final RequestLine line, - final Headers headers, - final Content body + final RequestLine line, final Headers headers, final Content body ) { - final State currentState = this.state.get(); - - // Check if circuit is open - if (currentState == State.OPEN) { - final long timeSinceFailure = System.currentTimeMillis() - this.lastFailureTime.get(); - - if (timeSinceFailure > this.timeoutMillis) { - // Timeout expired - try half-open - EcsLogger.info("com.artipie.http") - .message("Circuit breaker HALF_OPEN, testing upstream after " + timeSinceFailure + "ms since last failure") - .eventCategory("circuit_breaker") - .eventAction("state_change") - .eventOutcome("success") - .log(); - this.state.compareAndSet(State.OPEN, State.HALF_OPEN); - } else { - // Still open - fail fast - EcsLogger.debug("com.artipie.http") - .message("Circuit breaker OPEN, failing fast with " + this.failureCount.get() + " failures (" + timeSinceFailure + "ms since last failure)") - .eventCategory("circuit_breaker") - .eventAction("fail_fast") - .eventOutcome("success") - .log(); - return CompletableFuture.completedFuture( - ResponseBuilder.serviceUnavailable( - "Circuit breaker open - upstream unavailable" - ).build() - ); - } + if (this.registry.isBlocked(this.remoteId)) { + return CompletableFuture.completedFuture( + ResponseBuilder.serviceUnavailable( + "Auto-blocked - remote unavailable: " + this.remoteId + ).build() + ); } - - // Try request return this.origin.response(line, headers, body) .handle((resp, error) -> { if (error != null) { - // Request failed - onFailure(error); + this.registry.recordFailure(this.remoteId); throw new CompletionException(error); } - - // Check response status - final int statusCode = resp.status().code(); - if (statusCode >= 500 && statusCode < 600) { - // Server error - count as failure - onFailure(new IllegalStateException("HTTP " + statusCode)); - throw new CompletionException( - new IllegalStateException("Upstream error: " + statusCode) - ); + if (resp.status().code() >= 500) { + this.registry.recordFailure(this.remoteId); + } else { + this.registry.recordSuccess(this.remoteId); } - - // Success - onSuccess(); return resp; }); } - - /** - * Handle successful request. - */ - private void onSuccess() { - final int failures = this.failureCount.getAndSet(0); - - final State currentState = this.state.get(); - if (currentState == State.HALF_OPEN) { - // Recovery successful - this.state.compareAndSet(State.HALF_OPEN, State.CLOSED); - EcsLogger.info("com.artipie.http") - .message("Circuit breaker CLOSED - upstream recovered after " + failures + " previous failures") - .eventCategory("circuit_breaker") - .eventAction("state_change") - .eventOutcome("success") - .log(); - } else if (failures > 0) { - // Reset failure count - EcsLogger.debug("com.artipie.http") - .message("Circuit breaker reset failure count (" + failures + " previous failures)") - .eventCategory("circuit_breaker") - .eventAction("failure_reset") - .eventOutcome("success") - .log(); - } - } - - /** - * Handle failed request. - * @param error Error that occurred - */ - private void onFailure(final Throwable error) { - final int failures = this.failureCount.incrementAndGet(); - this.lastFailureTime.set(System.currentTimeMillis()); - - if (failures >= this.failureThreshold) { - // Open circuit - final boolean wasOpen = this.state.getAndSet(State.OPEN) == State.OPEN; - if (!wasOpen) { - EcsLogger.warn("com.artipie.http") - .message("Circuit breaker OPENED after " + failures + " failures (threshold: " + this.failureThreshold + ")") - .eventCategory("circuit_breaker") - .eventAction("state_change") - .eventOutcome("failure") - .field("error.message", error.getMessage()) - .log(); - } - } else { - EcsLogger.debug("com.artipie.http") - .message("Circuit breaker failure recorded (" + failures + "/" + this.failureThreshold + " failures)") - .eventCategory("circuit_breaker") - .eventAction("failure_record") - .eventOutcome("failure") - .field("error.message", error.getMessage()) - .log(); - } - } - - /** - * Get current circuit state (for testing/monitoring). - * @return Current state - */ - public State getState() { - return this.state.get(); - } - - /** - * Get current failure count (for testing/monitoring). - * @return Failure count - */ - public int getFailureCount() { - return this.failureCount.get(); - } } diff --git a/artipie-core/src/main/java/com/artipie/http/slice/LoggingSlice.java b/artipie-core/src/main/java/com/artipie/http/slice/LoggingSlice.java index 3598b61d4..7723c9313 100644 --- a/artipie-core/src/main/java/com/artipie/http/slice/LoggingSlice.java +++ b/artipie-core/src/main/java/com/artipie/http/slice/LoggingSlice.java @@ -58,10 +58,9 @@ public CompletableFuture response( // Log request at DEBUG level (diagnostic only) if (this.level.intValue() <= Level.FINE.intValue()) { EcsLogger.debug("com.artipie.http") - .message("HTTP request") + .message("HTTP request: " + msg.toString()) .eventCategory("http") .eventAction("request") - .field("http.request.body.content", msg.toString()) .log(); } @@ -74,10 +73,9 @@ public CompletableFuture response( // Log response at DEBUG level (diagnostic only) if (LoggingSlice.this.level.intValue() <= Level.FINE.intValue()) { EcsLogger.debug("com.artipie.http") - .message("HTTP response") + .message("HTTP response: " + sb.toString()) .eventCategory("http") .eventAction("response") - .field("http.response.body.content", sb.toString()) .log(); } diff --git a/artipie-core/src/main/java/com/artipie/http/slice/TrimPathSlice.java b/artipie-core/src/main/java/com/artipie/http/slice/TrimPathSlice.java index 2faf70344..30a4f1ff3 100644 --- a/artipie-core/src/main/java/com/artipie/http/slice/TrimPathSlice.java +++ b/artipie-core/src/main/java/com/artipie/http/slice/TrimPathSlice.java @@ -138,9 +138,10 @@ private static String asPath(final String result) { if (result == null || result.isEmpty()) { return "/"; } - if (result.charAt(0) != '/') { - return '/' + result; + String path = result; + if (path.charAt(0) != '/') { + path = '/' + path; } - return result; + return path.replaceAll("/+", "/"); } } diff --git a/artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockRegistry.java b/artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockRegistry.java new file mode 100644 index 000000000..ee1ddac30 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockRegistry.java @@ -0,0 +1,110 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.timeout; + +import java.time.Instant; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Thread-safe registry tracking auto-block state for remote endpoints. + * Uses Fibonacci backoff for increasing block durations. + * Industry-standard approach used by Nexus and Artifactory. + * + * @since 1.20.13 + */ +public final class AutoBlockRegistry { + + /** + * Fibonacci multiplier sequence. + */ + private static final long[] FIBONACCI = {1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89}; + + private final AutoBlockSettings settings; + private final ConcurrentMap states; + + public AutoBlockRegistry(final AutoBlockSettings settings) { + this.settings = settings; + this.states = new ConcurrentHashMap<>(); + } + + /** + * Check if a remote is currently blocked. + * If the block has expired, transitions to PROBING state and returns false. + */ + public boolean isBlocked(final String remoteId) { + final BlockState state = this.states.getOrDefault( + remoteId, BlockState.online() + ); + if (state.status() == BlockState.Status.BLOCKED) { + if (Instant.now().isAfter(state.blockedUntil())) { + this.states.put( + remoteId, + new BlockState( + state.failureCount(), state.fibonacciIndex(), + state.blockedUntil(), BlockState.Status.PROBING + ) + ); + return false; + } + return true; + } + return false; + } + + /** + * Get the current status of a remote: "online", "blocked", or "probing". + */ + public String status(final String remoteId) { + final BlockState state = this.states.getOrDefault( + remoteId, BlockState.online() + ); + if (state.status() == BlockState.Status.BLOCKED + && Instant.now().isAfter(state.blockedUntil())) { + return "probing"; + } + return state.status().name().toLowerCase(Locale.ROOT); + } + + /** + * Record a failure for a remote. If the failure threshold is reached, + * blocks the remote with Fibonacci-increasing duration. + */ + public void recordFailure(final String remoteId) { + this.states.compute(remoteId, (key, current) -> { + final BlockState state = + current != null ? current : BlockState.online(); + final int failures = state.failureCount() + 1; + if (failures >= this.settings.failureThreshold()) { + final int fibIdx = state.status() == BlockState.Status.ONLINE + ? 0 + : Math.min( + state.fibonacciIndex() + 1, FIBONACCI.length - 1 + ); + final long blockMs = Math.min( + this.settings.initialBlockDuration().toMillis() + * FIBONACCI[fibIdx], + this.settings.maxBlockDuration().toMillis() + ); + return new BlockState( + failures, fibIdx, Instant.now().plusMillis(blockMs), + BlockState.Status.BLOCKED + ); + } + return new BlockState( + failures, state.fibonacciIndex(), + state.blockedUntil(), state.status() + ); + }); + } + + /** + * Record a success for a remote. Resets to ONLINE state. + */ + public void recordSuccess(final String remoteId) { + this.states.put(remoteId, BlockState.online()); + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockSettings.java b/artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockSettings.java new file mode 100644 index 000000000..79766ecec --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/timeout/AutoBlockSettings.java @@ -0,0 +1,23 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.timeout; + +import java.time.Duration; + +/** + * Configuration for auto-block behavior. All values configurable via YAML. + * + * @since 1.20.13 + */ +public record AutoBlockSettings( + int failureThreshold, + Duration initialBlockDuration, + Duration maxBlockDuration +) { + + public static AutoBlockSettings defaults() { + return new AutoBlockSettings(3, Duration.ofSeconds(40), Duration.ofMinutes(5)); + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/timeout/BlockState.java b/artipie-core/src/main/java/com/artipie/http/timeout/BlockState.java new file mode 100644 index 000000000..039b35491 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/timeout/BlockState.java @@ -0,0 +1,21 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.timeout; + +import java.time.Instant; + +/** + * Immutable block state for a remote endpoint. + * + * @since 1.20.13 + */ +record BlockState(int failureCount, int fibonacciIndex, Instant blockedUntil, Status status) { + + enum Status { ONLINE, BLOCKED, PROBING } + + static BlockState online() { + return new BlockState(0, 0, Instant.MIN, Status.ONLINE); + } +} diff --git a/artipie-core/src/main/java/com/artipie/http/timeout/TimeoutSettings.java b/artipie-core/src/main/java/com/artipie/http/timeout/TimeoutSettings.java new file mode 100644 index 000000000..aa1cac6bb --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/http/timeout/TimeoutSettings.java @@ -0,0 +1,93 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.timeout; + +import java.time.Duration; +import java.util.Objects; + +/** + * Immutable timeout configuration with hierarchical override support. + * Resolution order: per-remote > per-repo > global > defaults. + * + * @since 1.20.13 + */ +public final class TimeoutSettings { + + public static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofSeconds(5); + public static final Duration DEFAULT_IDLE_TIMEOUT = Duration.ofSeconds(30); + public static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(120); + + private final Duration connectionTimeout; + private final Duration idleTimeout; + private final Duration requestTimeout; + + public TimeoutSettings( + final Duration connectionTimeout, + final Duration idleTimeout, + final Duration requestTimeout + ) { + this.connectionTimeout = Objects.requireNonNull(connectionTimeout); + this.idleTimeout = Objects.requireNonNull(idleTimeout); + this.requestTimeout = Objects.requireNonNull(requestTimeout); + } + + public static TimeoutSettings defaults() { + return new TimeoutSettings( + DEFAULT_CONNECTION_TIMEOUT, DEFAULT_IDLE_TIMEOUT, DEFAULT_REQUEST_TIMEOUT + ); + } + + public static Builder builder() { + return new Builder(); + } + + public Duration connectionTimeout() { + return this.connectionTimeout; + } + + public Duration idleTimeout() { + return this.idleTimeout; + } + + public Duration requestTimeout() { + return this.requestTimeout; + } + + public static final class Builder { + private Duration connectionTimeout; + private Duration idleTimeout; + private Duration requestTimeout; + + public Builder connectionTimeout(final Duration val) { + this.connectionTimeout = val; + return this; + } + + public Builder idleTimeout(final Duration val) { + this.idleTimeout = val; + return this; + } + + public Builder requestTimeout(final Duration val) { + this.requestTimeout = val; + return this; + } + + public TimeoutSettings buildWithParent(final TimeoutSettings parent) { + return new TimeoutSettings( + this.connectionTimeout != null + ? this.connectionTimeout : parent.connectionTimeout(), + this.idleTimeout != null + ? this.idleTimeout : parent.idleTimeout(), + this.requestTimeout != null + ? this.requestTimeout : parent.requestTimeout() + ); + } + + public TimeoutSettings build() { + return buildWithParent(TimeoutSettings.defaults()); + } + } +} diff --git a/artipie-core/src/main/java/com/artipie/index/ArtifactDocument.java b/artipie-core/src/main/java/com/artipie/index/ArtifactDocument.java new file mode 100644 index 000000000..0fc4a659f --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/index/ArtifactDocument.java @@ -0,0 +1,42 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.index; + +import java.time.Instant; +import java.util.Objects; + +/** + * Artifact document for the search index. + * + * @param repoType Repository type (e.g., "maven", "npm", "pypi") + * @param repoName Repository name + * @param artifactPath Full artifact path (unique per repo) + * @param artifactName Human-readable artifact name (tokenized for search) + * @param version Artifact version + * @param size Artifact size in bytes + * @param createdAt Creation timestamp + * @param owner Owner/uploader username (nullable) + * @since 1.20.13 + */ +public record ArtifactDocument( + String repoType, + String repoName, + String artifactPath, + String artifactName, + String version, + long size, + Instant createdAt, + String owner +) { + + /** + * Ctor. + */ + public ArtifactDocument { + Objects.requireNonNull(repoType, "repoType"); + Objects.requireNonNull(repoName, "repoName"); + Objects.requireNonNull(artifactPath, "artifactPath"); + } +} diff --git a/artipie-core/src/main/java/com/artipie/index/ArtifactIndex.java b/artipie-core/src/main/java/com/artipie/index/ArtifactIndex.java new file mode 100644 index 000000000..70454190b --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/index/ArtifactIndex.java @@ -0,0 +1,139 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.index; + +import java.io.Closeable; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * Artifact search index interface. + * Supports full-text search, exact path lookup, and artifact-to-repo location. + * + * @since 1.20.13 + */ +public interface ArtifactIndex extends Closeable { + + /** + * Index (upsert) an artifact document. + * If a document with the same repoName+artifactPath exists, it is replaced. + * + * @param doc Artifact document to index + * @return Future completing when indexed + */ + CompletableFuture index(ArtifactDocument doc); + + /** + * Remove an artifact from the index. + * + * @param repoName Repository name + * @param artifactPath Artifact path + * @return Future completing when removed + */ + CompletableFuture remove(String repoName, String artifactPath); + + /** + * Full-text search across all indexed artifacts. + * + * @param query Search query string + * @param maxResults Maximum results to return + * @param offset Starting offset for pagination + * @return Search result with matching documents + */ + CompletableFuture search(String query, int maxResults, int offset); + + /** + * Locate which repositories contain a given artifact path. + * Uses path_prefix matching — slower, used as fallback. + * + * @param artifactPath Artifact path to locate + * @return List of repository names containing this artifact + */ + CompletableFuture> locate(String artifactPath); + + /** + * Locate which repositories contain an artifact by its indexed name. + * Uses the {@code name} column with B-tree index — O(log n), fast. + * This is the primary operation for group lookup when the adapter type + * is known and the name can be parsed from the URL. + * + * @param artifactName Artifact name as stored in the DB (adapter-specific format) + * @return List of repository names containing this artifact + */ + default CompletableFuture> locateByName(final String artifactName) { + return locate(artifactName); + } + + /** + * Whether the index has completed its initial warmup scan. + * @return true if warmup is complete and the index can be trusted + */ + default boolean isWarmedUp() { + return false; + } + + /** + * Mark the index as warmed up after initial scan completes. + */ + default void setWarmedUp() { + // no-op by default + } + + /** + * Get index statistics. + * @return map of stat name to value + */ + default CompletableFuture> getStats() { + return CompletableFuture.completedFuture(Map.of()); + } + + /** + * Index a batch of documents efficiently (single commit). + * Default implementation falls back to individual index() calls. + * + * @param docs Collection of documents to index + * @return Future completing when batch is indexed + */ + default CompletableFuture indexBatch(final java.util.Collection docs) { + CompletableFuture result = CompletableFuture.completedFuture(null); + for (final ArtifactDocument doc : docs) { + result = result.thenCompose(v -> index(doc)); + } + return result; + } + + /** + * No-op implementation that performs no indexing or searching. + */ + ArtifactIndex NOP = new ArtifactIndex() { + @Override + public CompletableFuture index(final ArtifactDocument doc) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture remove(final String rn, final String ap) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture search( + final String q, final int max, final int off + ) { + return CompletableFuture.completedFuture(SearchResult.EMPTY); + } + + @Override + public CompletableFuture> locate(final String path) { + return CompletableFuture.completedFuture(List.of()); + } + + @Override + public void close() { + // nop + } + }; +} diff --git a/artipie-core/src/main/java/com/artipie/index/SearchResult.java b/artipie-core/src/main/java/com/artipie/index/SearchResult.java new file mode 100644 index 000000000..e5acf32d1 --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/index/SearchResult.java @@ -0,0 +1,46 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.index; + +import java.util.List; + +/** + * Search result from the artifact index. + * + * @param documents Matching artifact documents + * @param totalHits Total number of matching documents + * @param offset Starting offset used for this result + * @param lastScoreDoc Opaque cursor for searchAfter pagination; + * null when there are no more pages or when using offset-based pagination. + * Callers should treat this as an opaque token for cursor-based paging. + * @since 1.20.13 + */ +public record SearchResult( + List documents, + long totalHits, + int offset, + Object lastScoreDoc +) { + + /** + * Backward-compatible constructor without cursor support. + * + * @param documents Matching artifact documents + * @param totalHits Total number of matching documents + * @param offset Starting offset used for this result + */ + public SearchResult( + final List documents, + final long totalHits, + final int offset + ) { + this(documents, totalHits, offset, null); + } + + /** + * Empty search result. + */ + public static final SearchResult EMPTY = new SearchResult(List.of(), 0, 0, null); +} diff --git a/artipie-core/src/main/java/com/artipie/layout/GradleLayout.java b/artipie-core/src/main/java/com/artipie/layout/GradleLayout.java deleted file mode 100644 index f4379e1fe..000000000 --- a/artipie-core/src/main/java/com/artipie/layout/GradleLayout.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com - * https://github.com/artipie/artipie/blob/master/LICENSE.txt - */ -package com.artipie.layout; - -import com.artipie.asto.Key; - -/** - * Gradle repository layout. - * Structure: {@code ////artifacts} - * where groupId x.y.z becomes folder structure x/y/z - * Folder per version is created. - * - * @since 1.0 - */ -public final class GradleLayout implements StorageLayout { - - /** - * Metadata key for groupId. - */ - public static final String GROUP_ID = "groupId"; - - /** - * Metadata key for artifactId. - */ - public static final String ARTIFACT_ID = "artifactId"; - - @Override - public Key artifactPath(final ArtifactInfo artifact) { - final String groupId = artifact.metadata(GROUP_ID); - final String artifactId = artifact.metadata(ARTIFACT_ID); - - if (groupId == null || artifactId == null) { - throw new IllegalArgumentException( - "Gradle layout requires 'groupId' and 'artifactId' metadata" - ); - } - - // Convert groupId dots to slashes (e.g., com.example -> com/example) - final String groupPath = groupId.replace('.', '/'); - - return new Key.From( - artifact.repository(), - groupPath, - artifactId, - artifact.version() - ); - } - - @Override - public Key metadataPath(final ArtifactInfo artifact) { - final String groupId = artifact.metadata(GROUP_ID); - final String artifactId = artifact.metadata(ARTIFACT_ID); - - if (groupId == null || artifactId == null) { - throw new IllegalArgumentException( - "Gradle layout requires 'groupId' and 'artifactId' metadata" - ); - } - - final String groupPath = groupId.replace('.', '/'); - - return new Key.From( - artifact.repository(), - groupPath, - artifactId, - artifact.version() - ); - } -} diff --git a/artipie-core/src/main/java/com/artipie/layout/LayoutFactory.java b/artipie-core/src/main/java/com/artipie/layout/LayoutFactory.java index 181ae4ac1..7f82fae9d 100644 --- a/artipie-core/src/main/java/com/artipie/layout/LayoutFactory.java +++ b/artipie-core/src/main/java/com/artipie/layout/LayoutFactory.java @@ -78,7 +78,7 @@ public static StorageLayout forType(final RepositoryType type) { layout = new NpmLayout(); break; case GRADLE: - layout = new GradleLayout(); + layout = new MavenLayout(); break; case COMPOSER: layout = new ComposerLayout(); diff --git a/artipie-core/src/main/java/com/artipie/scheduling/ArtifactEvent.java b/artipie-core/src/main/java/com/artipie/scheduling/ArtifactEvent.java index 133b602c2..9baf31c22 100644 --- a/artipie-core/src/main/java/com/artipie/scheduling/ArtifactEvent.java +++ b/artipie-core/src/main/java/com/artipie/scheduling/ArtifactEvent.java @@ -62,6 +62,12 @@ public final class ArtifactEvent { */ private final Optional release; + /** + * Path prefix for index-based group lookups (e.g., "com/google/guava/guava/32.1.3-jre"). + * Nullable — only set by proxy package processors. + */ + private final String pathPrefix; + /** * Ctor for the event to remove all artifact versions. * @param repoType Repository type @@ -69,7 +75,7 @@ public final class ArtifactEvent { * @param artifactName Artifact name */ public ArtifactEvent(String repoType, String repoName, String artifactName) { - this(repoType, repoName, ArtifactEvent.DEF_OWNER, artifactName, "", 0L, 0L, Optional.empty(), Type.DELETE_ALL); + this(repoType, repoName, ArtifactEvent.DEF_OWNER, artifactName, "", 0L, 0L, Optional.empty(), null, Type.DELETE_ALL); } /** @@ -81,7 +87,7 @@ public ArtifactEvent(String repoType, String repoName, String artifactName) { */ public ArtifactEvent(String repoType, String repoName, String artifactName, String version) { - this(repoType, repoName, ArtifactEvent.DEF_OWNER, artifactName, version, 0L, 0L, Optional.empty(), Type.DELETE_VERSION); + this(repoType, repoName, ArtifactEvent.DEF_OWNER, artifactName, version, 0L, 0L, Optional.empty(), null, Type.DELETE_VERSION); } /** @@ -92,11 +98,14 @@ public ArtifactEvent(String repoType, String repoName, * @param version Artifact version * @param size Artifact size * @param created Artifact created date + * @param release Remote release date + * @param pathPrefix Path prefix for index lookups (nullable) * @param etype Event type */ private ArtifactEvent(String repoType, String repoName, String owner, String artifactName, String version, long size, - long created, Optional release, Type etype) { + long created, Optional release, String pathPrefix, + Type etype) { this.repoType = repoType; this.repoName = repoName; this.owner = owner; @@ -105,6 +114,7 @@ private ArtifactEvent(String repoType, String repoName, String owner, this.size = size; this.created = created; this.release = release == null ? Optional.empty() : release; + this.pathPrefix = pathPrefix; this.eventType = etype; } @@ -121,7 +131,7 @@ private ArtifactEvent(String repoType, String repoName, String owner, public ArtifactEvent(final String repoType, final String repoName, final String owner, final String artifactName, final String version, final long size, final long created) { - this(repoType, repoName, owner, artifactName, version, size, created, Optional.empty(), Type.INSERT); + this(repoType, repoName, owner, artifactName, version, size, created, Optional.empty(), null, Type.INSERT); } /** @@ -130,7 +140,7 @@ public ArtifactEvent(final String repoType, final String repoName, final String public ArtifactEvent(final String repoType, final String repoName, final String owner, final String artifactName, final String version, final long size, final long created, final Type etype) { - this(repoType, repoName, owner, artifactName, version, size, created, Optional.empty(), etype); + this(repoType, repoName, owner, artifactName, version, size, created, Optional.empty(), null, etype); } /** @@ -147,7 +157,25 @@ public ArtifactEvent(final String repoType, final String repoName, final String public ArtifactEvent(final String repoType, final String repoName, final String owner, final String artifactName, final String version, final long size, final long created, final Long release) { - this(repoType, repoName, owner, artifactName, version, size, created, Optional.ofNullable(release), Type.INSERT); + this(repoType, repoName, owner, artifactName, version, size, created, Optional.ofNullable(release), null, Type.INSERT); + } + + /** + * Ctor to insert artifact data with explicit created and release timestamps and path prefix. + * @param repoType Repository type + * @param repoName Repository name + * @param owner Owner username + * @param artifactName Artifact name + * @param version Artifact version + * @param size Artifact size + * @param created Artifact created (uploaded) date + * @param release Remote release date (nullable) + * @param pathPrefix Path prefix for index lookups (nullable) + */ + public ArtifactEvent(final String repoType, final String repoName, final String owner, + final String artifactName, final String version, final long size, + final long created, final Long release, final String pathPrefix) { + this(repoType, repoName, owner, artifactName, version, size, created, Optional.ofNullable(release), pathPrefix, Type.INSERT); } /** @@ -162,7 +190,7 @@ public ArtifactEvent(final String repoType, final String repoName, final String public ArtifactEvent(final String repoType, final String repoName, final String owner, final String artifactName, final String version, final long size) { this(repoType, repoName, owner, artifactName, version, size, - System.currentTimeMillis(), Optional.empty(), Type.INSERT); + System.currentTimeMillis(), Optional.empty(), null, Type.INSERT); } /** @@ -229,6 +257,14 @@ public String owner() { return this.owner; } + /** + * Path prefix for index-based group lookups. + * @return Path prefix or null if not set + */ + public String pathPrefix() { + return this.pathPrefix; + } + /** * Event type. * @return The type of event diff --git a/artipie-core/src/main/java/com/artipie/scheduling/EventsProcessor.java b/artipie-core/src/main/java/com/artipie/scheduling/EventsProcessor.java index a6158c5ab..2465109f4 100644 --- a/artipie-core/src/main/java/com/artipie/scheduling/EventsProcessor.java +++ b/artipie-core/src/main/java/com/artipie/scheduling/EventsProcessor.java @@ -16,9 +16,18 @@ * set by {@link org.quartz} mechanism via setters. Note, that job instance is created by * {@link org.quartz} on every execution, but job data is not. *

- * In the case of {@link EventProcessingError} processor tries to process the event three times, - * if on the third time processing failed, job is shut down and event is not returned to queue. + * In the case of {@link EventProcessingError} processor retries each individual event up to three + * times. If all attempts fail, the event is dropped (logged) and processing continues with the + * next event. The job is never stopped due to individual event failures. *

+ * Supports two data-binding modes: + *

    + *
  • Direct (RAM mode): Queue and Consumer are set directly via + * {@link #setElements(Queue)} and {@link #setAction(Consumer)}.
    • + *
  • Registry (JDBC mode): Registry keys are set via + * {@link #setElements_key(String)} and {@link #setAction_key(String)}, + * and actual objects are looked up from {@link JobDataRegistry}.
    • + *
* Read more. * @param Elements type to process * @since 1.3 @@ -43,32 +52,40 @@ public final class EventsProcessor extends QuartzJob { @Override @SuppressWarnings("PMD.CognitiveComplexity") public void execute(final JobExecutionContext context) { + this.resolveFromRegistry(context); if (this.action == null || this.elements == null) { super.stopJob(context); } else { int cnt = 0; - int error = 0; while (!this.elements.isEmpty()) { final T item = this.elements.poll(); if (item != null) { - try { - cnt = cnt + 1; - this.action.accept(item); - } catch (final EventProcessingError ex) { + boolean processed = false; + for (int attempt = 0; attempt < EventsProcessor.MAX_RETRY; attempt++) { + try { + this.action.accept(item); + cnt = cnt + 1; + processed = true; + break; + } catch (final EventProcessingError ex) { + EcsLogger.error("com.artipie.scheduling") + .message("Event processing failed (attempt " + + (attempt + 1) + "/" + MAX_RETRY + ")") + .eventCategory("scheduling") + .eventAction("event_process") + .eventOutcome("failure") + .error(ex) + .log(); + } + } + if (!processed) { EcsLogger.error("com.artipie.scheduling") - .message("Event processing failed (retry " + error + "/" + MAX_RETRY + ")") + .message("Dropping event after " + MAX_RETRY + + " failed attempts") .eventCategory("scheduling") - .eventAction("event_process") + .eventAction("event_drop") .eventOutcome("failure") - .error(ex) .log(); - if (error > EventsProcessor.MAX_RETRY) { - this.stopJob(context); - break; - } - error = error + 1; - cnt = cnt - 1; - this.elements.add(item); } } } @@ -83,7 +100,7 @@ public void execute(final JobExecutionContext context) { } /** - * Set elements queue from job context. + * Set elements queue from job context (RAM mode). * @param queue Queue with elements to process */ public void setElements(final Queue queue) { @@ -91,11 +108,44 @@ public void setElements(final Queue queue) { } /** - * Set elements consumer from job context. + * Set elements consumer from job context (RAM mode). * @param consumer Action to consume the element */ public void setAction(final Consumer consumer) { this.action = consumer; } + /** + * Set registry key for elements queue (JDBC mode). + * @param key Registry key to look up the queue from {@link JobDataRegistry} + */ + @SuppressWarnings("PMD.MethodNamingConventions") + public void setElements_key(final String key) { + this.elements = JobDataRegistry.lookup(key); + } + + /** + * Set registry key for action consumer (JDBC mode). + * @param key Registry key to look up the consumer from {@link JobDataRegistry} + */ + @SuppressWarnings("PMD.MethodNamingConventions") + public void setAction_key(final String key) { + this.action = JobDataRegistry.lookup(key); + } + + /** + * Resolve elements and action from the job data registry if registry keys + * are present in the context and the fields are not yet set. + * @param context Job execution context + */ + private void resolveFromRegistry(final JobExecutionContext context) { + final org.quartz.JobDataMap data = context.getMergedJobDataMap(); + if (this.elements == null && data.containsKey("elements_key")) { + this.elements = JobDataRegistry.lookup(data.getString("elements_key")); + } + if (this.action == null && data.containsKey("action_key")) { + this.action = JobDataRegistry.lookup(data.getString("action_key")); + } + } + } diff --git a/artipie-core/src/main/java/com/artipie/scheduling/JobDataRegistry.java b/artipie-core/src/main/java/com/artipie/scheduling/JobDataRegistry.java new file mode 100644 index 000000000..c1d642a9d --- /dev/null +++ b/artipie-core/src/main/java/com/artipie/scheduling/JobDataRegistry.java @@ -0,0 +1,69 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.scheduling; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Static registry for non-serializable Quartz job data. + *

+ * When Quartz uses JDBC job store (clustering mode), all {@code JobDataMap} + * entries must be serializable because they are persisted in the database. + * However, runtime objects like {@link java.util.Queue} and + * {@link java.util.function.Consumer} cannot be serialized. + *

+ * This registry allows jobs to store non-serializable data by key in JVM + * memory and place only the key (a {@code String}) in the {@code JobDataMap}. + * The job retrieves the actual object from the registry at execution time. + *

+ * In a clustered setup, each node maintains its own registry. Since Quartz + * ensures a given trigger fires on only one node at a time, the node that + * scheduled the job always has the data in its registry. + * + * @since 1.20.13 + */ +public final class JobDataRegistry { + + /** + * In-memory store for non-serializable job data. + */ + private static final Map DATA = new ConcurrentHashMap<>(); + + /** + * Private ctor. + */ + private JobDataRegistry() { + // Utility class + } + + /** + * Register a non-serializable value by key. + * @param key Unique key for the data + * @param value Runtime object (Queue, Consumer, etc.) + */ + public static void register(final String key, final Object value) { + DATA.put(key, value); + } + + /** + * Look up a previously registered value. + * @param key Registry key + * @param Expected type + * @return The registered object, or null if not found + */ + @SuppressWarnings("unchecked") + public static T lookup(final String key) { + return (T) DATA.get(key); + } + + /** + * Remove a registered value. + * @param key Registry key to remove + */ + public static void remove(final String key) { + DATA.remove(key); + } +} diff --git a/artipie-core/src/main/java/com/artipie/scheduling/QuartzJob.java b/artipie-core/src/main/java/com/artipie/scheduling/QuartzJob.java index 97908d959..3f876eeed 100644 --- a/artipie-core/src/main/java/com/artipie/scheduling/QuartzJob.java +++ b/artipie-core/src/main/java/com/artipie/scheduling/QuartzJob.java @@ -10,7 +10,6 @@ import org.quartz.JobExecutionContext; import org.quartz.JobKey; import org.quartz.SchedulerException; -import org.quartz.impl.StdSchedulerFactory; /** * Super class for classes, which implement {@link Job} interface. @@ -21,6 +20,11 @@ public abstract class QuartzJob implements Job { /** * Stop the job and log error. + * Uses {@code context.getScheduler()} to get the correct scheduler + * instance (important for JDBC/clustered mode where + * {@code new StdSchedulerFactory().getScheduler()} would return + * a different default scheduler, making deleteJob a no-op on the + * real JDBC store). * @param context Job context */ protected void stopJob(final JobExecutionContext context) { @@ -33,7 +37,7 @@ protected void stopJob(final JobExecutionContext context) { .eventOutcome("failure") .field("process.name", key.toString()) .log(); - new StdSchedulerFactory().getScheduler().deleteJob(key); + context.getScheduler().deleteJob(key); EcsLogger.error("com.artipie.scheduling") .message("Job stopped") .eventCategory("scheduling") diff --git a/artipie-core/src/main/java/com/artipie/security/perms/PermissionConfig.java b/artipie-core/src/main/java/com/artipie/security/perms/PermissionConfig.java index 551dbe0ff..e6b110b9a 100644 --- a/artipie-core/src/main/java/com/artipie/security/perms/PermissionConfig.java +++ b/artipie-core/src/main/java/com/artipie/security/perms/PermissionConfig.java @@ -13,6 +13,10 @@ import java.util.Collection; import java.util.Set; import java.util.stream.Collectors; +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.json.JsonString; +import javax.json.JsonValue; /** * Permission configuration. @@ -196,4 +200,125 @@ public boolean isEmpty() { return this.seq == null || this.seq.isEmpty(); } } + + /** + * Permission config from JSON object. Used by database-backed policy + * to bridge JSON permission data into the PermissionConfig interface. + * @since 1.21 + */ + final class FromJsonObject implements PermissionConfig { + + /** + * JSON object to read permission from. + */ + private final JsonObject json; + + /** + * Ctor. + * @param json JSON object to read permission from + */ + public FromJsonObject(final JsonObject json) { + this.json = json; + } + + @Override + public String string(final String key) { + if (this.json.containsKey(key)) { + return this.json.getString(key); + } + return null; + } + + @Override + public Set sequence(final String key) { + final JsonArray arr = this.json.getJsonArray(key); + return arr.stream() + .map(v -> ((JsonString) v).getString()) + .collect(Collectors.toSet()); + } + + @Override + public Set keys() { + return this.json.keySet(); + } + + @Override + public PermissionConfig config(final String key) { + final JsonValue val = this.json.get(key); + if (val instanceof JsonObject) { + return new FromJsonObject((JsonObject) val); + } else if (val instanceof JsonArray) { + return new FromJsonArray((JsonArray) val); + } + throw new IllegalArgumentException( + String.format("JSON sub-config not found for key: %s", key) + ); + } + + @Override + public boolean isEmpty() { + return this.json == null || this.json.isEmpty(); + } + } + + /** + * Permission config from JSON array. Used by database-backed policy + * to bridge JSON permission data into the PermissionConfig interface. + * In this implementation, method {@link FromJsonArray#keys()} returns + * the array elements as a set of strings. + * @since 1.21 + */ + final class FromJsonArray implements PermissionConfig { + + /** + * JSON array. + */ + private final JsonArray arr; + + /** + * Ctor. + * @param arr JSON array + */ + public FromJsonArray(final JsonArray arr) { + this.arr = arr; + } + + @Override + public Set keys() { + return this.arr.stream() + .map(v -> ((JsonString) v).getString()) + .collect(Collectors.toSet()); + } + + @Override + public String string(final String index) { + return this.arr.getString(Integer.parseInt(index)); + } + + @Override + public Collection sequence(final String index) { + return this.arr.getJsonArray(Integer.parseInt(index)).stream() + .map(v -> ((JsonString) v).getString()) + .collect(Collectors.toSet()); + } + + @Override + public PermissionConfig config(final String index) { + final int ind = Integer.parseInt(index); + final JsonValue val = this.arr.get(ind); + if (val instanceof JsonObject) { + return new FromJsonObject((JsonObject) val); + } else if (val instanceof JsonArray) { + return new FromJsonArray((JsonArray) val); + } + throw new IllegalArgumentException( + String.format("Sub config by index %s not found", index) + ); + } + + @Override + public boolean isEmpty() { + return this.arr == null || this.arr.isEmpty(); + } + } } diff --git a/artipie-core/src/test/java/com/artipie/cache/ValkeyConnectionTest.java b/artipie-core/src/test/java/com/artipie/cache/ValkeyConnectionTest.java new file mode 100644 index 000000000..06bf8a986 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/cache/ValkeyConnectionTest.java @@ -0,0 +1,113 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cache; + +import java.time.Duration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +/** + * Tests for {@link ValkeyConnection}. + * Tests that do not require a running Valkey/Redis server verify + * pool configuration and lifecycle. Tests that require a server + * are gated by the VALKEY_HOST environment variable. + */ +final class ValkeyConnectionTest { + + @Test + void failsToConnectToNonExistentHost() { + Assertions.assertThrows( + Exception.class, + () -> new ValkeyConnection( + "192.0.2.1", + 9999, + Duration.ofMillis(200) + ), + "Should fail when Valkey/Redis is not available" + ); + } + + @Test + void failsToConnectWithCustomPoolSize() { + Assertions.assertThrows( + Exception.class, + () -> new ValkeyConnection( + "192.0.2.1", + 9999, + Duration.ofMillis(200), + 4 + ), + "Should fail with custom pool size when Valkey/Redis is not available" + ); + } + + @Test + @EnabledIfEnvironmentVariable(named = "VALKEY_HOST", matches = ".+") + void connectsAndClosesWithDefaultPoolSize() throws Exception { + final String host = System.getenv("VALKEY_HOST"); + final int port = Integer.parseInt( + System.getenv().getOrDefault("VALKEY_PORT", "6379") + ); + try (ValkeyConnection conn = new ValkeyConnection( + host, port, Duration.ofSeconds(2) + )) { + Assertions.assertEquals( + 8, conn.poolSize(), + "Default pool size should be 8" + ); + Assertions.assertNotNull( + conn.async(), + "async() should return non-null commands" + ); + } + } + + @Test + @EnabledIfEnvironmentVariable(named = "VALKEY_HOST", matches = ".+") + void connectsWithCustomPoolSizeAndPings() throws Exception { + final String host = System.getenv("VALKEY_HOST"); + final int port = Integer.parseInt( + System.getenv().getOrDefault("VALKEY_PORT", "6379") + ); + try (ValkeyConnection conn = new ValkeyConnection( + host, port, Duration.ofSeconds(2), 4 + )) { + Assertions.assertEquals( + 4, conn.poolSize(), + "Custom pool size should be 4" + ); + Assertions.assertTrue( + conn.pingAsync().get(), + "pingAsync should return true when connected" + ); + } + } + + @Test + @EnabledIfEnvironmentVariable(named = "VALKEY_HOST", matches = ".+") + void roundRobinsAcrossConnections() throws Exception { + final String host = System.getenv("VALKEY_HOST"); + final int port = Integer.parseInt( + System.getenv().getOrDefault("VALKEY_PORT", "6379") + ); + try (ValkeyConnection conn = new ValkeyConnection( + host, port, Duration.ofSeconds(2), 3 + )) { + final var first = conn.async(); + final var second = conn.async(); + final var third = conn.async(); + final var fourth = conn.async(); + Assertions.assertSame( + first, fourth, + "Fourth call should return same commands as first (round-robin)" + ); + Assertions.assertNotSame( + first, second, + "Consecutive calls should return different commands objects" + ); + } + } +} diff --git a/artipie-core/src/test/java/com/artipie/cluster/ClusterEventBusTest.java b/artipie-core/src/test/java/com/artipie/cluster/ClusterEventBusTest.java new file mode 100644 index 000000000..eea47980e --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/cluster/ClusterEventBusTest.java @@ -0,0 +1,152 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cluster; + +import com.artipie.cache.ValkeyConnection; +import java.time.Duration; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +/** + * Tests for {@link ClusterEventBus}. + *

+ * Tests that do not require a running Valkey/Redis server verify + * construction expectations and channel naming conventions. + * Integration tests that require a real server are gated by the + * VALKEY_HOST environment variable. + * + * @since 1.20.13 + */ +final class ClusterEventBusTest { + + @Test + void channelPrefixIsConsistent() { + Assertions.assertEquals( + "artipie:events:", + ClusterEventBus.CHANNEL_PREFIX, + "Channel prefix must follow the artipie:events: convention" + ); + } + + @Test + @EnabledIfEnvironmentVariable(named = "VALKEY_HOST", matches = ".+") + void createsAndClosesEventBus() throws Exception { + final String host = System.getenv("VALKEY_HOST"); + final int port = Integer.parseInt( + System.getenv().getOrDefault("VALKEY_PORT", "6379") + ); + try (ValkeyConnection conn = new ValkeyConnection( + host, port, Duration.ofSeconds(2) + )) { + try (ClusterEventBus bus = new ClusterEventBus(conn)) { + Assertions.assertNotNull( + bus.instanceId(), + "Instance ID should be non-null" + ); + Assertions.assertFalse( + bus.instanceId().isEmpty(), + "Instance ID should not be empty" + ); + Assertions.assertEquals( + 0, bus.topicCount(), + "No topics should be subscribed initially" + ); + } + } + } + + @Test + @EnabledIfEnvironmentVariable(named = "VALKEY_HOST", matches = ".+") + void subscribesAndCountsTopics() throws Exception { + final String host = System.getenv("VALKEY_HOST"); + final int port = Integer.parseInt( + System.getenv().getOrDefault("VALKEY_PORT", "6379") + ); + try (ValkeyConnection conn = new ValkeyConnection( + host, port, Duration.ofSeconds(2) + )) { + try (ClusterEventBus bus = new ClusterEventBus(conn)) { + bus.subscribe("test.topic1", payload -> { }); + bus.subscribe("test.topic2", payload -> { }); + Assertions.assertEquals( + 2, bus.topicCount(), + "Should track two subscribed topics" + ); + bus.subscribe("test.topic1", payload -> { }); + Assertions.assertEquals( + 2, bus.topicCount(), + "Adding second handler to same topic should not increase count" + ); + } + } + } + + @Test + @EnabledIfEnvironmentVariable(named = "VALKEY_HOST", matches = ".+") + void publishAndReceiveAcrossInstances() throws Exception { + final String host = System.getenv("VALKEY_HOST"); + final int port = Integer.parseInt( + System.getenv().getOrDefault("VALKEY_PORT", "6379") + ); + try (ValkeyConnection conn = new ValkeyConnection( + host, port, Duration.ofSeconds(2) + )) { + final CopyOnWriteArrayList received = + new CopyOnWriteArrayList<>(); + final CountDownLatch latch = new CountDownLatch(1); + try (ClusterEventBus bus1 = new ClusterEventBus(conn); + ClusterEventBus bus2 = new ClusterEventBus(conn)) { + bus2.subscribe("cross.test", payload -> { + received.add(payload); + latch.countDown(); + }); + Thread.sleep(200); + bus1.publish("cross.test", "{\"action\":\"test\"}"); + final boolean arrived = latch.await(5, TimeUnit.SECONDS); + Assertions.assertTrue( + arrived, + "Message should arrive at second bus instance" + ); + Assertions.assertEquals( + 1, received.size(), + "Exactly one message should be received" + ); + Assertions.assertEquals( + "{\"action\":\"test\"}", received.get(0), + "Payload should match what was published" + ); + } + } + } + + @Test + @EnabledIfEnvironmentVariable(named = "VALKEY_HOST", matches = ".+") + void selfPublishedMessagesAreIgnored() throws Exception { + final String host = System.getenv("VALKEY_HOST"); + final int port = Integer.parseInt( + System.getenv().getOrDefault("VALKEY_PORT", "6379") + ); + try (ValkeyConnection conn = new ValkeyConnection( + host, port, Duration.ofSeconds(2) + )) { + final CopyOnWriteArrayList received = + new CopyOnWriteArrayList<>(); + try (ClusterEventBus bus = new ClusterEventBus(conn)) { + bus.subscribe("self.test", received::add); + Thread.sleep(200); + bus.publish("self.test", "should-not-arrive"); + Thread.sleep(1000); + Assertions.assertTrue( + received.isEmpty(), + "Self-published messages should be filtered out" + ); + } + } + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/cache/BaseCachedProxySliceContentEncodingTest.java b/artipie-core/src/test/java/com/artipie/http/cache/BaseCachedProxySliceContentEncodingTest.java new file mode 100644 index 000000000..31eb45539 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/cache/BaseCachedProxySliceContentEncodingTest.java @@ -0,0 +1,379 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import com.artipie.asto.Content; +import com.artipie.asto.Key; +import com.artipie.asto.cache.Cache; +import com.artipie.asto.memory.InMemoryStorage; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.RsStatus; +import com.artipie.http.headers.Header; +import com.artipie.http.rq.RequestLine; +import com.artipie.http.rq.RqMethod; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.StreamSupport; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * Regression tests: {@link BaseCachedProxySlice} must not propagate {@code Content-Encoding} + * headers from upstream to clients after Jetty auto-decodes compressed bodies. + * + *

Root cause: Jetty's {@code GZIPContentDecoder} decodes gzip response bodies but + * leaves {@code Content-Encoding: gzip} in the upstream response headers. When + * {@code BaseCachedProxySlice} passes those headers directly to the caller (via + * {@code fetchDirect}, {@code handleRootPath} or from the metadata cache hit path), + * clients receive plain bytes paired with a gzip header and fail with + * {@code Z_DATA_ERROR: zlib: incorrect header check}. + * + * @since 1.20.13 + */ +final class BaseCachedProxySliceContentEncodingTest { + + // ===== Unit tests for the stripContentEncoding() helper ===== + + @ParameterizedTest + @ValueSource(strings = {"gzip", "GZIP", "deflate", "br", "x-gzip"}) + void stripContentEncodingRemovesKnownEncodings(final String encoding) { + final Headers headers = new Headers(List.of( + new Header("Content-Encoding", encoding), + new Header("Content-Type", "application/octet-stream"), + new Header("Content-Length", "1234") + )); + final Headers result = BaseCachedProxySlice.stripContentEncoding(headers); + assertFalse( + hasHeader(result, "Content-Encoding"), + "Content-Encoding must be stripped for encoding: " + encoding + ); + assertFalse( + hasHeader(result, "Content-Length"), + "Content-Length must be stripped when encoding was decoded" + ); + assertEquals( + "application/octet-stream", + firstHeader(result, "Content-Type"), + "Content-Type must be preserved" + ); + } + + @Test + void stripContentEncodingIsNoopWhenAbsent() { + final Headers headers = new Headers(List.of( + new Header("Content-Type", "text/plain"), + new Header("Content-Length", "42"), + new Header("ETag", "\"deadbeef\"") + )); + final Headers result = BaseCachedProxySlice.stripContentEncoding(headers); + assertEquals( + "text/plain", + firstHeader(result, "Content-Type") + ); + assertEquals( + "42", + firstHeader(result, "Content-Length"), + "Content-Length preserved when no transfer encoding" + ); + assertEquals( + "\"deadbeef\"", + firstHeader(result, "ETag") + ); + } + + @Test + void stripContentEncodingIsNoopForIdentityEncoding() { + final Headers headers = new Headers(List.of( + new Header("Content-Encoding", "identity"), + new Header("Content-Type", "text/plain"), + new Header("Content-Length", "10") + )); + final Headers result = BaseCachedProxySlice.stripContentEncoding(headers); + // "identity" is not decoded by Jetty, so nothing should be stripped + assertEquals( + "identity", + firstHeader(result, "Content-Encoding"), + "identity encoding must not be stripped" + ); + assertEquals( + "10", + firstHeader(result, "Content-Length"), + "Content-Length preserved for identity encoding" + ); + } + + // ===== Integration tests via a minimal concrete subclass ===== + + @Test + void fetchDirectDoesNotPropagateContentEncodingGzip() { + final byte[] decodedBody = "plain bytes after gzip decode".getBytes(); + final MinimalProxySlice slice = new MinimalProxySlice( + // Upstream returns decoded bytes but STILL has Content-Encoding: gzip in headers + (line, headers, body) -> CompletableFuture.completedFuture( + ResponseBuilder.ok() + .header("Content-Encoding", "gzip") + .header("Content-Type", "application/java-archive") + .header("Content-Length", "100") + .body(decodedBody) + .build() + ), + false // non-cacheable → always goes through fetchDirect + ); + final Response response = slice.response( + new RequestLine(RqMethod.GET, "/com/example/foo/1.0/foo-1.0.jar"), + Headers.EMPTY, + Content.EMPTY + ).join(); + assertEquals(RsStatus.OK, response.status()); + assertFalse( + hasHeader(response.headers(), "Content-Encoding"), + "fetchDirect must strip Content-Encoding: gzip" + ); + assertEquals( + "application/java-archive", + firstHeader(response.headers(), "Content-Type"), + "Content-Type must be preserved" + ); + // Note: Content-Length may be present with the correct (decoded) body size — + // we only require that the stale compressed Content-Length is gone. + // If Content-Length is present it must match the actual decoded body. + final String contentLength = firstHeader(response.headers(), "Content-Length"); + if (contentLength != null) { + assertEquals( + String.valueOf(decodedBody.length), + contentLength, + "If Content-Length is set it must reflect the decoded body size, not compressed size" + ); + } + } + + @Test + void handleRootPathDoesNotPropagateContentEncodingGzip() { + final byte[] decodedBody = "root response".getBytes(); + final MinimalProxySlice slice = new MinimalProxySlice( + (line, headers, body) -> CompletableFuture.completedFuture( + ResponseBuilder.ok() + .header("Content-Encoding", "gzip") + .header("Content-Type", "application/json") + .body(decodedBody) + .build() + ), + true + ); + // "/" triggers handleRootPath() + final Response response = slice.response( + new RequestLine(RqMethod.GET, "/"), + Headers.EMPTY, + Content.EMPTY + ).join(); + assertEquals(RsStatus.OK, response.status()); + assertFalse( + hasHeader(response.headers(), "Content-Encoding"), + "handleRootPath must strip Content-Encoding: gzip" + ); + } + + @Test + void cacheResponseDoesNotStoreContentEncodingGzip() { + final InMemoryStorage storage = new InMemoryStorage(); + final CachedArtifactMetadataStore store = new CachedArtifactMetadataStore(storage); + final byte[] decodedBody = "cached content".getBytes(); + + final MinimalProxySlice slice = new MinimalProxySlice( + (line, headers, body) -> CompletableFuture.completedFuture( + ResponseBuilder.ok() + .header("Content-Encoding", "gzip") + .header("Content-Type", "application/java-archive") + .header("ETag", "\"v1\"") + .header("Content-Length", "500") + .body(decodedBody) + .build() + ), + true, // cacheable + storage + ); + // First request: fetches and caches + slice.response( + new RequestLine(RqMethod.GET, "/com/example/lib/1.0/lib-1.0.jar"), + Headers.EMPTY, + Content.EMPTY + ).join(); + // Load the stored metadata and verify Content-Encoding was NOT persisted + final Key key = new Key.From("com/example/lib/1.0/lib-1.0.jar"); + final Optional meta = store.load(key).join(); + if (meta.isPresent()) { + assertFalse( + hasHeader(meta.get().headers(), "Content-Encoding"), + "Metadata store must NOT contain Content-Encoding: gzip" + ); + } + // Second request: serves from cache — must not have Content-Encoding: gzip + final Response cached = slice.response( + new RequestLine(RqMethod.GET, "/com/example/lib/1.0/lib-1.0.jar"), + Headers.EMPTY, + Content.EMPTY + ).join(); + assertEquals(RsStatus.OK, cached.status()); + assertFalse( + hasHeader(cached.headers(), "Content-Encoding"), + "Cache hit response must not have Content-Encoding: gzip" + ); + } + + @Test + void cacheHitPathStripsContentEncodingFromExistingMetadata() throws Exception { + // Simulate metadata that was stored BEFORE the fix (poisoned with Content-Encoding: gzip) + final InMemoryStorage storage = new InMemoryStorage(); + final CachedArtifactMetadataStore store = new CachedArtifactMetadataStore(storage); + final Key key = new Key.From("com/example/old/1.0/old-1.0.jar"); + final byte[] decodedBody = "old cached content".getBytes(); + // Manually save poisoned metadata (with Content-Encoding: gzip) + final Headers poisonedHeaders = new Headers(List.of( + new Header("Content-Encoding", "gzip"), + new Header("Content-Type", "application/java-archive"), + new Header("Content-Length", "999") + )); + store.save( + key, + poisonedHeaders, + new CachedArtifactMetadataStore.ComputedDigests(decodedBody.length, java.util.Map.of()) + ).join(); + // Save content to storage too + storage.save(key, new Content.From(decodedBody)).join(); + + final MinimalProxySlice slice = new MinimalProxySlice( + // Upstream should not be called for a cache hit + (line, headers, body) -> CompletableFuture.failedFuture( + new AssertionError("Upstream must not be called on cache hit") + ), + true, + storage, + (cacheKey, supplier, control) -> + CompletableFuture.completedFuture(Optional.of(new Content.From(decodedBody))) + ); + final Response response = slice.response( + new RequestLine(RqMethod.GET, "/com/example/old/1.0/old-1.0.jar"), + Headers.EMPTY, + Content.EMPTY + ).join(); + assertEquals(RsStatus.OK, response.status()); + assertFalse( + hasHeader(response.headers(), "Content-Encoding"), + "Cache hit must strip Content-Encoding: gzip from previously-poisoned metadata" + ); + } + + // ===== Helpers ===== + + private static boolean hasHeader(final Headers headers, final String name) { + return StreamSupport.stream(headers.spliterator(), false) + .anyMatch(h -> name.equalsIgnoreCase(h.getKey())); + } + + private static String firstHeader(final Headers headers, final String name) { + return StreamSupport.stream(headers.spliterator(), false) + .filter(h -> name.equalsIgnoreCase(h.getKey())) + .map(Header::getValue) + .findFirst() + .orElse(null); + } + + /** + * Minimal concrete subclass of {@link BaseCachedProxySlice} for tests. + * Delegates all abstract/hook methods to simple defaults. + */ + private static final class MinimalProxySlice extends BaseCachedProxySlice { + + /** + * Whether paths are cacheable. + */ + private final boolean cacheable; + + /** + * Ctor for non-storage-backed tests (fetchDirect / handleRootPath). + * @param upstream Upstream slice + * @param cacheable Whether isCacheable returns true + */ + MinimalProxySlice( + final com.artipie.http.Slice upstream, + final boolean cacheable + ) { + super( + upstream, + Cache.NOP, + "test-repo", + "test", + "http://upstream", + Optional.empty(), + Optional.empty(), + ProxyCacheConfig.defaults() + ); + this.cacheable = cacheable; + } + + /** + * Ctor for storage-backed tests (cacheResponse). + * @param upstream Upstream slice + * @param cacheable Whether isCacheable returns true + * @param storage Backing storage + */ + MinimalProxySlice( + final com.artipie.http.Slice upstream, + final boolean cacheable, + final com.artipie.asto.Storage storage + ) { + super( + upstream, + new com.artipie.asto.cache.FromStorageCache(storage), + "test-repo", + "test", + "http://upstream", + Optional.of(storage), + Optional.empty(), + ProxyCacheConfig.defaults() + ); + this.cacheable = cacheable; + } + + /** + * Ctor for cache-hit tests with a custom Cache implementation. + * @param upstream Upstream slice + * @param cacheable Whether isCacheable returns true + * @param storage Backing storage + * @param cache Custom cache (to inject pre-stored content) + */ + MinimalProxySlice( + final com.artipie.http.Slice upstream, + final boolean cacheable, + final com.artipie.asto.Storage storage, + final Cache cache + ) { + super( + upstream, + cache, + "test-repo", + "test", + "http://upstream", + Optional.of(storage), + Optional.empty(), + ProxyCacheConfig.defaults() + ); + this.cacheable = cacheable; + } + + @Override + protected boolean isCacheable(final String path) { + return this.cacheable; + } + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/cache/ConditionalRequestTest.java b/artipie-core/src/test/java/com/artipie/http/cache/ConditionalRequestTest.java new file mode 100644 index 000000000..b045c4fa9 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/cache/ConditionalRequestTest.java @@ -0,0 +1,134 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import com.artipie.http.Headers; +import com.artipie.http.headers.Header; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for {@link ConditionalRequest}. + */ +class ConditionalRequestTest { + + @Test + void buildsIfNoneMatchHeader() { + final Headers headers = ConditionalRequest.conditionalHeaders( + Optional.of("\"abc123\""), Optional.empty() + ); + assertThat( + headers.stream() + .filter(h -> "If-None-Match".equalsIgnoreCase(h.getKey())) + .findFirst() + .map(Header::getValue) + .orElse(""), + equalTo("\"abc123\"") + ); + } + + @Test + void buildsIfModifiedSinceHeader() { + final Headers headers = ConditionalRequest.conditionalHeaders( + Optional.empty(), Optional.of("Sat, 15 Feb 2025 12:00:00 GMT") + ); + assertThat( + headers.stream() + .filter(h -> "If-Modified-Since".equalsIgnoreCase(h.getKey())) + .findFirst() + .map(Header::getValue) + .orElse(""), + equalTo("Sat, 15 Feb 2025 12:00:00 GMT") + ); + } + + @Test + void buildsBothHeaders() { + final Headers headers = ConditionalRequest.conditionalHeaders( + Optional.of("\"etag-val\""), Optional.of("Mon, 01 Jan 2024 00:00:00 GMT") + ); + assertThat( + headers.stream() + .filter(h -> "If-None-Match".equalsIgnoreCase(h.getKey())) + .findFirst() + .isPresent(), + is(true) + ); + assertThat( + headers.stream() + .filter(h -> "If-Modified-Since".equalsIgnoreCase(h.getKey())) + .findFirst() + .isPresent(), + is(true) + ); + } + + @Test + void returnsEmptyHeadersWhenNoMetadata() { + final Headers headers = ConditionalRequest.conditionalHeaders( + Optional.empty(), Optional.empty() + ); + assertThat(headers, equalTo(Headers.EMPTY)); + } + + @Test + void extractsEtagFromHeaders() { + final Headers headers = new Headers(List.of( + new Header("Content-Type", "application/octet-stream"), + new Header("ETag", "\"xyz789\"") + )); + assertThat( + ConditionalRequest.extractEtag(headers), + equalTo(Optional.of("\"xyz789\"")) + ); + } + + @Test + void extractsEtagCaseInsensitive() { + final Headers headers = new Headers(List.of( + new Header("etag", "\"lowercase\"") + )); + assertThat( + ConditionalRequest.extractEtag(headers), + equalTo(Optional.of("\"lowercase\"")) + ); + } + + @Test + void returnsEmptyWhenNoEtag() { + final Headers headers = new Headers(List.of( + new Header("Content-Type", "text/plain") + )); + assertThat( + ConditionalRequest.extractEtag(headers), + equalTo(Optional.empty()) + ); + } + + @Test + void extractsLastModified() { + final Headers headers = new Headers(List.of( + new Header("Last-Modified", "Sat, 15 Feb 2025 12:00:00 GMT") + )); + assertThat( + ConditionalRequest.extractLastModified(headers), + equalTo(Optional.of("Sat, 15 Feb 2025 12:00:00 GMT")) + ); + } + + @Test + void returnsEmptyWhenNoLastModified() { + assertThat( + ConditionalRequest.extractLastModified(Headers.EMPTY), + equalTo(Optional.empty()) + ); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/cache/DedupStrategyTest.java b/artipie-core/src/test/java/com/artipie/http/cache/DedupStrategyTest.java new file mode 100644 index 000000000..9d4e39f54 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/cache/DedupStrategyTest.java @@ -0,0 +1,32 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.equalTo; + +/** + * Tests for {@link DedupStrategy}. + */ +class DedupStrategyTest { + + @Test + void hasThreeValues() { + assertThat( + DedupStrategy.values(), + arrayContaining(DedupStrategy.NONE, DedupStrategy.STORAGE, DedupStrategy.SIGNAL) + ); + } + + @Test + void valueOfWorks() { + assertThat(DedupStrategy.valueOf("SIGNAL"), equalTo(DedupStrategy.SIGNAL)); + assertThat(DedupStrategy.valueOf("NONE"), equalTo(DedupStrategy.NONE)); + assertThat(DedupStrategy.valueOf("STORAGE"), equalTo(DedupStrategy.STORAGE)); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/cache/DigestComputerTest.java b/artipie-core/src/test/java/com/artipie/http/cache/DigestComputerTest.java new file mode 100644 index 000000000..7c970c918 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/cache/DigestComputerTest.java @@ -0,0 +1,192 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link DigestComputer}. + */ +class DigestComputerTest { + + @Test + void computesSha256() { + final byte[] content = "hello".getBytes(StandardCharsets.UTF_8); + final Map digests = DigestComputer.compute( + content, Set.of(DigestComputer.SHA256) + ); + assertThat( + digests, + hasEntry( + DigestComputer.SHA256, + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ) + ); + } + + @Test + void computesSha1() { + final byte[] content = "hello".getBytes(StandardCharsets.UTF_8); + final Map digests = DigestComputer.compute( + content, Set.of(DigestComputer.SHA1) + ); + assertThat( + digests, + hasEntry( + DigestComputer.SHA1, + "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d" + ) + ); + } + + @Test + void computesMd5() { + final byte[] content = "hello".getBytes(StandardCharsets.UTF_8); + final Map digests = DigestComputer.compute( + content, Set.of(DigestComputer.MD5) + ); + assertThat( + digests, + hasEntry(DigestComputer.MD5, "5d41402abc4b2a76b9719d911017c592") + ); + } + + @Test + void computesMavenDigests() { + final byte[] content = "test content".getBytes(StandardCharsets.UTF_8); + final Map digests = DigestComputer.compute( + content, DigestComputer.MAVEN_DIGESTS + ); + assertThat("should have SHA-256", digests.containsKey(DigestComputer.SHA256), is(true)); + assertThat("should have SHA-1", digests.containsKey(DigestComputer.SHA1), is(true)); + assertThat("should have MD5", digests.containsKey(DigestComputer.MD5), is(true)); + assertThat("should have 3 entries", digests.size(), equalTo(3)); + } + + + @Test + void returnsEmptyForNullAlgorithms() { + final Map digests = DigestComputer.compute( + new byte[]{1, 2, 3}, null + ); + assertThat(digests, equalTo(Collections.emptyMap())); + } + + @Test + void returnsEmptyForEmptyAlgorithms() { + final Map digests = DigestComputer.compute( + new byte[]{1, 2, 3}, Collections.emptySet() + ); + assertThat(digests, equalTo(Collections.emptyMap())); + } + + @Test + void throwsForUnsupportedAlgorithm() { + assertThrows( + IllegalArgumentException.class, + () -> DigestComputer.compute( + new byte[]{1}, Set.of("UNSUPPORTED-ALGO") + ) + ); + } + + @Test + void throwsForNullContent() { + assertThrows( + NullPointerException.class, + () -> DigestComputer.compute(null, Set.of(DigestComputer.SHA256)) + ); + } + + @Test + void computesEmptyContent() { + final Map digests = DigestComputer.compute( + new byte[0], Set.of(DigestComputer.SHA256) + ); + assertThat( + digests, + hasEntry( + DigestComputer.SHA256, + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ) + ); + } + + @Test + void streamingDigestsMatchBatchDigests() { + final byte[] content = "streaming test data for digest verification" + .getBytes(StandardCharsets.UTF_8); + final Map batch = DigestComputer.compute( + content, DigestComputer.MAVEN_DIGESTS + ); + final Map digests = + DigestComputer.createDigests(DigestComputer.MAVEN_DIGESTS); + DigestComputer.updateDigests(digests, ByteBuffer.wrap(content)); + final Map streaming = DigestComputer.finalizeDigests(digests); + assertThat(streaming, equalTo(batch)); + } + + @Test + void streamingDigestsWithMultipleChunks() { + final byte[] full = "hello world streaming digest" + .getBytes(StandardCharsets.UTF_8); + final Map batch = DigestComputer.compute( + full, Set.of(DigestComputer.SHA256, DigestComputer.MD5) + ); + final Map digests = + DigestComputer.createDigests( + Set.of(DigestComputer.SHA256, DigestComputer.MD5) + ); + final int mid = full.length / 2; + DigestComputer.updateDigests( + digests, ByteBuffer.wrap(full, 0, mid) + ); + DigestComputer.updateDigests( + digests, ByteBuffer.wrap(full, mid, full.length - mid) + ); + final Map streaming = DigestComputer.finalizeDigests(digests); + assertThat(streaming, equalTo(batch)); + } + + @Test + void streamingDigestsWithEmptyContent() { + final Map batch = DigestComputer.compute( + new byte[0], Set.of(DigestComputer.SHA256) + ); + final Map digests = + DigestComputer.createDigests(Set.of(DigestComputer.SHA256)); + final Map streaming = DigestComputer.finalizeDigests(digests); + assertThat(streaming, equalTo(batch)); + } + + @Test + void createDigestsThrowsForUnsupported() { + assertThrows( + IllegalArgumentException.class, + () -> DigestComputer.createDigests(Set.of("BOGUS")) + ); + } + + @Test + void createDigestsReturnsEmptyForNull() { + assertThat( + DigestComputer.createDigests(null), + equalTo(Collections.emptyMap()) + ); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/cache/NegativeCacheRegistryTest.java b/artipie-core/src/test/java/com/artipie/http/cache/NegativeCacheRegistryTest.java new file mode 100644 index 000000000..8c63b7c15 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/cache/NegativeCacheRegistryTest.java @@ -0,0 +1,50 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for {@link NegativeCacheRegistry}. + */ +class NegativeCacheRegistryTest { + + @AfterEach + void tearDown() { + NegativeCacheRegistry.instance().clear(); + } + + @Test + void registersAndCountsCaches() { + final NegativeCacheRegistry reg = NegativeCacheRegistry.instance(); + final NegativeCache cache1 = new NegativeCache("maven", "central"); + final NegativeCache cache2 = new NegativeCache("npm", "proxy"); + reg.register("maven", "central", cache1); + reg.register("npm", "proxy", cache2); + assertThat(reg.size(), equalTo(2)); + } + + @Test + void unregistersCache() { + final NegativeCacheRegistry reg = NegativeCacheRegistry.instance(); + reg.register("maven", "central", new NegativeCache("maven", "central")); + reg.unregister("maven", "central"); + assertThat(reg.size(), equalTo(0)); + } + + @Test + void clearRemovesAll() { + final NegativeCacheRegistry reg = NegativeCacheRegistry.instance(); + reg.register("a", "b", new NegativeCache("a", "b")); + reg.register("c", "d", new NegativeCache("c", "d")); + reg.clear(); + assertThat(reg.size(), equalTo(0)); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/cache/RequestDeduplicatorTest.java b/artipie-core/src/test/java/com/artipie/http/cache/RequestDeduplicatorTest.java new file mode 100644 index 000000000..6b10f4eba --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/cache/RequestDeduplicatorTest.java @@ -0,0 +1,240 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import com.artipie.asto.Key; +import com.artipie.http.cache.RequestDeduplicator.FetchSignal; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for {@link RequestDeduplicator}. + */ +class RequestDeduplicatorTest { + + @Test + @Timeout(5) + void signalStrategyDeduplicatesConcurrentRequests() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + final Key key = new Key.From("test/artifact.jar"); + final AtomicInteger fetchCount = new AtomicInteger(0); + final CompletableFuture blocker = new CompletableFuture<>(); + // First request: starts the fetch, blocks until we complete manually + final CompletableFuture first = dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return blocker; + } + ); + // Second request for same key: should join the existing one + final CompletableFuture second = dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return CompletableFuture.completedFuture(FetchSignal.SUCCESS); + } + ); + assertThat("fetch should only run once", fetchCount.get(), equalTo(1)); + assertThat("first not done yet", first.isDone(), is(false)); + assertThat("second not done yet", second.isDone(), is(false)); + // Complete the fetch + blocker.complete(FetchSignal.SUCCESS); + assertThat(first.get(1, TimeUnit.SECONDS), equalTo(FetchSignal.SUCCESS)); + assertThat(second.get(1, TimeUnit.SECONDS), equalTo(FetchSignal.SUCCESS)); + } + + @Test + @Timeout(5) + void signalStrategyPropagatesNotFound() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + final Key key = new Key.From("missing/artifact.jar"); + final CompletableFuture blocker = new CompletableFuture<>(); + final CompletableFuture first = dedup.deduplicate( + key, () -> blocker + ); + final CompletableFuture second = dedup.deduplicate( + key, () -> CompletableFuture.completedFuture(FetchSignal.SUCCESS) + ); + blocker.complete(FetchSignal.NOT_FOUND); + assertThat(first.get(1, TimeUnit.SECONDS), equalTo(FetchSignal.NOT_FOUND)); + assertThat(second.get(1, TimeUnit.SECONDS), equalTo(FetchSignal.NOT_FOUND)); + } + + @Test + @Timeout(5) + void signalStrategyPropagatesError() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + final Key key = new Key.From("error/artifact.jar"); + final CompletableFuture blocker = new CompletableFuture<>(); + final CompletableFuture first = dedup.deduplicate( + key, () -> blocker + ); + final CompletableFuture second = dedup.deduplicate( + key, () -> CompletableFuture.completedFuture(FetchSignal.SUCCESS) + ); + // Complete with exception — should signal ERROR + blocker.completeExceptionally(new RuntimeException("upstream down")); + assertThat(first.get(1, TimeUnit.SECONDS), equalTo(FetchSignal.ERROR)); + assertThat(second.get(1, TimeUnit.SECONDS), equalTo(FetchSignal.ERROR)); + } + + @Test + @Timeout(5) + void signalStrategyCleansUpAfterCompletion() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + final Key key = new Key.From("cleanup/artifact.jar"); + assertThat("initially empty", dedup.inFlightCount(), equalTo(0)); + final CompletableFuture blocker = new CompletableFuture<>(); + dedup.deduplicate(key, () -> blocker); + assertThat("one in-flight", dedup.inFlightCount(), equalTo(1)); + blocker.complete(FetchSignal.SUCCESS); + // Allow async cleanup + Thread.sleep(50); + assertThat("cleaned up", dedup.inFlightCount(), equalTo(0)); + } + + @Test + @Timeout(5) + void signalStrategyAllowsNewRequestAfterCompletion() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + final Key key = new Key.From("reuse/artifact.jar"); + final AtomicInteger fetchCount = new AtomicInteger(0); + // First request + final CompletableFuture first = dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return CompletableFuture.completedFuture(FetchSignal.SUCCESS); + } + ); + first.get(1, TimeUnit.SECONDS); + Thread.sleep(50); + // Second request for same key after completion — should start new fetch + final CompletableFuture second = dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return CompletableFuture.completedFuture(FetchSignal.SUCCESS); + } + ); + second.get(1, TimeUnit.SECONDS); + assertThat("should have fetched twice", fetchCount.get(), equalTo(2)); + } + + @Test + @Timeout(5) + void noneStrategyDoesNotDeduplicate() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.NONE); + final Key key = new Key.From("none/artifact.jar"); + final AtomicInteger fetchCount = new AtomicInteger(0); + final CountDownLatch latch = new CountDownLatch(1); + final CompletableFuture first = dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return CompletableFuture.supplyAsync(() -> { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return FetchSignal.SUCCESS; + }); + } + ); + final CompletableFuture second = dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return CompletableFuture.completedFuture(FetchSignal.SUCCESS); + } + ); + // Both should have been called (no dedup) + second.get(1, TimeUnit.SECONDS); + assertThat("both fetches should have been invoked", fetchCount.get(), equalTo(2)); + latch.countDown(); + first.get(1, TimeUnit.SECONDS); + } + + @Test + @Timeout(5) + void storageStrategyDoesNotDeduplicate() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.STORAGE); + final Key key = new Key.From("storage/artifact.jar"); + final AtomicInteger fetchCount = new AtomicInteger(0); + dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return CompletableFuture.completedFuture(FetchSignal.SUCCESS); + } + ).get(1, TimeUnit.SECONDS); + dedup.deduplicate( + key, + () -> { + fetchCount.incrementAndGet(); + return CompletableFuture.completedFuture(FetchSignal.SUCCESS); + } + ).get(1, TimeUnit.SECONDS); + assertThat("STORAGE strategy delegates each call", fetchCount.get(), equalTo(2)); + } + + @Test + @Timeout(5) + void shutdownStopsCleanupAndClearsInFlight() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + final CompletableFuture neverComplete = new CompletableFuture<>(); + final CompletableFuture result = dedup.deduplicate( + new Key.From("shutdown/test"), () -> neverComplete + ); + assertThat("one in-flight before shutdown", dedup.inFlightCount(), equalTo(1)); + dedup.shutdown(); + assertThat("in-flight cleared after shutdown", dedup.inFlightCount(), equalTo(0)); + assertThat("result is done", result.isDone(), is(true)); + assertThat("result is ERROR", result.join(), equalTo(FetchSignal.ERROR)); + } + + @Test + @Timeout(5) + void closeIsIdempotent() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + dedup.close(); + dedup.close(); + assertThat("double close does not throw", true, is(true)); + } + + @Test + void differentKeysAreNotDeduplicated() throws Exception { + final RequestDeduplicator dedup = new RequestDeduplicator(DedupStrategy.SIGNAL); + final AtomicInteger fetchCount = new AtomicInteger(0); + final CompletableFuture blocker1 = new CompletableFuture<>(); + final CompletableFuture blocker2 = new CompletableFuture<>(); + dedup.deduplicate( + new Key.From("key1"), () -> { + fetchCount.incrementAndGet(); + return blocker1; + } + ); + dedup.deduplicate( + new Key.From("key2"), () -> { + fetchCount.incrementAndGet(); + return blocker2; + } + ); + assertThat("different keys should both fetch", fetchCount.get(), equalTo(2)); + blocker1.complete(FetchSignal.SUCCESS); + blocker2.complete(FetchSignal.SUCCESS); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/cache/SidecarFileTest.java b/artipie-core/src/test/java/com/artipie/http/cache/SidecarFileTest.java new file mode 100644 index 000000000..0c8741464 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/cache/SidecarFileTest.java @@ -0,0 +1,44 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.cache; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link SidecarFile}. + */ +class SidecarFileTest { + + @Test + void createsWithPathAndContent() { + final String path = "com/example/foo/1.0/foo-1.0.jar.sha256"; + final byte[] content = "abc123".getBytes(StandardCharsets.UTF_8); + final SidecarFile sidecar = new SidecarFile(path, content); + assertThat(sidecar.path(), equalTo(path)); + assertThat(sidecar.content(), equalTo(content)); + } + + @Test + void rejectsNullPath() { + assertThrows( + NullPointerException.class, + () -> new SidecarFile(null, new byte[]{1}) + ); + } + + @Test + void rejectsNullContent() { + assertThrows( + NullPointerException.class, + () -> new SidecarFile("path", null) + ); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/misc/ConfigDefaultsTest.java b/artipie-core/src/test/java/com/artipie/http/misc/ConfigDefaultsTest.java new file mode 100644 index 000000000..3828e9cc3 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/misc/ConfigDefaultsTest.java @@ -0,0 +1,50 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +import org.junit.jupiter.api.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +/** + * Tests for {@link ConfigDefaults}. + * + * @since 1.20.13 + */ +final class ConfigDefaultsTest { + + @Test + void returnsDefaultWhenNotSet() { + assertThat( + ConfigDefaults.get("ARTIPIE_TEST_NONEXISTENT_XYZ_123", "fallback"), + equalTo("fallback") + ); + } + + @Test + void returnsDefaultIntWhenNotSet() { + assertThat( + ConfigDefaults.getInt("ARTIPIE_TEST_NONEXISTENT_INT_456", 42), + equalTo(42) + ); + } + + @Test + void returnsDefaultOnInvalidInt() { + // System env won't have this, so it falls through to default + assertThat( + ConfigDefaults.getInt("ARTIPIE_TEST_NONEXISTENT_789", 100), + equalTo(100) + ); + } + + @Test + void returnsDefaultLongWhenNotSet() { + assertThat( + ConfigDefaults.getLong("ARTIPIE_TEST_NONEXISTENT_LONG", 120000L), + equalTo(120000L) + ); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/misc/DispatchedStorageTest.java b/artipie-core/src/test/java/com/artipie/http/misc/DispatchedStorageTest.java new file mode 100644 index 000000000..5762afe10 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/misc/DispatchedStorageTest.java @@ -0,0 +1,299 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +import com.artipie.asto.Content; +import com.artipie.asto.Key; +import com.artipie.asto.ListResult; +import com.artipie.asto.Meta; +import com.artipie.asto.Storage; +import com.artipie.asto.memory.InMemoryStorage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; + +/** + * Tests for {@link DispatchedStorage}. + *

+ * Uses {@link SlowStorage} wrapper to ensure delegate futures complete + * asynchronously, guaranteeing that downstream callbacks observe the + * dispatched executor thread rather than the calling thread. + * + * @since 1.20.13 + */ +final class DispatchedStorageTest { + + /** + * Delegate in-memory storage. + */ + private InMemoryStorage memory; + + /** + * Storage under test (dispatched wrapper around async delegate). + */ + private DispatchedStorage storage; + + @BeforeEach + void setUp() { + this.memory = new InMemoryStorage(); + this.storage = new DispatchedStorage(new SlowStorage(this.memory)); + } + + @Test + void readOpsRunOnReadPool() throws Exception { + final Key key = new Key.From("test-read"); + this.memory.save( + key, + new Content.From("data".getBytes(StandardCharsets.UTF_8)) + ).join(); + final AtomicReference threadName = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + this.storage.exists(key) + .whenComplete( + (val, err) -> { + threadName.set(Thread.currentThread().getName()); + latch.countDown(); + } + ); + assertThat( + "Latch should count down within timeout", + latch.await(5, TimeUnit.SECONDS), is(true) + ); + assertThat( + "exists() completion should run on the read pool", + threadName.get(), startsWith("artipie-io-read-") + ); + } + + @Test + void writeOpsRunOnWritePool() throws Exception { + final Key key = new Key.From("test-write"); + final AtomicReference threadName = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + this.storage.save( + key, + new Content.From("data".getBytes(StandardCharsets.UTF_8)) + ).whenComplete( + (val, err) -> { + threadName.set(Thread.currentThread().getName()); + latch.countDown(); + } + ); + assertThat( + "Latch should count down within timeout", + latch.await(5, TimeUnit.SECONDS), is(true) + ); + assertThat( + "save() completion should run on the write pool", + threadName.get(), startsWith("artipie-io-write-") + ); + } + + @Test + void listOpsRunOnListPool() throws Exception { + final AtomicReference threadName = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + this.storage.list(Key.ROOT) + .whenComplete( + (val, err) -> { + threadName.set(Thread.currentThread().getName()); + latch.countDown(); + } + ); + assertThat( + "Latch should count down within timeout", + latch.await(5, TimeUnit.SECONDS), is(true) + ); + assertThat( + "list() completion should run on the list pool", + threadName.get(), startsWith("artipie-io-list-") + ); + } + + @Test + void deleteAllRunsOnWritePool() throws Exception { + final Key key = new Key.From("test-delete-all", "item"); + this.memory.save( + key, + new Content.From("data".getBytes(StandardCharsets.UTF_8)) + ).join(); + final AtomicReference threadName = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + this.storage.deleteAll(new Key.From("test-delete-all")) + .whenComplete( + (val, err) -> { + threadName.set(Thread.currentThread().getName()); + latch.countDown(); + } + ); + assertThat( + "Latch should count down within timeout", + latch.await(5, TimeUnit.SECONDS), is(true) + ); + assertThat( + "deleteAll() completion should run on the write pool", + threadName.get(), startsWith("artipie-io-write-") + ); + } + + @Test + void identifierIncludesDelegate() { + final DispatchedStorage direct = new DispatchedStorage(this.memory); + assertThat( + "identifier should contain delegate's identifier", + direct.identifier(), + containsString(this.memory.identifier()) + ); + } + + @Test + void exclusivelyDelegatesCorrectly() throws Exception { + final Key key = new Key.From("exclusive-key"); + this.memory.save( + key, + new Content.From("exclusive-data".getBytes(StandardCharsets.UTF_8)) + ).join(); + final Boolean result = this.storage.exclusively( + key, + (Storage sto) -> { + final CompletionStage stage = sto.exists(key); + return stage; + } + ).toCompletableFuture().get(); + assertThat( + "exclusively() should delegate correctly and return result", + result, is(true) + ); + } + + /** + * Storage wrapper that makes all operations genuinely asynchronous + * by adding a small delay. This ensures delegate futures are not + * already complete when the dispatch mechanism registers its callback, + * making thread-pool assertions deterministic. + */ + private static final class SlowStorage implements Storage { + + /** + * Underlying storage. + */ + private final Storage origin; + + /** + * Ctor. + * @param origin Delegate storage + */ + SlowStorage(final Storage origin) { + this.origin = origin; + } + + @Override + public CompletableFuture exists(final Key key) { + return this.delayed(this.origin.exists(key)); + } + + @Override + public CompletableFuture> list(final Key prefix) { + return this.delayed(this.origin.list(prefix)); + } + + @Override + public CompletableFuture list( + final Key prefix, final String delimiter + ) { + return this.delayed(this.origin.list(prefix, delimiter)); + } + + @Override + public CompletableFuture save( + final Key key, final Content content + ) { + return this.delayed(this.origin.save(key, content)); + } + + @Override + public CompletableFuture move( + final Key source, final Key destination + ) { + return this.delayed(this.origin.move(source, destination)); + } + + @Override + public CompletableFuture metadata(final Key key) { + return this.delayed(this.origin.metadata(key)); + } + + @Override + public CompletableFuture value(final Key key) { + return this.delayed(this.origin.value(key)); + } + + @Override + public CompletableFuture delete(final Key key) { + return this.delayed(this.origin.delete(key)); + } + + @Override + public CompletableFuture deleteAll(final Key prefix) { + return this.delayed(this.origin.deleteAll(prefix)); + } + + @Override + public CompletionStage exclusively( + final Key key, + final Function> operation + ) { + return this.origin.exclusively(key, operation); + } + + @Override + public String identifier() { + return this.origin.identifier(); + } + + /** + * Add a small async delay so the returned future is not already + * completed when the caller receives it. + * @param source Original future + * @param Result type + * @return Delayed future + */ + private CompletableFuture delayed( + final CompletableFuture source + ) { + final CompletableFuture result = new CompletableFuture<>(); + source.whenComplete( + (val, err) -> CompletableFuture.runAsync( + () -> { + try { + Thread.sleep(10); + } catch (final InterruptedException ex) { + Thread.currentThread().interrupt(); + } + if (err != null) { + result.completeExceptionally(err); + } else { + result.complete(val); + } + } + ) + ); + return result; + } + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/misc/RepoNameMeterFilterTest.java b/artipie-core/src/test/java/com/artipie/http/misc/RepoNameMeterFilterTest.java new file mode 100644 index 000000000..8b053a0bf --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/misc/RepoNameMeterFilterTest.java @@ -0,0 +1,63 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +/** + * Tests for {@link RepoNameMeterFilter}. + * + * @since 1.20.13 + */ +final class RepoNameMeterFilterTest { + + @Test + void allowsReposUnderLimit() { + final MeterRegistry registry = new SimpleMeterRegistry(); + registry.config().meterFilter(new RepoNameMeterFilter(3)); + registry.counter("test.requests", "repo_name", "repo1").increment(); + registry.counter("test.requests", "repo_name", "repo2").increment(); + registry.counter("test.requests", "repo_name", "repo3").increment(); + assertThat( + "Should have 3 distinct counters", + registry.find("test.requests").counters().size(), + equalTo(3) + ); + } + + @Test + void capsReposOverLimit() { + final MeterRegistry registry = new SimpleMeterRegistry(); + registry.config().meterFilter(new RepoNameMeterFilter(2)); + registry.counter("test.requests", "repo_name", "repo1").increment(); + registry.counter("test.requests", "repo_name", "repo2").increment(); + registry.counter("test.requests", "repo_name", "repo3").increment(); + registry.counter("test.requests", "repo_name", "repo4").increment(); + // repo3 and repo4 should be bucketed into "_other" + assertThat( + "Should have at most 3 counters (2 named + _other)", + registry.find("test.requests").counters().size(), + lessThanOrEqualTo(3) + ); + } + + @Test + void passesMetersWithoutRepoTag() { + final MeterRegistry registry = new SimpleMeterRegistry(); + registry.config().meterFilter(new RepoNameMeterFilter(1)); + registry.counter("test.other", "method", "GET").increment(); + assertThat( + "Meters without repo_name tag should pass through", + registry.find("test.other").counters().size(), + equalTo(1) + ); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/misc/StorageExecutorsTest.java b/artipie-core/src/test/java/com/artipie/http/misc/StorageExecutorsTest.java new file mode 100644 index 000000000..8a1489e7d --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/misc/StorageExecutorsTest.java @@ -0,0 +1,68 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.misc; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; +import static org.hamcrest.Matchers.startsWith; + +/** + * Tests for {@link StorageExecutors}. + * + * @since 1.20.13 + */ +final class StorageExecutorsTest { + + @Test + void readPoolUsesNamedThreads() throws ExecutionException, InterruptedException { + final String name = StorageExecutors.READ.submit( + () -> Thread.currentThread().getName() + ).get(); + assertThat("Read pool thread should have correct name", + name, startsWith("artipie-io-read-")); + } + + @Test + void writePoolUsesNamedThreads() throws ExecutionException, InterruptedException { + final String name = StorageExecutors.WRITE.submit( + () -> Thread.currentThread().getName() + ).get(); + assertThat("Write pool thread should have correct name", + name, startsWith("artipie-io-write-")); + } + + @Test + void listPoolUsesNamedThreads() throws ExecutionException, InterruptedException { + final String name = StorageExecutors.LIST.submit( + () -> Thread.currentThread().getName() + ).get(); + assertThat("List pool thread should have correct name", + name, startsWith("artipie-io-list-")); + } + + @Test + void poolsAreDistinct() { + assertThat("READ and WRITE should be different pools", + StorageExecutors.READ, is(not(sameInstance(StorageExecutors.WRITE)))); + assertThat("READ and LIST should be different pools", + StorageExecutors.READ, is(not(sameInstance(StorageExecutors.LIST)))); + assertThat("WRITE and LIST should be different pools", + StorageExecutors.WRITE, is(not(sameInstance(StorageExecutors.LIST)))); + } + + @Test + void threadsAreDaemons() throws ExecutionException, InterruptedException { + final Boolean isDaemon = StorageExecutors.READ.submit( + () -> Thread.currentThread().isDaemon() + ).get(); + assertThat("Pool threads should be daemon threads", isDaemon, is(true)); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/retry/RetrySliceTest.java b/artipie-core/src/test/java/com/artipie/http/retry/RetrySliceTest.java new file mode 100644 index 000000000..cbb488607 --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/retry/RetrySliceTest.java @@ -0,0 +1,229 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.retry; + +import com.artipie.asto.Content; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.Slice; +import com.artipie.http.rq.RequestLine; +import com.artipie.http.rq.RqMethod; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +/** + * Tests for {@link RetrySlice}. + */ +class RetrySliceTest { + + @Test + void returnsSuccessWithoutRetry() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + calls.incrementAndGet(); + return CompletableFuture.completedFuture( + ResponseBuilder.ok().build() + ); + }; + final Response response = new RetrySlice(origin, 2, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(200)); + assertThat(calls.get(), equalTo(1)); + } + + @Test + void retriesOn500AndSucceeds() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + if (calls.incrementAndGet() <= 1) { + return CompletableFuture.completedFuture( + ResponseBuilder.internalError().build() + ); + } + return CompletableFuture.completedFuture( + ResponseBuilder.ok().build() + ); + }; + final Response response = new RetrySlice(origin, 2, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(200)); + assertThat(calls.get(), equalTo(2)); + } + + @Test + void doesNotRetryOn404() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + calls.incrementAndGet(); + return CompletableFuture.completedFuture( + ResponseBuilder.notFound().build() + ); + }; + final Response response = new RetrySlice(origin, 2, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(404)); + assertThat(calls.get(), equalTo(1)); + } + + @Test + void respectsMaxRetries() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + calls.incrementAndGet(); + return CompletableFuture.completedFuture( + ResponseBuilder.internalError().build() + ); + }; + final Response response = new RetrySlice(origin, 3, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(500)); + // 1 initial + 3 retries = 4 total calls + assertThat(calls.get(), equalTo(4)); + } + + @Test + void retriesOnException() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + if (calls.incrementAndGet() <= 1) { + final CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new RuntimeException("connection reset")); + return future; + } + return CompletableFuture.completedFuture( + ResponseBuilder.ok().build() + ); + }; + final Response response = new RetrySlice(origin, 2, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(200)); + assertThat(calls.get(), equalTo(2)); + } + + @Test + void zeroRetriesMeansNoRetry() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + calls.incrementAndGet(); + return CompletableFuture.completedFuture( + ResponseBuilder.internalError().build() + ); + }; + final Response response = new RetrySlice(origin, 0, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(500)); + assertThat(calls.get(), equalTo(1)); + } + + @Test + void doesNotRetryOn400() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + calls.incrementAndGet(); + return CompletableFuture.completedFuture( + ResponseBuilder.badRequest().build() + ); + }; + final Response response = new RetrySlice(origin, 2, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(400)); + assertThat(calls.get(), equalTo(1)); + } + + @Test + void usesDefaultConfiguration() throws Exception { + final Slice origin = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + final Response response = new RetrySlice(origin) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(200)); + } + + @Test + void retriesOn503() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final Slice origin = (line, headers, body) -> { + if (calls.incrementAndGet() <= 1) { + return CompletableFuture.completedFuture( + ResponseBuilder.unavailable().build() + ); + } + return CompletableFuture.completedFuture( + ResponseBuilder.ok().build() + ); + }; + final Response response = new RetrySlice(origin, 2, Duration.ofMillis(1), 1.0) + .response( + new RequestLine(RqMethod.GET, "/test"), + Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(response.status().code(), equalTo(200)); + assertThat(calls.get(), equalTo(2)); + } + + @Test + void retriesWithJitter() throws Exception { + final AtomicInteger calls = new AtomicInteger(0); + final List timestamps = Collections.synchronizedList(new ArrayList<>()); + final Slice failing = (line, headers, body) -> { + timestamps.add(System.nanoTime()); + calls.incrementAndGet(); + return CompletableFuture.completedFuture( + ResponseBuilder.internalError().build() + ); + }; + final RetrySlice retry = new RetrySlice(failing, 2, Duration.ofMillis(100), 2.0); + retry.response( + new RequestLine("GET", "/test"), + Headers.EMPTY, + Content.EMPTY + ).handle((resp, err) -> null).join(); + assertThat(calls.get(), equalTo(3)); + if (timestamps.size() >= 3) { + final long firstRetryDelay = + (timestamps.get(1) - timestamps.get(0)) / 1_000_000; + assertThat( + "First retry delay >= 90ms", + firstRetryDelay, + greaterThanOrEqualTo(90L) + ); + } + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/slice/CircuitBreakerSliceTest.java b/artipie-core/src/test/java/com/artipie/http/slice/CircuitBreakerSliceTest.java new file mode 100644 index 000000000..4bb4763db --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/slice/CircuitBreakerSliceTest.java @@ -0,0 +1,78 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.slice; + +import com.artipie.asto.Content; +import com.artipie.http.Headers; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.Slice; +import com.artipie.http.rq.RequestLine; +import com.artipie.http.timeout.AutoBlockRegistry; +import com.artipie.http.timeout.AutoBlockSettings; +import org.junit.jupiter.api.Test; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +final class CircuitBreakerSliceTest { + + @Test + void passesRequestsWhenHealthy() throws Exception { + final AutoBlockRegistry registry = new AutoBlockRegistry(AutoBlockSettings.defaults()); + final Slice origin = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + final CircuitBreakerSlice slice = new CircuitBreakerSlice(origin, registry, "test-remote"); + final var resp = slice.response( + new RequestLine("GET", "/test"), Headers.EMPTY, Content.EMPTY + ).join(); + assertThat(resp.status().code(), equalTo(200)); + } + + @Test + void failsFastWhenBlocked() throws Exception { + final AutoBlockRegistry registry = new AutoBlockRegistry(new AutoBlockSettings( + 1, Duration.ofMinutes(5), Duration.ofMinutes(60) + )); + final Slice origin = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + final CircuitBreakerSlice slice = new CircuitBreakerSlice(origin, registry, "test-remote"); + registry.recordFailure("test-remote"); + final var resp = slice.response( + new RequestLine("GET", "/test"), Headers.EMPTY, Content.EMPTY + ).join(); + assertThat(resp.status().code(), equalTo(503)); + } + + @Test + void recordsFailureOnServerError() throws Exception { + final AutoBlockRegistry registry = new AutoBlockRegistry(new AutoBlockSettings( + 2, Duration.ofMinutes(5), Duration.ofMinutes(60) + )); + final Slice origin = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.internalError().build()); + final CircuitBreakerSlice slice = new CircuitBreakerSlice(origin, registry, "test-remote"); + slice.response(new RequestLine("GET", "/t"), Headers.EMPTY, Content.EMPTY).join(); + slice.response(new RequestLine("GET", "/t"), Headers.EMPTY, Content.EMPTY).join(); + assertThat("Blocked after 2 failures", registry.isBlocked("test-remote"), equalTo(true)); + } + + @Test + void recordsSuccessOnOk() throws Exception { + final AutoBlockRegistry registry = new AutoBlockRegistry(new AutoBlockSettings( + 1, Duration.ofMinutes(5), Duration.ofMinutes(60) + )); + final Slice origin = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + final CircuitBreakerSlice slice = new CircuitBreakerSlice(origin, registry, "test-remote"); + registry.recordFailure("test-remote"); // block it + registry.recordSuccess("test-remote"); // unblock via direct registry + // Should pass through now + final var resp = slice.response( + new RequestLine("GET", "/test"), Headers.EMPTY, Content.EMPTY + ).join(); + assertThat(resp.status().code(), equalTo(200)); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/timeout/AutoBlockRegistryTest.java b/artipie-core/src/test/java/com/artipie/http/timeout/AutoBlockRegistryTest.java new file mode 100644 index 000000000..93b1ac39b --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/timeout/AutoBlockRegistryTest.java @@ -0,0 +1,105 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.timeout; + +import java.time.Duration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +final class AutoBlockRegistryTest { + + private AutoBlockRegistry registry; + + @BeforeEach + void setUp() { + this.registry = new AutoBlockRegistry(new AutoBlockSettings( + 3, Duration.ofMillis(100), Duration.ofMinutes(60) + )); + } + + @Test + void startsUnblocked() { + assertThat(this.registry.isBlocked("remote-1"), is(false)); + assertThat(this.registry.status("remote-1"), equalTo("online")); + } + + @Test + void blocksAfterThresholdFailures() { + this.registry.recordFailure("remote-1"); + this.registry.recordFailure("remote-1"); + assertThat( + "Not blocked after 2", + this.registry.isBlocked("remote-1"), is(false) + ); + this.registry.recordFailure("remote-1"); + assertThat( + "Blocked after 3", + this.registry.isBlocked("remote-1"), is(true) + ); + assertThat(this.registry.status("remote-1"), equalTo("blocked")); + } + + @Test + void unblocksAfterDuration() throws Exception { + final AutoBlockRegistry fast = new AutoBlockRegistry(new AutoBlockSettings( + 1, Duration.ofMillis(50), Duration.ofMinutes(60) + )); + fast.recordFailure("remote-1"); + assertThat(fast.isBlocked("remote-1"), is(true)); + Thread.sleep(100); + assertThat(fast.isBlocked("remote-1"), is(false)); + assertThat(fast.status("remote-1"), equalTo("probing")); + } + + @Test + void resetsOnSuccess() { + this.registry.recordFailure("remote-1"); + this.registry.recordFailure("remote-1"); + this.registry.recordFailure("remote-1"); + assertThat(this.registry.isBlocked("remote-1"), is(true)); + this.registry.recordSuccess("remote-1"); + assertThat(this.registry.isBlocked("remote-1"), is(false)); + assertThat(this.registry.status("remote-1"), equalTo("online")); + } + + @Test + void usesFibonacciBackoff() throws Exception { + final AutoBlockRegistry fast = new AutoBlockRegistry(new AutoBlockSettings( + 1, Duration.ofMillis(50), Duration.ofHours(1) + )); + // First block: 50ms (fib[0]=1) + fast.recordFailure("r1"); + assertThat(fast.isBlocked("r1"), is(true)); + Thread.sleep(80); + assertThat("Unblocked after first interval", fast.isBlocked("r1"), is(false)); + // Second block: 50ms (fib[1]=1, same duration) + fast.recordFailure("r1"); + assertThat(fast.isBlocked("r1"), is(true)); + Thread.sleep(80); + assertThat( + "Unblocked after second interval", fast.isBlocked("r1"), is(false) + ); + // Third block: 100ms (fib[2]=2) + fast.recordFailure("r1"); + assertThat(fast.isBlocked("r1"), is(true)); + Thread.sleep(60); + assertThat( + "Still blocked during longer interval", + fast.isBlocked("r1"), is(true) + ); + } + + @Test + void tracksMultipleRemotesIndependently() { + this.registry.recordFailure("remote-a"); + this.registry.recordFailure("remote-a"); + this.registry.recordFailure("remote-a"); + assertThat(this.registry.isBlocked("remote-a"), is(true)); + assertThat(this.registry.isBlocked("remote-b"), is(false)); + } +} diff --git a/artipie-core/src/test/java/com/artipie/http/timeout/TimeoutSettingsTest.java b/artipie-core/src/test/java/com/artipie/http/timeout/TimeoutSettingsTest.java new file mode 100644 index 000000000..a6a2dd2ef --- /dev/null +++ b/artipie-core/src/test/java/com/artipie/http/timeout/TimeoutSettingsTest.java @@ -0,0 +1,60 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.http.timeout; + +import java.time.Duration; +import org.junit.jupiter.api.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +final class TimeoutSettingsTest { + + @Test + void usesDefaults() { + final TimeoutSettings settings = TimeoutSettings.defaults(); + assertThat(settings.connectionTimeout(), equalTo(Duration.ofSeconds(5))); + assertThat(settings.idleTimeout(), equalTo(Duration.ofSeconds(30))); + assertThat(settings.requestTimeout(), equalTo(Duration.ofSeconds(120))); + } + + @Test + void overridesWithCustomValues() { + final TimeoutSettings settings = new TimeoutSettings( + Duration.ofSeconds(3), Duration.ofSeconds(15), Duration.ofSeconds(60) + ); + assertThat(settings.connectionTimeout(), equalTo(Duration.ofSeconds(3))); + assertThat(settings.idleTimeout(), equalTo(Duration.ofSeconds(15))); + assertThat(settings.requestTimeout(), equalTo(Duration.ofSeconds(60))); + } + + @Test + void mergesWithParent() { + final TimeoutSettings parent = new TimeoutSettings( + Duration.ofSeconds(10), Duration.ofSeconds(60), Duration.ofSeconds(180) + ); + final TimeoutSettings child = TimeoutSettings.builder() + .connectionTimeout(Duration.ofSeconds(3)) + .buildWithParent(parent); + assertThat(child.connectionTimeout(), equalTo(Duration.ofSeconds(3))); + assertThat( + "inherits idle from parent", + child.idleTimeout(), equalTo(Duration.ofSeconds(60)) + ); + assertThat( + "inherits request from parent", + child.requestTimeout(), equalTo(Duration.ofSeconds(180)) + ); + } + + @Test + void builderWithoutParentUsesDefaults() { + final TimeoutSettings settings = TimeoutSettings.builder() + .connectionTimeout(Duration.ofSeconds(2)) + .build(); + assertThat(settings.connectionTimeout(), equalTo(Duration.ofSeconds(2))); + assertThat(settings.idleTimeout(), equalTo(Duration.ofSeconds(30))); + assertThat(settings.requestTimeout(), equalTo(Duration.ofSeconds(120))); + } +} diff --git a/artipie-core/src/test/java/com/artipie/layout/GradleLayoutTest.java b/artipie-core/src/test/java/com/artipie/layout/GradleLayoutTest.java deleted file mode 100644 index a8cb0cf94..000000000 --- a/artipie-core/src/test/java/com/artipie/layout/GradleLayoutTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com - * https://github.com/artipie/artipie/blob/master/LICENSE.txt - */ -package com.artipie.layout; - -import com.artipie.asto.Key; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Assertions; - -import java.util.HashMap; -import java.util.Map; - -/** - * Tests for {@link GradleLayout}. - */ -class GradleLayoutTest { - - @Test - void testArtifactPathWithSimpleGroupId() { - final GradleLayout layout = new GradleLayout(); - final Map meta = new HashMap<>(); - meta.put(GradleLayout.GROUP_ID, "com.example"); - meta.put(GradleLayout.ARTIFACT_ID, "my-library"); - - final BaseArtifactInfo info = new BaseArtifactInfo( - "gradle-repo", - "my-library", - "1.0.0", - meta - ); - - final Key path = layout.artifactPath(info); - Assertions.assertEquals( - "gradle-repo/com/example/my-library/1.0.0", - path.string() - ); - } - - @Test - void testArtifactPathWithComplexGroupId() { - final GradleLayout layout = new GradleLayout(); - final Map meta = new HashMap<>(); - meta.put(GradleLayout.GROUP_ID, "org.springframework.boot"); - meta.put(GradleLayout.ARTIFACT_ID, "spring-boot-starter"); - - final BaseArtifactInfo info = new BaseArtifactInfo( - "gradle-central", - "spring-boot-starter", - "2.7.0", - meta - ); - - final Key path = layout.artifactPath(info); - Assertions.assertEquals( - "gradle-central/org/springframework/boot/spring-boot-starter/2.7.0", - path.string() - ); - } - - @Test - void testMetadataPath() { - final GradleLayout layout = new GradleLayout(); - final Map meta = new HashMap<>(); - meta.put(GradleLayout.GROUP_ID, "com.example"); - meta.put(GradleLayout.ARTIFACT_ID, "my-library"); - - final BaseArtifactInfo info = new BaseArtifactInfo( - "gradle-repo", - "my-library", - "1.0.0", - meta - ); - - final Key path = layout.metadataPath(info); - Assertions.assertEquals( - "gradle-repo/com/example/my-library/1.0.0", - path.string() - ); - } - - @Test - void testMissingGroupIdThrowsException() { - final GradleLayout layout = new GradleLayout(); - final Map meta = new HashMap<>(); - meta.put(GradleLayout.ARTIFACT_ID, "my-library"); - - final BaseArtifactInfo info = new BaseArtifactInfo( - "gradle-repo", - "my-library", - "1.0.0", - meta - ); - - Assertions.assertThrows( - IllegalArgumentException.class, - () -> layout.artifactPath(info) - ); - } - - @Test - void testMissingArtifactIdThrowsException() { - final GradleLayout layout = new GradleLayout(); - final Map meta = new HashMap<>(); - meta.put(GradleLayout.GROUP_ID, "com.example"); - - final BaseArtifactInfo info = new BaseArtifactInfo( - "gradle-repo", - "my-library", - "1.0.0", - meta - ); - - Assertions.assertThrows( - IllegalArgumentException.class, - () -> layout.artifactPath(info) - ); - } -} diff --git a/artipie-core/src/test/java/com/artipie/layout/LayoutFactoryTest.java b/artipie-core/src/test/java/com/artipie/layout/LayoutFactoryTest.java index 59ee852f5..547d3157d 100644 --- a/artipie-core/src/test/java/com/artipie/layout/LayoutFactoryTest.java +++ b/artipie-core/src/test/java/com/artipie/layout/LayoutFactoryTest.java @@ -57,7 +57,7 @@ void testForTypeGradle() { final StorageLayout layout = LayoutFactory.forType( LayoutFactory.RepositoryType.GRADLE ); - Assertions.assertInstanceOf(GradleLayout.class, layout); + Assertions.assertInstanceOf(MavenLayout.class, layout); } @Test diff --git a/artipie-main/Dockerfile b/artipie-main/Dockerfile index 22b005e99..4113e42a5 100644 --- a/artipie-main/Dockerfile +++ b/artipie-main/Dockerfile @@ -25,7 +25,7 @@ RUN addgroup -g 2020 -S artipie && \ chown artipie:artipie /opt/apm/elastic-apm-agent.jar ENV TMPDIR=/var/artipie/cache/tmp -ENV ARTIPIE_VERSION=1.20.12 +ENV ARTIPIE_VERSION=1.22.0 USER 2021:2020 diff --git a/artipie-main/docker-compose/.env.example b/artipie-main/docker-compose/.env.example index 5657556ff..b599cc55f 100644 --- a/artipie-main/docker-compose/.env.example +++ b/artipie-main/docker-compose/.env.example @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------- # Artipie Configuration # ----------------------------------------------------------------------------- -ARTIPIE_VERSION=1.20.12 +ARTIPIE_VERSION=1.22.0 ARTIPIE_USER_NAME=artipie ARTIPIE_USER_PASS=changeme ARTIPIE_CONFIG=/etc/artipie/artipie.yml diff --git a/artipie-main/docker-compose/artipie/repo/gradle_proxy.yaml b/artipie-main/docker-compose/artipie/repo/gradle_proxy.yaml index b840dc18a..d9960bd25 100644 --- a/artipie-main/docker-compose/artipie/repo/gradle_proxy.yaml +++ b/artipie-main/docker-compose/artipie/repo/gradle_proxy.yaml @@ -5,3 +5,4 @@ repo: path: /var/artipie/data remotes: - url: https://repo1.maven.org/maven2 + - url: https://plugins.gradle.org/m2 diff --git a/artipie-main/docker-compose/artipie/security/users/ayd.asraf@auto1.local.yml b/artipie-main/docker-compose/artipie/security/users/ayd.asraf@auto1.local.yml new file mode 100644 index 000000000..6b08fb748 --- /dev/null +++ b/artipie-main/docker-compose/artipie/security/users/ayd.asraf@auto1.local.yml @@ -0,0 +1,5 @@ +type: okta +roles: + - reader + - admin +email: "ayd.asraf@auto1.com" \ No newline at end of file diff --git a/artipie-main/docker-compose/docker-compose.yaml b/artipie-main/docker-compose/docker-compose.yaml index e45855fa9..f6b084acd 100644 --- a/artipie-main/docker-compose/docker-compose.yaml +++ b/artipie-main/docker-compose/docker-compose.yaml @@ -84,6 +84,7 @@ services: restart: unless-stopped depends_on: - artipie + - artipie-ui ports: - "8081:80" - "8443:443" @@ -121,7 +122,7 @@ services: - artipie-net artipie-db: - image: postgres:15-alpine + image: postgres:17.8-alpine container_name: artipie-db restart: unless-stopped networks: @@ -185,6 +186,24 @@ services: depends_on: - artipie + artipie-ui: + build: + context: ../../artipie-ui + dockerfile: Dockerfile + container_name: artipie-ui + restart: unless-stopped + depends_on: + - artipie + environment: + - API_BASE_URL=http://localhost:8086/api/v1 + - GRAFANA_URL=http://localhost:3000/goto/bfgfvn3efggsge?orgId=1 + - APP_TITLE=Artipie + - DEFAULT_PAGE_SIZE=25 + ports: + - "8090:80" + networks: + - artipie-net + grafana: image: grafana/grafana:latest container_name: grafana diff --git a/artipie-main/docker-compose/nginx/conf.d/default.conf b/artipie-main/docker-compose/nginx/conf.d/default.conf index cc8dbebb1..a6ec16aac 100644 --- a/artipie-main/docker-compose/nginx/conf.d/default.conf +++ b/artipie-main/docker-compose/nginx/conf.d/default.conf @@ -2,6 +2,15 @@ server { listen 80; server_name localhost; + # Artipie UI — management console + location /ui/ { + proxy_pass http://artipie-ui:80/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location / { client_max_body_size 500M; proxy_pass http://artipie:8080; @@ -33,6 +42,15 @@ server { ssl_certificate_key /etc/nginx/ssl/nginx.key; client_max_body_size 500M; + # Artipie UI — management console + location /ui/ { + proxy_pass http://artipie-ui:80/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location / { client_max_body_size 500M; proxy_pass http://artipie:8080; diff --git a/artipie-main/pom.xml b/artipie-main/pom.xml index f80d816e4..b7f14d106 100644 --- a/artipie-main/pom.xml +++ b/artipie-main/pom.xml @@ -26,7 +26,7 @@ SOFTWARE. artipie com.artipie - 1.20.12 + 1.22.0 4.0.0 artipie-main @@ -41,13 +41,13 @@ SOFTWARE. com.artipie artipie-core - 1.20.12 + 1.22.0 compile com.artipie asto-core - 1.20.12 + 1.22.0 @@ -56,25 +56,22 @@ SOFTWARE. - - com.artipie - asto-etcd - 1.20.12 - - - com.artipie - asto-redis - 1.20.12 - com.artipie asto-s3 - 1.20.12 + 1.22.0 com.artipie asto-vertx-file - 1.20.12 + 1.22.0 + + + + io.reactivex.rxjava3 + rxjava + 3.1.6 com.jcabi @@ -98,13 +95,13 @@ SOFTWARE. com.artipie vertx-server - 1.20.12 + 1.22.0 compile com.artipie http-client - 1.20.12 + 1.22.0 compile @@ -127,11 +124,6 @@ SOFTWARE. guava ${guava.version} - - io.etcd - jetcd-core - 0.7.1 - io.prometheus simpleclient @@ -241,7 +233,7 @@ SOFTWARE. org.apache.logging.log4j log4j-core - 2.22.1 + 2.24.3 runtime @@ -263,37 +255,25 @@ SOFTWARE. com.artipie files-adapter - 1.20.12 + 1.22.0 compile com.artipie npm-adapter - 1.20.12 + 1.22.0 compile com.artipie hexpm-adapter - 1.20.12 + 1.22.0 compile com.artipie maven-adapter - 1.20.12 - compile - - - com.jcabi - jcabi-xml - - - - - com.artipie - gradle-adapter - 1.20.12 + 1.22.0 compile @@ -302,70 +282,71 @@ SOFTWARE. + com.artipie rpm-adapter - 1.20.12 + 1.22.0 compile com.artipie gem-adapter - 1.20.12 + 1.22.0 compile com.artipie composer-adapter - 1.20.12 + 1.22.0 compile com.artipie go-adapter - 1.20.12 + 1.22.0 compile com.artipie nuget-adapter - 1.20.12 + 1.22.0 compile com.artipie pypi-adapter - 1.20.12 + 1.22.0 compile com.artipie helm-adapter - 1.20.12 + 1.22.0 compile com.artipie docker-adapter - 1.20.12 + 1.22.0 compile com.artipie debian-adapter - 1.20.12 + 1.22.0 compile com.artipie conda-adapter - 1.20.12 + 1.22.0 compile com.artipie conan-adapter - 1.20.12 + 1.22.0 compile @@ -394,8 +375,51 @@ SOFTWARE. 1.5.1 test + + org.flywaydb + flyway-core + 10.22.0 + + + org.flywaydb + flyway-database-postgresql + 10.22.0 + + + org.mindrot + jbcrypt + 0.4 + + + + io.vertx + vertx-web-client + ${vertx.version} + + + + io.vertx + vertx-junit5 + test + + + + src/main/resources + true + + artipie.properties + + + + src/main/resources + false + + artipie.properties + + + org.apache.maven.plugins diff --git a/artipie-main/src/main/java/com/artipie/RepositorySlices.java b/artipie-main/src/main/java/com/artipie/RepositorySlices.java index 8239b7360..99b3adac4 100644 --- a/artipie-main/src/main/java/com/artipie/RepositorySlices.java +++ b/artipie-main/src/main/java/com/artipie/RepositorySlices.java @@ -7,7 +7,7 @@ import com.artipie.adapters.docker.DockerProxy; import com.artipie.adapters.file.FileProxy; import com.artipie.adapters.go.GoProxy; -import com.artipie.adapters.gradle.GradleProxy; + import com.artipie.adapters.maven.MavenProxy; import com.artipie.adapters.php.ComposerGroupSlice; import com.artipie.adapters.php.ComposerProxy; @@ -31,7 +31,7 @@ import com.artipie.cooldown.CooldownSupport; import com.artipie.files.FilesSlice; import com.artipie.gem.http.GemSlice; -import com.artipie.gradle.http.GradleSlice; + import com.artipie.helm.http.HelmSlice; import com.artipie.hex.http.HexSlice; import com.artipie.http.ContentLengthRestriction; @@ -42,6 +42,7 @@ import com.artipie.http.Slice; import com.artipie.http.TimeoutSlice; import com.artipie.group.GroupSlice; +import com.artipie.index.ArtifactIndex; import com.artipie.http.auth.Authentication; import com.artipie.http.auth.BasicAuthScheme; import com.artipie.http.auth.CombinedAuthScheme; @@ -90,8 +91,11 @@ import java.util.Optional; import java.util.Queue; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -101,6 +105,27 @@ public class RepositorySlices { + /** + * Thread counter for the resolve executor pool. + */ + private static final AtomicInteger RESOLVE_COUNTER = new AtomicInteger(0); + + /** + * Dedicated executor for blocking slice resolution operations (e.g. Jetty client start). + * Prevents blocking the Vert.x event loop when proxy repositories are initialized. + */ + private static final ExecutorService RESOLVE_EXECUTOR = + Executors.newFixedThreadPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), + r -> { + final Thread t = new Thread( + r, "slice-resolve-" + RESOLVE_COUNTER.incrementAndGet() + ); + t.setDaemon(true); + return t; + } + ); + /** * Pattern to trim path before passing it to adapters' slice. */ @@ -155,6 +180,8 @@ public RepositorySlices( this.cooldownMetadata = CooldownSupport.createMetadataService(this.cooldown, settings); this.sharedClients = new SharedJettyClients(); this.slices = CacheBuilder.newBuilder() + .maximumSize(500) + .expireAfterAccess(30, java.util.concurrent.TimeUnit.MINUTES) .removalListener( (RemovalListener) notification -> notification.getValue() .client() @@ -212,8 +239,7 @@ public Slice slice(final Key name, final int port, final int depth) { .eventAction("slice_resolve") .eventOutcome("success") .field("repository.name", name.string()) - .field("port", port) - .field("source", "cache") + .field("url.port", port) .log(); return cached.slice(); } @@ -221,13 +247,12 @@ public Slice slice(final Key name, final int port, final int depth) { if (resolved.isPresent()) { this.slices.put(skey, resolved.get()); EcsLogger.debug("com.artipie.settings") - .message("Repository slice resolved and cached") + .message("Repository slice resolved and cached from config") .eventCategory("repository") .eventAction("slice_resolve") .eventOutcome("success") .field("repository.name", name.string()) - .field("port", port) - .field("source", "config") + .field("url.port", port) .log(); return resolved.get().slice(); } @@ -238,7 +263,7 @@ public Slice slice(final Key name, final int port, final int depth) { .eventAction("slice_resolve") .eventOutcome("failure") .field("repository.name", name.string()) - .field("port", port) + .field("url.port", port) .log(); return new SliceSimple( () -> ResponseBuilder.notFound() @@ -301,18 +326,14 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i try { switch (cfg.type()) { case "file": - // Use streaming browsing for fast directory listings slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new FilesSlice( - cfg.storage(), - securityPolicy(), - authentication(), - tokens.auth(), - cfg.name(), - artifactEvents() - ), - cfg.storage() + new FilesSlice( + cfg.storage(), + securityPolicy(), + authentication(), + tokens.auth(), + cfg.name(), + artifactEvents() ) ); break; @@ -328,46 +349,34 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i break; case "npm": slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new NpmSlice( - cfg.url(), cfg.storage(), securityPolicy(), authentication(), tokens.auth(), tokens, cfg.name(), artifactEvents(), true // JWT-only, no npm tokens - ), - cfg.storage() + new NpmSlice( + cfg.url(), cfg.storage(), securityPolicy(), authentication(), tokens.auth(), tokens, cfg.name(), artifactEvents(), true ) ); break; case "gem": slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new GemSlice( - cfg.storage(), - securityPolicy(), - authentication(), - tokens.auth(), - cfg.name(), - artifactEvents() - ), - cfg.storage() + new GemSlice( + cfg.storage(), + securityPolicy(), + authentication(), + tokens.auth(), + cfg.name(), + artifactEvents() ) ); break; case "helm": slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new HelmSlice( - cfg.storage(), cfg.url().toString(), securityPolicy(), authentication(), tokens.auth(), cfg.name(), artifactEvents() - ), - cfg.storage() + new HelmSlice( + cfg.storage(), cfg.url().toString(), securityPolicy(), authentication(), tokens.auth(), cfg.name(), artifactEvents() ) ); break; case "rpm": slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new RpmSlice(cfg.storage(), securityPolicy(), authentication(), - tokens.auth(), new com.artipie.rpm.RepoConfig.FromYaml(cfg.settings(), cfg.name()), Optional.empty()), - cfg.storage() - ) + new RpmSlice(cfg.storage(), securityPolicy(), authentication(), + tokens.auth(), new com.artipie.rpm.RepoConfig.FromYaml(cfg.settings(), cfg.name()), Optional.empty()) ); break; case "php": @@ -389,20 +398,17 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i slice = trimPathSlice( new PathPrefixStripSlice( - new com.artipie.http.slice.BrowsableSlice( - new PhpComposer( - new AstoRepository( - cfg.storage(), - Optional.of(baseUrl), - Optional.of(cfg.name()) - ), - securityPolicy(), - authentication(), - tokens.auth(), - cfg.name(), - artifactEvents() + new PhpComposer( + new AstoRepository( + cfg.storage(), + Optional.of(baseUrl), + Optional.of(cfg.name()) ), - cfg.storage() + securityPolicy(), + authentication(), + tokens.auth(), + cfg.name(), + artifactEvents() ), "direct-dists" ) @@ -413,17 +419,14 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i clientSlices = clientLease.client(); slice = trimPathSlice( new PathPrefixStripSlice( - new com.artipie.http.slice.BrowsableSlice( - new TimeoutSlice( - new ComposerProxy( - clientSlices, - cfg, - settings.artifactMetadata().flatMap(queues -> queues.proxyEventQueues(cfg)), - this.cooldown - ), - settings.httpClientSettings().proxyTimeout() + new TimeoutSlice( + new ComposerProxy( + clientSlices, + cfg, + settings.artifactMetadata().flatMap(queues -> queues.proxyEventQueues(cfg)), + this.cooldown ), - cfg.storage() + settings.httpClientSettings().proxyTimeout() ), "direct-dists" ) @@ -431,58 +434,20 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i break; case "nuget": slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new NuGet( - cfg.url(), new com.artipie.nuget.AstoRepository(cfg.storage()), - securityPolicy(), authentication(), tokens.auth(), cfg.name(), artifactEvents() - ), - cfg.storage() + new NuGet( + cfg.url(), new com.artipie.nuget.AstoRepository(cfg.storage()), + securityPolicy(), authentication(), tokens.auth(), cfg.name(), artifactEvents() ) ); break; case "gradle": - // Use streaming browsing for fast directory listings - slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new GradleSlice(cfg.storage(), securityPolicy(), - authentication(), cfg.name(), artifactEvents()), - cfg.storage() - ) - ); - break; - case "gradle-proxy": - clientLease = jettyClientSlices(cfg); - clientSlices = clientLease.client(); - final Slice gradleProxySlice = new CombinedAuthzSliceWrap( - new TimeoutSlice( - new GradleProxy( - clientSlices, - cfg, - settings.artifactMetadata().flatMap(queues -> queues.proxyEventQueues(cfg)), - this.cooldown - ), - settings.httpClientSettings().proxyTimeout() - ), - authentication(), - tokens.auth(), - new OperationControl( - securityPolicy(), - new AdapterBasicPermission(cfg.name(), Action.Standard.READ) - ) - ); - // Browsing disabled for proxy repos - files are fetched on-demand from upstream - slice = trimPathSlice(gradleProxySlice); - break; case "maven": - // Use streaming browsing for fast directory listings slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new MavenSlice(cfg.storage(), securityPolicy(), - authentication(), tokens.auth(), cfg.name(), artifactEvents()), - cfg.storage() - ) + new MavenSlice(cfg.storage(), securityPolicy(), + authentication(), tokens.auth(), cfg.name(), artifactEvents()) ); break; + case "gradle-proxy": case "maven-proxy": clientLease = jettyClientSlices(cfg); clientSlices = clientLease.client(); @@ -508,18 +473,14 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i slice = trimPathSlice(mavenProxySlice); break; case "go": - // Use streaming browsing for fast directory listings slice = trimPathSlice( - new com.artipie.http.slice.BrowsableSlice( - new GoSlice( - cfg.storage(), - securityPolicy(), - authentication(), - tokens.auth(), - cfg.name(), - artifactEvents() - ), - cfg.storage() + new GoSlice( + cfg.storage(), + securityPolicy(), + authentication(), + tokens.auth(), + cfg.name(), + artifactEvents() ) ); break; @@ -604,7 +565,11 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i case "npm-group": final Slice npmGroupSlice = new GroupSlice( this::slice, cfg.name(), cfg.members(), port, depth, - cfg.groupMemberTimeout().orElse(120L) + cfg.groupMemberTimeout().orElse(120L), + java.util.Collections.emptyList(), + Optional.of(this.settings.artifactIndex()), + proxyMembers(cfg.members()), + "npm-group" ); // Create audit slice that aggregates results from ALL members // This is critical for vulnerability scanning - local repos return {}, @@ -661,9 +626,22 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i break; case "file-group": case "php-group": + final GroupSlice composerDelegate = new GroupSlice( + this::slice, cfg.name(), cfg.members(), port, depth, + cfg.groupMemberTimeout().orElse(120L), + java.util.Collections.emptyList(), + Optional.of(this.settings.artifactIndex()), + proxyMembers(cfg.members()), + cfg.type() + ); slice = trimPathSlice( new CombinedAuthzSliceWrap( - new ComposerGroupSlice(this::slice, cfg.name(), cfg.members(), port), + new ComposerGroupSlice( + composerDelegate, + this::slice, cfg.name(), cfg.members(), port, + this.settings.prefixes().prefixes().stream() + .findFirst().orElse("") + ), authentication(), tokens.auth(), new OperationControl( @@ -677,7 +655,11 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i // Maven groups need special metadata merging final GroupSlice mavenDelegate = new GroupSlice( this::slice, cfg.name(), cfg.members(), port, depth, - cfg.groupMemberTimeout().orElse(120L) + cfg.groupMemberTimeout().orElse(120L), + java.util.Collections.emptyList(), + Optional.of(this.settings.artifactIndex()), + proxyMembers(cfg.members()), + "maven-group" ); slice = trimPathSlice( new CombinedAuthzSliceWrap( @@ -707,7 +689,11 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i new CombinedAuthzSliceWrap( new GroupSlice( this::slice, cfg.name(), cfg.members(), port, depth, - cfg.groupMemberTimeout().orElse(120L) + cfg.groupMemberTimeout().orElse(120L), + java.util.Collections.emptyList(), + Optional.of(this.settings.artifactIndex()), + proxyMembers(cfg.members()), + cfg.type() ), authentication(), tokens.auth(), @@ -844,7 +830,17 @@ private SliceValue sliceFromConfig(final RepoConfig cfg, final int port, final i wrapIntoCommonSlices(slice, cfg), Optional.ofNullable(clientLease) ); - } catch (RuntimeException | Error ex) { + } catch (final Exception ex) { + if (clientLease != null) { + clientLease.close(); + } + if (ex instanceof RuntimeException) { + throw (RuntimeException) ex; + } + throw new IllegalStateException( + String.format("Failed to construct adapter slice for '%s'", cfg.name()), ex + ); + } catch (final Error ex) { if (clientLease != null) { clientLease.close(); } @@ -889,6 +885,41 @@ private static Slice trimPathSlice(final Slice original) { return new TrimPathSlice(original, RepositorySlices.PATTERN); } + /** + * Determine which group members are proxy repositories. + * A member is a proxy if its repo type ends with "-proxy". + * + * @param members Member repository names + * @return Set of member names that are proxy repositories + */ + private Set proxyMembers(final List members) { + return members.stream() + .filter(this::isProxyOrContainsProxy) + .collect(java.util.stream.Collectors.toSet()); + } + + /** + * Check if a member is a proxy repo or a group that contains proxy repos. + * Nested groups that contain proxies must be treated as proxy-like because + * their content is only indexed after being cached from upstream. + * @param name Member repository name + * @return True if proxy or group containing proxies + */ + private boolean isProxyOrContainsProxy(final String name) { + return this.repos.config(name) + .map(c -> { + final String type = c.type(); + if (type.endsWith("-proxy")) { + return true; + } + if (type.endsWith("-group")) { + return c.members().stream().anyMatch(this::isProxyOrContainsProxy); + } + return false; + }) + .orElse(false); + } + /** * Slice's cache key. @@ -984,13 +1015,19 @@ public void close() { private static final class SharedClient { private final HttpClientSettingsKey key; private final JettyClientSlices client; + private final CompletableFuture startFuture; private final AtomicInteger references = new AtomicInteger(0); private final AtomicBoolean metricsRegistered = new AtomicBoolean(false); SharedClient(final HttpClientSettingsKey key) { this.key = key; this.client = new JettyClientSlices(key.toSettings()); - this.client.start(); + // Start the Jetty client on the dedicated resolve executor to avoid + // blocking the Vert.x event loop. The start() call can take 100ms+ + // due to SSL context initialization and socket setup. + this.startFuture = CompletableFuture.runAsync( + this.client::start, RESOLVE_EXECUTOR + ); } void retain() { @@ -998,14 +1035,21 @@ void retain() { } int release() { - final int remaining = this.references.decrementAndGet(); - if (remaining < 0) { - throw new IllegalStateException("Jetty client reference count became negative"); + final int remaining = this.references.updateAndGet(current -> Math.max(0, current - 1)); + if (remaining == 0 && this.references.get() == 0) { + EcsLogger.debug("com.artipie") + .message(String.format("Jetty client reference count reached zero for settings key '%s'", this.key.metricId())) + .eventCategory("http_client") + .eventAction("client_release") + .log(); } return remaining; } JettyClientSlices client() { + // Ensure the client has finished starting before returning it. + // The actual start() runs on RESOLVE_EXECUTOR, not the calling thread. + this.startFuture.join(); return this.client; } @@ -1057,6 +1101,8 @@ private double connectionMetric(final ToIntFunction extr } void stop() { + // Wait for start to complete before stopping to avoid race conditions. + this.startFuture.join(); this.client.stop(); } } diff --git a/artipie-main/src/main/java/com/artipie/VertxMain.java b/artipie-main/src/main/java/com/artipie/VertxMain.java index cf2ebb891..9553bf81f 100644 --- a/artipie-main/src/main/java/com/artipie/VertxMain.java +++ b/artipie-main/src/main/java/com/artipie/VertxMain.java @@ -6,12 +6,15 @@ package com.artipie; import com.artipie.api.RepositoryEvents; -import com.artipie.api.RestApi; +import com.artipie.api.v1.AsyncApiVerticle; import com.artipie.asto.Key; import com.artipie.auth.JwtTokens; import com.artipie.http.BaseSlice; import com.artipie.http.MainSlice; import com.artipie.http.Slice; +import com.artipie.http.misc.ConfigDefaults; +import com.artipie.http.misc.RepoNameMeterFilter; +import com.artipie.http.misc.StorageExecutors; import com.artipie.http.slice.LoggingSlice; import com.artipie.jetty.http3.Http3Server; import com.artipie.jetty.http3.SslFactoryFromYaml; @@ -26,6 +29,8 @@ import com.artipie.settings.repo.RepoConfig; import com.artipie.http.log.EcsLogger; import com.artipie.settings.repo.Repositories; +import com.artipie.db.DbManager; +import com.artipie.db.migration.YamlToDbMigrator; import com.artipie.vertx.VertxSliceServer; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; @@ -105,6 +110,11 @@ public final class VertxMain { */ private com.artipie.settings.ConfigWatchService configWatch; + /** + * Vert.x instance - must be closed on shutdown to release event loops and worker threads. + */ + private Vertx vertx; + /** * Ctor. * @@ -126,8 +136,44 @@ public VertxMain(final Path config, final int port) { * @throws IOException In case of error reading settings. */ public int start(final int apiPort) throws IOException { - quartz = new QuartzService(); - this.settings = new SettingsFromPath(this.config).find(quartz); + // Pre-parse YAML to detect DB configuration for Quartz JDBC clustering + final com.amihaiemil.eoyaml.YamlMapping yamlContent = + com.amihaiemil.eoyaml.Yaml.createYamlInput(this.config.toFile()).readYamlMapping(); + final com.amihaiemil.eoyaml.YamlMapping meta = yamlContent.yamlMapping("meta"); + final Optional sharedDs; + if (meta != null && meta.yamlMapping("artifacts_database") != null) { + final javax.sql.DataSource ds = + new com.artipie.db.ArtifactDbFactory(meta, "artifacts").initialize(); + sharedDs = Optional.of(ds); + DbManager.migrate(ds); + // Resolve repos and security dirs from YAML config, not relative to config file. + // artipie.yml may be mounted at /etc/artipie/ while data lives at /var/artipie/. + final Path configDir = this.config.toAbsolutePath().getParent(); + final com.amihaiemil.eoyaml.YamlMapping storageYaml = meta.yamlMapping("storage"); + final Path reposDir = storageYaml != null && storageYaml.string("path") != null + ? Path.of(storageYaml.string("path")) + : configDir.resolve("repo"); + final com.amihaiemil.eoyaml.YamlMapping policyYaml = meta.yamlMapping("policy"); + final com.amihaiemil.eoyaml.YamlMapping policyStorage = + policyYaml != null ? policyYaml.yamlMapping("storage") : null; + final Path securityDir = policyStorage != null && policyStorage.string("path") != null + ? Path.of(policyStorage.string("path")) + : configDir.resolve("security"); + new YamlToDbMigrator( + ds, securityDir, reposDir, this.config.toAbsolutePath() + ).migrate(); + quartz = new QuartzService(ds); + EcsLogger.info("com.artipie") + .message("Quartz JDBC clustering enabled with shared DataSource") + .eventCategory("scheduling") + .eventAction("quartz_jdbc_init") + .eventOutcome("success") + .log(); + } else { + sharedDs = Optional.empty(); + quartz = new QuartzService(); + } + this.settings = new SettingsFromPath(this.config).find(quartz, sharedDs); // Apply logging configuration from YAML settings if (settings.logging().configured()) { settings.logging().apply(); @@ -141,10 +187,10 @@ public int start(final int apiPort) throws IOException { - final Vertx vertx = VertxMain.vertx(settings.metrics()); + this.vertx = VertxMain.vertx(settings.metrics()); final com.artipie.settings.JwtSettings jwtSettings = settings.jwtSettings(); final JWTAuth jwt = JWTAuth.create( - vertx.getDelegate(), new JWTAuthOptions().addPubSecKey( + this.vertx.getDelegate(), new JWTAuthOptions().addPubSecKey( new PubSecKeyOptions().setAlgorithm("HS256").setBuffer(jwtSettings.secret()) ) ); @@ -164,7 +210,7 @@ public int start(final int apiPort) throws IOException { } } // Listen for repository change events to refresh runtime without restart - vertx.getDelegate().eventBus().consumer( + this.vertx.getDelegate().eventBus().consumer( RepositoryEvents.ADDRESS, msg -> { try { @@ -186,7 +232,7 @@ public int start(final int apiPort) throws IOException { .log(); return; } - vertx.getDelegate().runOnContext( + VertxMain.this.vertx.getDelegate().runOnContext( nothing -> { slices.invalidateRepo(name); repos.config(name).ifPresent(cfg -> cfg.port().ifPresent( @@ -211,7 +257,7 @@ public int start(final int apiPort) throws IOException { this.listenOn( slice, prt, - vertx, + VertxMain.this.vertx, settings.metrics(), settings.httpServerRequestTimeout() ); @@ -236,7 +282,7 @@ public int start(final int apiPort) throws IOException { .log(); return; } - vertx.getDelegate().runOnContext( + VertxMain.this.vertx.getDelegate().runOnContext( nothing -> slices.invalidateRepo(name) ); } @@ -255,7 +301,7 @@ public int start(final int apiPort) throws IOException { .log(); return; } - vertx.getDelegate().runOnContext( + VertxMain.this.vertx.getDelegate().runOnContext( nothing -> { slices.invalidateRepo(name); slices.invalidateRepo(target); @@ -279,7 +325,7 @@ public int start(final int apiPort) throws IOException { final int main = this.listenOn( new MainSlice(settings, slices), this.port, - vertx, + this.vertx, settings.metrics(), settings.httpServerRequestTimeout() ); @@ -290,27 +336,27 @@ public int start(final int apiPort) throws IOException { .eventOutcome("success") .field("url.port", main) .log(); - this.startRepos(vertx, settings, repos, this.port, slices); + this.startRepos(this.vertx, settings, repos, this.port, slices); - // Deploy RestApi verticle with multiple instances for CPU scaling + // Deploy AsyncApiVerticle with multiple instances for CPU scaling // Use 2x CPU cores to handle concurrent API requests efficiently final int apiInstances = Runtime.getRuntime().availableProcessors() * 2; final DeploymentOptions deployOpts = new DeploymentOptions() .setInstances(apiInstances); - vertx.deployVerticle( - () -> new RestApi(settings, apiPort, jwt), + this.vertx.deployVerticle( + () -> new AsyncApiVerticle(settings, apiPort, jwt, sharedDs.orElse(null)), deployOpts, result -> { if (result.succeeded()) { EcsLogger.info("com.artipie.api") - .message("RestApi deployed with " + apiInstances + " instances") + .message("AsyncApiVerticle deployed with " + apiInstances + " instances") .eventCategory("api") .eventAction("api_deploy") .eventOutcome("success") .log(); } else { EcsLogger.error("com.artipie.api") - .message("Failed to deploy RestApi") + .message("Failed to deploy AsyncApiVerticle") .eventCategory("api") .eventAction("api_deploy") .eventOutcome("failure") @@ -333,13 +379,23 @@ public int start(final int apiPort) throws IOException { final String metricsPath = metricsEndpoint.get().getKey(); final long metricsCacheTtlMs = 10_000L; // 10 second cache TTL as requested final MeterRegistry metricsRegistry = BackendRegistries.getDefaultNow(); - + StorageExecutors.registerMetrics(metricsRegistry); + settings.artifactMetadata().ifPresent( + evtQueues -> io.micrometer.core.instrument.Gauge.builder( + "artipie.events.queue.size", + evtQueues.eventQueue(), + java.util.Queue::size + ).tag("type", "events") + .description("Size of the artifact events queue") + .register(metricsRegistry) + ); + final DeploymentOptions metricsOpts = new DeploymentOptions() .setWorker(true) .setWorkerPoolName("metrics-scraper") .setWorkerPoolSize(2); - vertx.deployVerticle( + this.vertx.deployVerticle( () -> new com.artipie.metrics.AsyncMetricsVerticle( metricsRegistry, metricsPort, metricsPath, metricsCacheTtlMs ), @@ -347,13 +403,12 @@ public int start(final int apiPort) throws IOException { metricsResult -> { if (metricsResult.succeeded()) { EcsLogger.info("com.artipie.metrics") - .message("AsyncMetricsVerticle deployed as worker verticle") + .message(String.format("AsyncMetricsVerticle deployed as worker verticle with cache TTL %dms", metricsCacheTtlMs)) .eventCategory("metrics") .eventAction("metrics_verticle_deploy") .eventOutcome("success") .field("destination.port", metricsPort) .field("url.path", metricsPath) - .field("cache.ttl.ms", metricsCacheTtlMs) .log(); } else { EcsLogger.error("com.artipie.metrics") @@ -401,23 +456,96 @@ public void stop() { .eventAction("server_stop") .eventOutcome("success") .log(); + // 1. Stop HTTP/3 servers + this.http3.forEach((port, server) -> { + try { + server.stop(); + EcsLogger.info("com.artipie") + .message("HTTP/3 server on port stopped") + .eventCategory("server") + .eventAction("http3_stop") + .eventOutcome("success") + .field("destination.port", port) + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie") + .message("Failed to stop HTTP/3 server") + .eventCategory("server") + .eventAction("http3_stop") + .eventOutcome("failure") + .field("destination.port", port) + .error(e) + .log(); + } + }); + // 2. Stop HTTP/1.1+2 servers this.servers.forEach(s -> { - s.stop(); - EcsLogger.info("com.artipie") - .message("Artipie's server on port was stopped") - .eventCategory("server") - .eventAction("server_stop") - .eventOutcome("success") - .field("destination.port", s.port()) - .log(); + try { + s.stop(); + EcsLogger.info("com.artipie") + .message("Artipie's server on port was stopped") + .eventCategory("server") + .eventAction("server_stop") + .eventOutcome("success") + .field("destination.port", s.port()) + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie") + .message("Failed to stop server") + .eventCategory("server") + .eventAction("server_stop") + .eventOutcome("failure") + .error(e) + .log(); + } }); + // 3. Stop QuartzService if (quartz != null) { - quartz.stop(); + try { + quartz.stop(); + } catch (final Exception e) { + EcsLogger.error("com.artipie") + .message("Failed to stop QuartzService") + .eventCategory("server") + .eventAction("quartz_stop") + .eventOutcome("failure") + .error(e) + .log(); + } } + // 4. Stop ConfigWatchService if (this.configWatch != null) { - this.configWatch.close(); + try { + this.configWatch.close(); + } catch (final Exception e) { + EcsLogger.error("com.artipie") + .message("Failed to close ConfigWatchService") + .eventCategory("server") + .eventAction("config_watch_stop") + .eventOutcome("failure") + .error(e) + .log(); + } } - // Close settings to cleanup storage resources (S3AsyncClient, etc.) + // 5. Shutdown BlockedThreadDiagnostics + try { + BlockedThreadDiagnostics.shutdownInstance(); + EcsLogger.info("com.artipie") + .message("BlockedThreadDiagnostics shut down") + .eventCategory("server") + .eventAction("diagnostics_shutdown") + .eventOutcome("success") + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie") + .message("Failed to shutdown BlockedThreadDiagnostics") + .eventCategory("server") + .eventAction("diagnostics_shutdown") + .eventOutcome("failure") + .error(e) + .log(); + } + // 6. Close settings (releases storage resources, S3AsyncClient, etc.) if (this.settings != null) { try { this.settings.close(); @@ -437,6 +565,44 @@ public void stop() { .log(); } } + // 7. Shutdown storage executor pools + try { + com.artipie.http.misc.StorageExecutors.shutdown(); + EcsLogger.info("com.artipie") + .message("Storage executor pools shut down") + .eventCategory("server") + .eventAction("executor_shutdown") + .eventOutcome("success") + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie") + .message("Failed to shutdown storage executor pools") + .eventCategory("server") + .eventAction("executor_shutdown") + .eventOutcome("failure") + .error(e) + .log(); + } + // 8. Close Vert.x instance (LAST - closes event loops and worker threads) + if (this.vertx != null) { + try { + this.vertx.close(); + EcsLogger.info("com.artipie") + .message("Vert.x instance closed") + .eventCategory("server") + .eventAction("vertx_close") + .eventOutcome("success") + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie") + .message("Failed to close Vert.x instance") + .eventCategory("server") + .eventAction("vertx_close") + .eventOutcome("failure") + .error(e) + .log(); + } + } EcsLogger.info("com.artipie") .message("Artipie shutdown complete") .eventCategory("server") @@ -693,21 +859,36 @@ private static Vertx vertx(final MetricsContext mctx) { // Add common tags to all metrics registry.config().commonTags("job", "artipie"); - // Configure registry to publish histogram buckets for all Timer metrics + // Add repo_name cardinality control filter (default max 50 distinct repos) registry.config().meterFilter( - new MeterFilter() { - @Override - public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { - if (id.getType() == Meter.Type.TIMER) { - return DistributionStatisticConfig.builder() - .percentilesHistogram(true) - .build() - .merge(config); + new RepoNameMeterFilter( + ConfigDefaults.getInt("ARTIPIE_METRICS_MAX_REPOS", 50) + ) + ); + + // Configure registry to publish histogram buckets for all Timer metrics + // Opt-in via ARTIPIE_METRICS_PERCENTILES_HISTOGRAM env var (default: false) + if (Boolean.parseBoolean( + ConfigDefaults.get("ARTIPIE_METRICS_PERCENTILES_HISTOGRAM", "false") + )) { + registry.config().meterFilter( + new MeterFilter() { + @Override + public DistributionStatisticConfig configure( + final Meter.Id id, + final DistributionStatisticConfig config + ) { + if (id.getType() == Meter.Type.TIMER) { + return DistributionStatisticConfig.builder() + .percentilesHistogram(true) + .build() + .merge(config); + } + return config; } - return config; } - } - ); + ); + } // Initialize MicrometerMetrics with the registry com.artipie.metrics.MicrometerMetrics.initialize(registry); diff --git a/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxy.java b/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxy.java index 5f42993e3..64c188762 100644 --- a/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxy.java +++ b/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxy.java @@ -32,6 +32,8 @@ import com.artipie.security.policy.Policy; import com.artipie.settings.repo.RepoConfig; +import com.artipie.http.log.EcsLogger; + import java.util.Optional; import java.util.Queue; import java.util.concurrent.CompletableFuture; @@ -73,7 +75,39 @@ public CompletableFuture response( final Headers headers, final Content body ) { - return this.delegate.response(line, headers, body); + final long start = System.currentTimeMillis(); + EcsLogger.info("com.artipie.docker.proxy") + .message("DockerProxy request") + .eventCategory("repository") + .eventAction("proxy_request") + .field("http.request.method", line.method().value()) + .field("url.path", line.uri().getPath()) + .log(); + return this.delegate.response(line, headers, body) + .whenComplete((resp, err) -> { + final long duration = System.currentTimeMillis() - start; + if (err != null) { + EcsLogger.error("com.artipie.docker.proxy") + .message("DockerProxy error") + .eventCategory("repository") + .eventAction("proxy_request") + .eventOutcome("failure") + .field("url.path", line.uri().getPath()) + .duration(duration) + .error(err) + .log(); + } else { + EcsLogger.info("com.artipie.docker.proxy") + .message("DockerProxy response") + .eventCategory("repository") + .eventAction("proxy_request") + .eventOutcome(resp.status().success() ? "success" : "failure") + .field("url.path", line.uri().getPath()) + .field("http.response.status_code", resp.status().code()) + .duration(duration) + .log(); + } + }); } /** diff --git a/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxyCooldownSlice.java b/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxyCooldownSlice.java index f22ae700d..94b50cbfe 100644 --- a/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxyCooldownSlice.java +++ b/artipie-main/src/main/java/com/artipie/adapters/docker/DockerProxyCooldownSlice.java @@ -84,7 +84,11 @@ public CompletableFuture response( final ManifestRequest request; try { request = ManifestRequest.from(line); - } catch (final IllegalArgumentException ignored) { + } catch (final IllegalArgumentException ex) { + EcsLogger.debug("com.artipie.docker") + .message("Failed to parse manifest request, falling through to origin") + .error(ex) + .log(); return this.origin.response(line, headers, body); } return this.origin.response(line, headers, body) @@ -362,7 +366,11 @@ private Optional release(final Headers headers) { .flatMap(value -> { try { return Optional.of(Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(value))); - } catch (final DateTimeParseException ignored) { + } catch (final DateTimeParseException ex) { + EcsLogger.debug("com.artipie.docker") + .message("Failed to parse date header for release time") + .error(ex) + .log(); return Optional.empty(); } }); diff --git a/artipie-main/src/main/java/com/artipie/adapters/gradle/GradleProxy.java b/artipie-main/src/main/java/com/artipie/adapters/gradle/GradleProxy.java deleted file mode 100644 index 34901468c..000000000 --- a/artipie-main/src/main/java/com/artipie/adapters/gradle/GradleProxy.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com - * https://github.com/artipie/artipie/blob/master/LICENSE.txt - */ -package com.artipie.adapters.gradle; - -import com.artipie.asto.Storage; -import com.artipie.asto.cache.Cache; -import com.artipie.asto.cache.FromStorageCache; -import com.artipie.asto.Content; -import com.artipie.cooldown.CooldownService; -import com.artipie.gradle.http.GradleProxySlice; -import com.artipie.http.Headers; -import com.artipie.http.Response; -import com.artipie.http.Slice; -import com.artipie.http.client.jetty.JettyClientSlices; -import com.artipie.http.client.auth.Authenticator; -import com.artipie.http.client.auth.GenericAuthenticator; -import com.artipie.http.group.GroupSlice; -import com.artipie.http.rq.RequestLine; -import com.artipie.scheduling.ProxyArtifactEvent; -import com.artipie.settings.repo.RepoConfig; - -import java.net.URI; -import java.util.Optional; -import java.util.Queue; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -/** - * Gradle proxy adapter with maven-proxy feature parity. - * Supports multiple remotes, authentication, priority ordering, and failover. - * - * @since 1.0 - */ -public final class GradleProxy implements Slice { - - /** - * Underlying slice implementation. - */ - private final Slice slice; - - /** - * Ctor. - * - * @param client HTTP client - * @param cfg Repository configuration - * @param queue Proxy artifact events queue - * @param cooldown Cooldown service - */ - public GradleProxy( - final JettyClientSlices client, - final RepoConfig cfg, - final Optional> queue, - final CooldownService cooldown - ) { - final Optional asto = cfg.storageOpt(); - - // Support multiple remotes with GroupSlice (like maven-proxy) - // Each remote gets its own GradleProxySlice, evaluated in priority order - this.slice = new GroupSlice( - cfg.remotes().stream().map( - remote -> new GradleProxySlice( - client, - remote.uri(), - // Support per-remote authentication (like maven-proxy) - GenericAuthenticator.create(client, remote.username(), remote.pwd()), - asto.map(FromStorageCache::new).orElse(Cache.NOP), - asto.flatMap(ignored -> queue), - cfg.name(), - cfg.type(), - cooldown, - asto // Pass storage for checksum persistence - ) - ).collect(Collectors.toList()) - ); - } - - @Override - public CompletableFuture response( - final RequestLine line, - final Headers headers, - final Content body - ) { - return this.slice.response(line, headers, body); - } -} diff --git a/artipie-main/src/main/java/com/artipie/adapters/npm/NpmProxyAdapter.java b/artipie-main/src/main/java/com/artipie/adapters/npm/NpmProxyAdapter.java index 274a1557b..20b25a481 100644 --- a/artipie-main/src/main/java/com/artipie/adapters/npm/NpmProxyAdapter.java +++ b/artipie-main/src/main/java/com/artipie/adapters/npm/NpmProxyAdapter.java @@ -24,7 +24,6 @@ import com.artipie.settings.repo.RepoConfig; import java.net.URL; -import java.time.Duration; import java.util.Optional; import java.util.Queue; import java.util.concurrent.CompletableFuture; @@ -100,11 +99,9 @@ public NpmProxyAdapter( return new CachedNpmProxySlice( npmProxySlice, asto, - Duration.ofHours(24), // 404 cache TTL - true, // negative caching enabled - cfg.name(), // repo name for cache isolation - remote.uri().toString(),// upstream URL for metrics - cfg.type() // repository type + cfg.name(), + remote.uri().toString(), + cfg.type() ); } ).collect(Collectors.toList()) diff --git a/artipie-main/src/main/java/com/artipie/adapters/php/ComposerGroupSlice.java b/artipie-main/src/main/java/com/artipie/adapters/php/ComposerGroupSlice.java index 50808ba13..0d15707cc 100644 --- a/artipie-main/src/main/java/com/artipie/adapters/php/ComposerGroupSlice.java +++ b/artipie-main/src/main/java/com/artipie/adapters/php/ComposerGroupSlice.java @@ -36,6 +36,13 @@ */ public final class ComposerGroupSlice implements Slice { + /** + * Delegate group slice for non-packages.json requests. + * Uses the standard GroupSlice with artifact index, proxy awareness, + * circuit breaker, and error handling. + */ + private final Slice delegate; + /** * Slice resolver for getting member slices. */ @@ -57,23 +64,40 @@ public final class ComposerGroupSlice implements Slice { private final int port; /** - * Constructor. + * Base path for metadata-url (e.g. "/test_prefix/php_group"). + * Built from global prefix + group name so Composer can resolve + * p2 URLs as host-absolute paths. + */ + private final String basePath; + + /** + * Constructor with delegate slice for standard group behavior. * + * @param delegate Delegate group slice (GroupSlice with index/proxy support) * @param resolver Slice resolver * @param group Group repository name * @param members List of member repository names * @param port Server port + * @param globalPrefix Global URL prefix (e.g. "test_prefix"), empty string if none */ public ComposerGroupSlice( + final Slice delegate, final SliceResolver resolver, final String group, final List members, - final int port + final int port, + final String globalPrefix ) { + this.delegate = delegate; this.resolver = resolver; this.group = group; this.members = members; this.port = port; + if (globalPrefix != null && !globalPrefix.isEmpty()) { + this.basePath = "/" + globalPrefix + "/" + group; + } else { + this.basePath = "/" + group; + } } @Override @@ -88,57 +112,60 @@ public CompletableFuture response( } final String path = line.uri().getPath(); - + // For packages.json, merge responses from all members if (path.endsWith("/packages.json") || path.equals("/packages.json")) { - EcsLogger.debug("com.artipie.composer") - .message("Merging packages.json from " + this.members.size() + " members") - .eventCategory("repository") - .eventAction("packages_merge") - .field("repository.name", this.group) - .log(); - // Get original path before any routing rewrites - // Priority: X-Original-Path (from ApiRoutingSlice) > X-FullPath (from TrimPathSlice) > current path - final String originalPath = headers.find("X-Original-Path").stream() - .findFirst() - .map(h -> h.getValue()) - .or(() -> headers.find("X-FullPath").stream() - .findFirst() - .map(h -> h.getValue()) - ) - .orElse(path); - EcsLogger.debug("com.artipie.composer") - .message("Path resolution for packages.json") - .eventCategory("repository") - .eventAction("path_resolve") - .field("url.path", path) - .field("url.original", originalPath) - .field("http.request.headers.X-FullPath", headers.find("X-FullPath").stream().findFirst().map(h -> h.getValue()).orElse("none")) - .field("http.request.headers.X-Original-Path", headers.find("X-Original-Path").stream().findFirst().map(h -> h.getValue()).orElse("none")) - .log(); - // Extract base path for metadata-url (everything before /packages.json) - final String basePath = extractBasePath(originalPath); - EcsLogger.debug("com.artipie.composer") - .message("Base path for metadata-url") - .eventCategory("repository") - .eventAction("path_resolve") - .field("url.path", basePath) - .log(); - return mergePackagesJson(line, headers, body, basePath); + return mergePackagesJson(line, headers, body); } - // For other requests (individual packages), try members sequentially - EcsLogger.debug("com.artipie.composer") - .message("Trying members for request") - .eventCategory("repository") - .eventAction("member_query") - .field("repository.name", this.group) - .field("url.path", path) - .log(); - // CRITICAL: Consume body once before sequential member queries - return body.asBytesFuture().thenCompose(requestBytes -> - tryMembersSequentially(0, line, headers) - ); + // For p2 metadata requests, try each member directly. + // The artifact index cannot match p2 paths (it stores package names, + // not filesystem paths), so the delegate GroupSlice would skip local + // members and return 404. + if (path.contains("/p2/")) { + return tryMembersForP2(line, headers, body); + } + + // For other requests (tarballs, artifacts), delegate to GroupSlice + // which has artifact index, proxy awareness, circuit breaker, and error handling + return this.delegate.response(line, headers, body); + } + + /** + * Try each member sequentially for p2 metadata requests. + * Returns the first successful response, or 404 if all members fail. + */ + private CompletableFuture tryMembersForP2( + final RequestLine line, + final Headers headers, + final Content body + ) { + return body.asBytesFuture().thenCompose(requestBytes -> { + CompletableFuture chain = CompletableFuture.completedFuture( + ResponseBuilder.notFound().build() + ); + for (final String member : this.members) { + chain = chain.thenCompose(prev -> { + if (prev.status() == RsStatus.OK) { + return CompletableFuture.completedFuture(prev); + } + final Slice memberSlice = this.resolver.slice( + new Key.From(member), this.port, 0 + ); + final RequestLine rewritten = rewritePath(line, member); + final Headers sanitized = dropFullPathHeader(headers); + return memberSlice.response(rewritten, sanitized, Content.EMPTY) + .thenApply(resp -> { + if (resp.status() == RsStatus.OK) { + return resp; + } + return prev; + }) + .exceptionally(ex -> prev); + }); + } + return chain; + }); } /** @@ -147,14 +174,12 @@ public CompletableFuture response( * @param line Request line * @param headers Headers * @param body Body - * @param basePath Base path for metadata-url (e.g., "/test_prefix/api/composer/php_group" or "/php_group") * @return Merged response */ private CompletableFuture mergePackagesJson( final RequestLine line, final Headers headers, - final Content body, - final String basePath + final Content body ) { // CRITICAL: Consume original body to prevent OneTimePublisher errors // GET requests have empty bodies, but Content is still reference-counted @@ -312,7 +337,7 @@ private CompletableFuture mergePackagesJson( if (hasSatisFormat) { // Use Satis format for group merged.add("packages", Json.createObjectBuilder()); // Empty object - merged.add("providers-url", basePath + "/p2/%package%.json"); + merged.add("providers-url", this.basePath + "/p2/%package%.json"); merged.add("providers", providersBuilder.build()); EcsLogger.debug("com.artipie.composer") .message("Using Satis format for group (" + providersBuilder.build().size() + " providers)") @@ -322,8 +347,9 @@ private CompletableFuture mergePackagesJson( .field("repository.name", this.group) .log(); } else { - // Use traditional format - merged.add("metadata-url", basePath + "/p2/%package%.json"); + // Use host-absolute metadata-url including global prefix. + // Composer (especially v1) needs absolute paths, not relative. + merged.add("metadata-url", this.basePath + "/p2/%package%.json"); merged.add("packages", packagesBuilder.build()); EcsLogger.debug("com.artipie.composer") .message("Using traditional format for group (" + packagesBuilder.build().size() + " packages)") @@ -347,66 +373,6 @@ private CompletableFuture mergePackagesJson( }); // Close thenCompose lambda for body consumption } - /** - * Try members sequentially until one returns a non-404 response. - * Body has already been consumed by caller. - * - * @param index Current member index - * @param line Request line - * @param headers Headers - * @return Response from first successful member or 404 - */ - private CompletableFuture tryMembersSequentially( - final int index, - final RequestLine line, - final Headers headers - ) { - if (index >= this.members.size()) { - EcsLogger.debug("com.artipie.composer") - .message("No member in group could serve request") - .eventCategory("repository") - .eventAction("member_query") - .eventOutcome("failure") - .field("repository.name", this.group) - .field("url.path", line.uri().getPath()) - .log(); - return ResponseBuilder.notFound().completedFuture(); - } - - final String member = this.members.get(index); - final Slice memberSlice = this.resolver.slice(new Key.From(member), this.port, 0); - final RequestLine rewritten = rewritePath(line, member); - final Headers sanitized = dropFullPathHeader(headers); - - EcsLogger.debug("com.artipie.composer") - .message("Trying member for request") - .eventCategory("repository") - .eventAction("member_query") - .field("repository.name", this.group) - .field("member.name", member) - .field("url.path", line.uri().getPath()) - .log(); - - return memberSlice.response(rewritten, sanitized, Content.EMPTY) - .thenCompose(resp -> { - EcsLogger.debug("com.artipie.composer") - .message("Member responded") - .eventCategory("repository") - .eventAction("member_query") - .field("member.name", member) - .field("http.response.status_code", resp.status().code()) - .field("url.path", line.uri().getPath()) - .log(); - - if (resp.status() == RsStatus.NOT_FOUND) { - // Try next member - return tryMembersSequentially(index + 1, line, sanitized); - } - - // Return this response (success or error) - return CompletableFuture.completedFuture(resp); - }); - } /** * Rewrite request line to include member repository name in path. @@ -447,24 +413,4 @@ private static Headers dropFullPathHeader(final Headers headers) { ); } - /** - * Extract base path from packages.json request path. - * Examples: - * - "/packages.json" -> "" - * - "/php_group/packages.json" -> "/php_group" - * - "/test_prefix/api/composer/php_group/packages.json" -> "/test_prefix/api/composer/php_group" - * - * @param path Full request path - * @return Base path (without /packages.json suffix) - */ - private static String extractBasePath(final String path) { - if (path.endsWith("/packages.json")) { - return path.substring(0, path.length() - "/packages.json".length()); - } - if (path.equals("/packages.json")) { - return ""; - } - // Fallback: return path as-is - return path; - } } diff --git a/artipie-main/src/main/java/com/artipie/api/AuthTokenRest.java b/artipie-main/src/main/java/com/artipie/api/AuthTokenRest.java index 80b17cc75..1588e1e55 100644 --- a/artipie-main/src/main/java/com/artipie/api/AuthTokenRest.java +++ b/artipie-main/src/main/java/com/artipie/api/AuthTokenRest.java @@ -90,7 +90,7 @@ private void getJwtToken(final RoutingContext routing) { new JsonObject().put("token", token).encode() ); } else { - routing.response().setStatusCode(HttpStatus.UNAUTHORIZED_401).send(); + sendError(routing, HttpStatus.UNAUTHORIZED_401, "Invalid credentials"); } } else { routing.fail(ar.cause()); diff --git a/artipie-main/src/main/java/com/artipie/api/AuthzHandler.java b/artipie-main/src/main/java/com/artipie/api/AuthzHandler.java index ae6a6a607..4bcfec6ce 100644 --- a/artipie-main/src/main/java/com/artipie/api/AuthzHandler.java +++ b/artipie-main/src/main/java/com/artipie/api/AuthzHandler.java @@ -7,6 +7,7 @@ import com.artipie.http.auth.AuthUser; import com.artipie.security.policy.Policy; import io.vertx.core.Handler; +import io.vertx.core.json.JsonObject; import io.vertx.ext.auth.User; import io.vertx.ext.web.RoutingContext; import java.security.Permission; @@ -51,7 +52,13 @@ public void handle(final RoutingContext context) { ).implies(this.perm)) { context.next(); } else { - context.response().setStatusCode(HttpStatus.SC_FORBIDDEN).end(); + context.response() + .setStatusCode(HttpStatus.SC_FORBIDDEN) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.SC_FORBIDDEN) + .put("message", "Access denied: insufficient permissions") + .encode()); } } } diff --git a/artipie-main/src/main/java/com/artipie/api/BaseRest.java b/artipie-main/src/main/java/com/artipie/api/BaseRest.java index dada4f03d..1d2115680 100644 --- a/artipie-main/src/main/java/com/artipie/api/BaseRest.java +++ b/artipie-main/src/main/java/com/artipie/api/BaseRest.java @@ -65,9 +65,12 @@ protected Handler errorHandler(final int code) { // Sanitize message - HTTP status messages can't contain control chars final String msg = sanitizeStatusMessage(context.failure().getMessage()); context.response() - .setStatusMessage(msg) .setStatusCode(status) - .end(); + .putHeader("Content-Type", "application/json") + .end(new io.vertx.core.json.JsonObject() + .put("code", status) + .put("message", msg) + .encode()); EcsLogger.warn("com.artipie.api") .message("REST API request failed") .eventCategory("api") @@ -104,6 +107,23 @@ private static String sanitizeStatusMessage(final String message) { return sanitized; } + /** + * Send a JSON error response with standard {code, message} format. + * @param context Routing context + * @param status HTTP status code + * @param message Error message + */ + protected static void sendError(final RoutingContext context, + final int status, final String message) { + context.response() + .setStatusCode(status) + .putHeader("Content-Type", "application/json") + .end(new io.vertx.core.json.JsonObject() + .put("code", status) + .put("message", message) + .encode()); + } + /** * Read body as JsonObject. * @param context RoutingContext diff --git a/artipie-main/src/main/java/com/artipie/api/CacheRest.java b/artipie-main/src/main/java/com/artipie/api/CacheRest.java deleted file mode 100644 index 9d990aab4..000000000 --- a/artipie-main/src/main/java/com/artipie/api/CacheRest.java +++ /dev/null @@ -1,367 +0,0 @@ -/* - * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com - * https://github.com/artipie/artipie/blob/master/LICENSE.txt - */ -package com.artipie.api; - -import com.artipie.group.GroupNegativeCache; -import com.artipie.http.log.EcsLogger; -import com.artipie.security.policy.Policy; -import com.artipie.api.perms.ApiCachePermission; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.RoutingContext; -import io.vertx.ext.web.openapi.RouterBuilder; -import org.eclipse.jetty.http.HttpStatus; - -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.Optional; - -/** - * REST API for cache management operations. - * Provides endpoints to invalidate negative caches without process restart. - * - *

Endpoints:

- *
    - *
  • GET /api/health - Health check endpoint
    • - *
  • GET /api/cache/negative/groups - List all registered groups
    • - *
  • GET /api/cache/negative/group/{groupName}/stats - Get cache stats for a group
    • - *
  • DELETE /api/cache/negative/group/{groupName} - Clear all negative cache for a group
    • - *
  • DELETE /api/cache/negative/group/{groupName}/package - Invalidate specific package
    • - *
  • DELETE /api/cache/negative/package - Invalidate package in ALL groups
    • - *
- * - * @since 1.0 - */ -public final class CacheRest extends BaseRest { - - /** - * Artipie policy. - */ - private final Policy policy; - - /** - * Ctor. - * @param policy Artipie policy for authorization - */ - public CacheRest(final Policy policy) { - this.policy = policy; - } - - @Override - public void init(final RouterBuilder rbr) { - // Health check (no auth required) - rbr.operation("healthCheck") - .handler(this::healthCheck) - .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); - - // List all registered groups - rbr.operation("listCacheGroups") - .handler(new AuthzHandler(this.policy, ApiCachePermission.READ)) - .handler(this::listGroups) - .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); - - // Get stats for a specific group - rbr.operation("getCacheStats") - .handler(new AuthzHandler(this.policy, ApiCachePermission.READ)) - .handler(this::groupStats) - .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); - - // Clear all negative cache for a group - rbr.operation("clearGroupCache") - .handler(new AuthzHandler(this.policy, ApiCachePermission.WRITE)) - .handler(this::clearGroup) - .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); - - // Invalidate specific package in a group - rbr.operation("invalidatePackageInGroup") - .handler(new AuthzHandler(this.policy, ApiCachePermission.WRITE)) - .handler(this::invalidatePackageInGroup) - .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); - - // Invalidate package in ALL groups - rbr.operation("invalidatePackageGlobally") - .handler(new AuthzHandler(this.policy, ApiCachePermission.WRITE)) - .handler(this::invalidatePackageGlobally) - .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); - } - - /** - * Initialize cache API routes directly on a router without OpenAPI validation. - * This is used to avoid breaking the main RouterBuilder async chain. - * @param router The router to add routes to - */ - public void initDirect(final io.vertx.ext.web.Router router) { - // List all registered groups - router.get("/api/v1/cache/negative/groups") - .handler(new AuthzHandler(this.policy, ApiCachePermission.READ)) - .handler(this::listGroups); - - // Get stats for a specific group - router.get("/api/v1/cache/negative/group/:groupName/stats") - .handler(new AuthzHandler(this.policy, ApiCachePermission.READ)) - .handler(this::groupStats); - - // Clear all negative cache for a group - router.delete("/api/v1/cache/negative/group/:groupName") - .handler(new AuthzHandler(this.policy, ApiCachePermission.WRITE)) - .handler(this::clearGroup); - - // Invalidate specific package in a group - router.delete("/api/v1/cache/negative/group/:groupName/package") - .handler(new AuthzHandler(this.policy, ApiCachePermission.WRITE)) - .handler(this::invalidatePackageInGroup); - - // Invalidate package in ALL groups - router.delete("/api/v1/cache/negative/package") - .handler(new AuthzHandler(this.policy, ApiCachePermission.WRITE)) - .handler(this::invalidatePackageGlobally); - } - - /** - * Health check endpoint. - * GET /api/health - */ - private void healthCheck(final RoutingContext ctx) { - ctx.response() - .setStatusCode(200) - .putHeader("Content-Type", "application/json") - .end("{\"status\":\"ok\"}"); - } - - /** - * List all registered group names. - * GET /api/cache/negative/groups - */ - private void listGroups(final RoutingContext ctx) { - final List groups = GroupNegativeCache.registeredGroups(); - - final JsonObject response = new JsonObject() - .put("groups", new JsonArray(groups)) - .put("count", groups.size()); - - ctx.response() - .setStatusCode(200) - .putHeader("Content-Type", "application/json") - .end(response.encode()); - } - - /** - * Get cache stats for a specific group. - * GET /api/cache/negative/group/{groupName}/stats - */ - private void groupStats(final RoutingContext ctx) { - final String groupName = ctx.pathParam("groupName"); - - final var instance = GroupNegativeCache.getInstance(groupName); - if (instance.isEmpty()) { - ctx.response() - .setStatusCode(404) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("error", "Group not found in cache registry") - .put("group", groupName) - .encode()); - return; - } - - final GroupNegativeCache cache = instance.get(); - final JsonObject response = new JsonObject() - .put("group", groupName) - .put("l1Size", cache.size()) - .put("twoTier", cache.isTwoTier()); - - ctx.response() - .setStatusCode(200) - .putHeader("Content-Type", "application/json") - .end(response.encode()); - } - - /** - * Clear all negative cache entries for a group. - * DELETE /api/cache/negative/group/{groupName} - */ - private void clearGroup(final RoutingContext ctx) { - final String groupName = ctx.pathParam("groupName"); - - EcsLogger.info("com.artipie.api") - .message("Clearing negative cache for group") - .eventCategory("cache") - .eventAction("clear_group") - .field("group.name", groupName) - .log(); - - GroupNegativeCache.clearGroup(groupName) - .whenComplete((v, err) -> { - if (err != null) { - EcsLogger.error("com.artipie.api") - .message("Failed to clear negative cache") - .eventCategory("cache") - .eventAction("clear_group") - .eventOutcome("failure") - .field("group.name", groupName) - .error(err) - .log(); - - ctx.response() - .setStatusCode(500) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("error", "Failed to clear cache") - .put("message", err.getMessage()) - .encode()); - } else { - ctx.response() - .setStatusCode(200) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("status", "cleared") - .put("group", groupName) - .encode()); - } - }); - } - - /** - * Invalidate specific package in a group. - * DELETE /api/cache/negative/group/{groupName}/package - * Body: {"path": "@scope/package-name"} - */ - private void invalidatePackageInGroup(final RoutingContext ctx) { - final String groupName = ctx.pathParam("groupName"); - final Optional packagePath = this.readPackagePath(ctx); - if (packagePath.isEmpty()) { - return; - } - - // URL decode the package path (in case it's encoded) - final String decodedPath = URLDecoder.decode(packagePath.get(), StandardCharsets.UTF_8); - - EcsLogger.info("com.artipie.api") - .message("Invalidating negative cache for package in group") - .eventCategory("cache") - .eventAction("invalidate_package") - .field("group.name", groupName) - .field("package.name", decodedPath) - .log(); - - GroupNegativeCache.invalidatePackageInGroup(groupName, decodedPath) - .whenComplete((v, err) -> { - if (err != null) { - EcsLogger.error("com.artipie.api") - .message("Failed to invalidate package cache") - .eventCategory("cache") - .eventAction("invalidate_package") - .eventOutcome("failure") - .field("group.name", groupName) - .field("package.name", decodedPath) - .error(err) - .log(); - - ctx.response() - .setStatusCode(500) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("error", "Failed to invalidate cache") - .put("message", err.getMessage()) - .encode()); - } else { - ctx.response() - .setStatusCode(200) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("status", "invalidated") - .put("group", groupName) - .put("package", decodedPath) - .encode()); - } - }); - } - - /** - * Invalidate package in ALL groups. - * DELETE /api/cache/negative/package - * Body: {"path": "@scope/package-name"} - */ - private void invalidatePackageGlobally(final RoutingContext ctx) { - final Optional packagePath = this.readPackagePath(ctx); - if (packagePath.isEmpty()) { - return; - } - - // URL decode the package path (in case it's encoded) - final String decodedPath = URLDecoder.decode(packagePath.get(), StandardCharsets.UTF_8); - - EcsLogger.info("com.artipie.api") - .message("Invalidating negative cache for package globally") - .eventCategory("cache") - .eventAction("invalidate_package_global") - .field("package.name", decodedPath) - .log(); - - GroupNegativeCache.invalidatePackageGlobally(decodedPath) - .whenComplete((v, err) -> { - if (err != null) { - EcsLogger.error("com.artipie.api") - .message("Failed to invalidate package cache globally") - .eventCategory("cache") - .eventAction("invalidate_package_global") - .eventOutcome("failure") - .field("package.name", decodedPath) - .error(err) - .log(); - - ctx.response() - .setStatusCode(500) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("error", "Failed to invalidate cache") - .put("message", err.getMessage()) - .encode()); - } else { - final List groups = GroupNegativeCache.registeredGroups(); - ctx.response() - .setStatusCode(200) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("status", "invalidated") - .put("package", decodedPath) - .put("groupsAffected", new JsonArray(groups)) - .encode()); - } - }); - } - - private Optional readPackagePath(final RoutingContext ctx) { - final JsonObject body; - try { - body = ctx.body().asJsonObject(); - } catch (final Exception e) { - ctx.response() - .setStatusCode(400) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("error", "Invalid JSON body") - .put("example", "{\"path\": \"@scope/package-name\"}") - .encode()); - return Optional.empty(); - } - - final String packagePath = body != null ? body.getString("path") : null; - if (packagePath == null || packagePath.isBlank()) { - ctx.response() - .setStatusCode(400) - .putHeader("Content-Type", "application/json") - .end(new JsonObject() - .put("error", "Missing 'path' in request body") - .put("example", "{\"path\": \"@scope/package-name\"}") - .encode()); - return Optional.empty(); - } - return Optional.of(packagePath); - } -} diff --git a/artipie-main/src/main/java/com/artipie/api/RepositoryRest.java b/artipie-main/src/main/java/com/artipie/api/RepositoryRest.java index 743492e9e..c3113e510 100644 --- a/artipie-main/src/main/java/com/artipie/api/RepositoryRest.java +++ b/artipie-main/src/main/java/com/artipie/api/RepositoryRest.java @@ -119,8 +119,7 @@ private void deletePackageFolder(final RoutingContext context) { final JsonObject body = BaseRest.readJsonObject(context); final String path = body == null ? null : body.getString("path", "").trim(); if (path == null || path.isEmpty()) { - context.response().setStatusCode(HttpStatus.BAD_REQUEST_400) - .end("path is required"); + sendError(context, HttpStatus.BAD_REQUEST_400, "path is required"); return; } final String actor = context.user().principal().getString(AuthTokenRest.SUB); @@ -137,8 +136,7 @@ private void deletePackageFolder(final RoutingContext context) { .userName(actor) .error(error) .log(); - context.response().setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) - .end(error.getMessage()); + sendError(context, HttpStatus.INTERNAL_SERVER_ERROR_500, error.getMessage()); } else if (deleted) { EcsLogger.info("com.artipie.api") .message("Package folder deleted via API") @@ -151,8 +149,7 @@ private void deletePackageFolder(final RoutingContext context) { .log(); context.response().setStatusCode(HttpStatus.NO_CONTENT_204).end(); } else { - context.response().setStatusCode(HttpStatus.NOT_FOUND_404) - .end("Package folder not found: " + path); + sendError(context, HttpStatus.NOT_FOUND_404, "Package folder not found: " + path); } }); } @@ -339,7 +336,7 @@ private void createOrUpdateRepo(final RoutingContext context) { context.response().setStatusCode(HttpStatus.OK_200).end(); } } else { - context.response().setStatusCode(HttpStatus.FORBIDDEN_403).end(); + sendError(context, HttpStatus.FORBIDDEN_403, "Insufficient permissions"); } } @@ -379,21 +376,19 @@ private void unblockCooldown(final RoutingContext context) { final RepositoryName name = new RepositoryName.FromRequest(context); final Optional repo = this.repositoryConfig(name); if (repo.isEmpty()) { - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "Repository not found"); return; } final String type = repo.get().getString("type", "").trim(); if (type.isEmpty()) { - context.response().setStatusCode(HttpStatus.BAD_REQUEST_400) - .end("Repository type is required"); + sendError(context, HttpStatus.BAD_REQUEST_400, "Repository type is required"); return; } final JsonObject body = BaseRest.readJsonObject(context); final String artifact = body.getString("artifact", "").trim(); final String version = body.getString("version", "").trim(); if (artifact.isEmpty() || version.isEmpty()) { - context.response().setStatusCode(HttpStatus.BAD_REQUEST_400) - .end("artifact and version are required"); + sendError(context, HttpStatus.BAD_REQUEST_400, "artifact and version are required"); return; } final String actor = context.user().principal().getString(AuthTokenRest.SUB); @@ -412,8 +407,7 @@ private void unblockCooldown(final RoutingContext context) { .field("package.version", version) .error(error) .log(); - context.response().setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) - .end(error.getMessage()); + sendError(context, HttpStatus.INTERNAL_SERVER_ERROR_500, error.getMessage()); } }); } @@ -436,8 +430,7 @@ private void deleteArtifact(final RoutingContext context) { final JsonObject body = BaseRest.readJsonObject(context); final String path = body == null ? null : body.getString("path", "").trim(); if (path == null || path.isEmpty()) { - context.response().setStatusCode(HttpStatus.BAD_REQUEST_400) - .end("path is required"); + sendError(context, HttpStatus.BAD_REQUEST_400, "path is required"); return; } final String actor = context.user().principal().getString(AuthTokenRest.SUB); @@ -450,12 +443,11 @@ private void deleteArtifact(final RoutingContext context) { .eventAction("artifact_delete") .eventOutcome("failure") .field("repository.name", rname.toString()) - .field("artifact.path", path) + .field("file.path", path) .userName(actor) .error(error) .log(); - context.response().setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) - .end(error.getMessage()); + sendError(context, HttpStatus.INTERNAL_SERVER_ERROR_500, error.getMessage()); } else if (deleted) { EcsLogger.info("com.artipie.api") .message("Artifact deleted via API") @@ -463,13 +455,12 @@ private void deleteArtifact(final RoutingContext context) { .eventAction("artifact_delete") .eventOutcome("success") .field("repository.name", rname.toString()) - .field("artifact.path", path) + .field("file.path", path) .userName(actor) .log(); context.response().setStatusCode(HttpStatus.NO_CONTENT_204).end(); } else { - context.response().setStatusCode(HttpStatus.NOT_FOUND_404) - .end("Artifact not found: " + path); + sendError(context, HttpStatus.NOT_FOUND_404, "Artifact not found: " + path); } }); } @@ -480,13 +471,12 @@ private void unblockAllCooldown(final RoutingContext context) { final RepositoryName name = new RepositoryName.FromRequest(context); final Optional repo = this.repositoryConfig(name); if (repo.isEmpty()) { - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "Repository not found"); return; } final String type = repo.get().getString("type", "").trim(); if (type.isEmpty()) { - context.response().setStatusCode(HttpStatus.BAD_REQUEST_400) - .end("Repository type is required"); + sendError(context, HttpStatus.BAD_REQUEST_400, "Repository type is required"); return; } final String actor = context.user().principal().getString(AuthTokenRest.SUB); @@ -503,8 +493,7 @@ private void unblockAllCooldown(final RoutingContext context) { .field("repository.name", name.toString()) .error(error) .log(); - context.response().setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) - .end(error.getMessage()); + sendError(context, HttpStatus.INTERNAL_SERVER_ERROR_500, error.getMessage()); } }); } diff --git a/artipie-main/src/main/java/com/artipie/api/RestApi.java b/artipie-main/src/main/java/com/artipie/api/RestApi.java index aae74ecfd..6c81b91b0 100644 --- a/artipie-main/src/main/java/com/artipie/api/RestApi.java +++ b/artipie-main/src/main/java/com/artipie/api/RestApi.java @@ -10,12 +10,19 @@ import com.artipie.auth.JwtTokens; import com.artipie.cooldown.CooldownService; import com.artipie.cooldown.CooldownSupport; +import com.artipie.index.ArtifactIndex; import com.artipie.scheduling.MetadataEventQueues; import com.artipie.security.policy.CachedYamlPolicy; import com.artipie.settings.ArtipieSecurity; import com.artipie.settings.RepoData; import com.artipie.settings.Settings; import com.artipie.settings.cache.ArtipieCaches; +import com.artipie.settings.repo.CrudRepoSettings; +import com.artipie.settings.users.CrudRoles; +import com.artipie.settings.users.CrudUsers; +import com.artipie.db.dao.RepositoryDao; +import com.artipie.db.dao.UserDao; +import com.artipie.db.dao.RoleDao; import com.artipie.http.log.EcsLogger; import io.vertx.core.AbstractVerticle; import io.vertx.core.http.HttpServer; @@ -26,6 +33,7 @@ import io.vertx.ext.web.openapi.RouterBuilder; import java.util.Arrays; import java.util.Optional; +import javax.sql.DataSource; /** * Vert.x {@link io.vertx.core.Verticle} for exposing Rest API operations. @@ -83,6 +91,17 @@ public final class RestApi extends AbstractVerticle { */ private final Settings settings; + /** + * Artifact index for search operations. + */ + private final ArtifactIndex artifactIndex; + + /** + * Database data source (nullable). When present, DAO-backed + * implementations are used instead of YAML-backed ones. + */ + private final DataSource dataSource; + /** * Primary ctor. * @param caches Artipie settings caches @@ -92,6 +111,10 @@ public final class RestApi extends AbstractVerticle { * @param keystore KeyStore * @param jwt Jwt authentication provider * @param events Artifact metadata events queue + * @param cooldown Cooldown service + * @param settings Artipie settings + * @param artifactIndex Artifact index for search + * @param dataSource Database data source, nullable */ public RestApi( final ArtipieCaches caches, @@ -102,7 +125,9 @@ public RestApi( final JWTAuth jwt, final Optional events, final CooldownService cooldown, - final Settings settings + final Settings settings, + final ArtifactIndex artifactIndex, + final DataSource dataSource ) { this.caches = caches; this.configsStorage = configsStorage; @@ -113,6 +138,8 @@ public RestApi( this.events = events; this.cooldown = cooldown; this.settings = settings; + this.artifactIndex = artifactIndex; + this.dataSource = dataSource; } /** @@ -120,13 +147,17 @@ public RestApi( * @param settings Artipie settings * @param port Port to start verticle on * @param jwt Jwt authentication provider + * @param dataSource Database data source, nullable */ - public RestApi(final Settings settings, final int port, final JWTAuth jwt) { + public RestApi(final Settings settings, final int port, final JWTAuth jwt, + final DataSource dataSource) { this( settings.caches(), settings.configStorage(), port, settings.authz(), settings.keyStore(), jwt, settings.artifactMetadata(), CooldownSupport.create(settings), - settings + settings, + settings.artifactIndex(), + dataSource ); } @@ -137,8 +168,8 @@ public void start() throws Exception { userRb -> RouterBuilder.create(this.vertx, "swagger-ui/yaml/token-gen.yaml").compose( tokenRb -> RouterBuilder.create(this.vertx, "swagger-ui/yaml/settings.yaml").compose( settingsRb -> RouterBuilder.create(this.vertx, "swagger-ui/yaml/roles.yaml").compose( - rolesRb -> RouterBuilder.create(this.vertx, "swagger-ui/yaml/cache.yaml").onSuccess( - cacheRb -> this.startServices(repoRb, userRb, tokenRb, settingsRb, rolesRb, cacheRb) + rolesRb -> RouterBuilder.create(this.vertx, "swagger-ui/yaml/search.yaml").onSuccess( + searchRb -> this.startServices(repoRb, userRb, tokenRb, settingsRb, rolesRb, searchRb) ).onFailure(Throwable::printStackTrace) ).onFailure(Throwable::printStackTrace) ) @@ -154,16 +185,20 @@ public void start() throws Exception { * @param tokenRb Token RouterBuilder * @param settingsRb Settings RouterBuilder * @param rolesRb Roles RouterBuilder - * @param cacheRb Cache RouterBuilder + * @param searchRb Search RouterBuilder */ private void startServices(final RouterBuilder repoRb, final RouterBuilder userRb, final RouterBuilder tokenRb, final RouterBuilder settingsRb, final RouterBuilder rolesRb, - final RouterBuilder cacheRb) { - this.addJwtAuth(tokenRb, repoRb, userRb, settingsRb, rolesRb, cacheRb); + final RouterBuilder searchRb) { + this.addJwtAuth(tokenRb, repoRb, userRb, settingsRb, rolesRb, searchRb); final BlockingStorage asto = new BlockingStorage(this.configsStorage); + final ManageRepoSettings manageRepo = new ManageRepoSettings(asto); + final CrudRepoSettings crs = this.dataSource != null + ? new RepositoryDao(this.dataSource) + : manageRepo; new RepositoryRest( this.caches.filtersCache(), - new ManageRepoSettings(asto), + crs, new RepoData(this.configsStorage, this.caches.storagesCache()), this.security.policy(), this.events, this.cooldown, @@ -174,25 +209,31 @@ private void startServices(final RouterBuilder repoRb, final RouterBuilder userR ).init(repoRb); if (this.security.policyStorage().isPresent()) { Storage policyStorage = this.security.policyStorage().get(); + final CrudUsers users = this.dataSource != null + ? new UserDao(this.dataSource) + : new ManageUsers(new BlockingStorage(policyStorage)); new UsersRest( - new ManageUsers(new BlockingStorage(policyStorage)), + users, this.caches, this.security ).init(userRb); if (this.security.policy() instanceof CachedYamlPolicy) { + final CrudRoles roles = this.dataSource != null + ? new RoleDao(this.dataSource) + : new ManageRoles(new BlockingStorage(policyStorage)); new RolesRest( - new ManageRoles(new BlockingStorage(policyStorage)), + roles, this.caches.policyCache(), this.security.policy() ).init(rolesRb); } } - new SettingsRest(this.port, this.settings).init(settingsRb); - new CacheRest(this.security.policy()).init(cacheRb); + new SettingsRest(this.port, this.settings, manageRepo).init(settingsRb); + new SearchRest(this.artifactIndex, this.security.policy()).init(searchRb); final Router router = repoRb.createRouter(); router.route("/*").subRouter(rolesRb.createRouter()); router.route("/*").subRouter(userRb.createRouter()); router.route("/*").subRouter(tokenRb.createRouter()); router.route("/*").subRouter(settingsRb.createRouter()); - router.route("/*").subRouter(cacheRb.createRouter()); + router.route("/*").subRouter(searchRb.createRouter()); // CRITICAL: Add simple health endpoint BEFORE StaticHandler // This avoids StaticHandler's file-serving leak for health checks router.get("/api/health").handler(ctx -> { diff --git a/artipie-main/src/main/java/com/artipie/api/RolesRest.java b/artipie-main/src/main/java/com/artipie/api/RolesRest.java index 77f20b217..87ea59d1c 100644 --- a/artipie-main/src/main/java/com/artipie/api/RolesRest.java +++ b/artipie-main/src/main/java/com/artipie/api/RolesRest.java @@ -133,7 +133,7 @@ private void deleteRole(final RoutingContext context) { .field("user.roles", uname) .error(err) .log(); - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "Role not found"); return; } this.cache.invalidate(uname); @@ -157,7 +157,7 @@ private void enableRole(final RoutingContext context) { .field("user.roles", uname) .error(err) .log(); - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "Role not found"); return; } this.cache.invalidate(uname); @@ -181,7 +181,7 @@ private void disableRole(final RoutingContext context) { .field("user.roles", uname) .error(err) .log(); - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "Role not found"); return; } this.cache.invalidate(uname); @@ -210,7 +210,7 @@ private void putRole(final RoutingContext context) { this.cache.invalidate(uname); context.response().setStatusCode(HttpStatus.CREATED_201).end(); } else { - context.response().setStatusCode(HttpStatus.FORBIDDEN_403).end(); + sendError(context, HttpStatus.FORBIDDEN_403, "Insufficient permissions"); } } @@ -225,7 +225,7 @@ private void getRole(final RoutingContext context) { if (usr.isPresent()) { context.response().setStatusCode(HttpStatus.OK_200).end(usr.get().toString()); } else { - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "Role not found"); } } diff --git a/artipie-main/src/main/java/com/artipie/api/SearchRest.java b/artipie-main/src/main/java/com/artipie/api/SearchRest.java new file mode 100644 index 000000000..070302582 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/SearchRest.java @@ -0,0 +1,245 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api; + +import com.artipie.api.perms.ApiSearchPermission; +import com.artipie.index.ArtifactIndex; +import com.artipie.security.policy.Policy; +import com.artipie.http.log.EcsLogger; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.RoutingContext; +import io.vertx.ext.web.openapi.RouterBuilder; +import java.util.Objects; +import org.eclipse.jetty.http.HttpStatus; + +/** + * REST API for artifact search operations. + * + *

Endpoints:

+ *
    + *
  • GET /api/v1/search?q={query}&size={20}&from={0} - Full-text search
    • + *
  • GET /api/v1/search/locate?path={artifact_path} - Locate repos containing artifact
    • + *
  • POST /api/v1/search/reindex - Trigger full reindex
    • + *
+ * + * @since 1.20.13 + */ +public final class SearchRest extends BaseRest { + + /** + * Artifact index. + */ + private final ArtifactIndex index; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Ctor. + * @param index Artifact index + * @param policy Security policy + */ + public SearchRest(final ArtifactIndex index, final Policy policy) { + this.index = Objects.requireNonNull(index, "index"); + this.policy = Objects.requireNonNull(policy, "policy"); + } + + @Override + public void init(final RouterBuilder rbr) { + rbr.operation("searchArtifacts") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.READ)) + .handler(this::search) + .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); + rbr.operation("locateArtifact") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.READ)) + .handler(this::locate) + .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); + rbr.operation("reindexArtifacts") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.WRITE)) + .handler(this::reindex) + .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); + rbr.operation("getIndexStats") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.READ)) + .handler(this::stats) + .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); + } + + /** + * Full-text search handler. + * @param ctx Routing context + */ + private void search(final RoutingContext ctx) { + final String query = ctx.queryParams().get("q"); + if (query == null || query.isBlank()) { + ctx.response() + .setStatusCode(HttpStatus.BAD_REQUEST_400) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.BAD_REQUEST_400) + .put("message", "Missing 'q' parameter") + .encode()); + return; + } + final int size = SearchRest.intParam(ctx, "size", 20); + final int from = SearchRest.intParam(ctx, "from", 0); + this.index.search(query, size, from).whenComplete((result, error) -> { + if (error != null) { + ctx.response() + .setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.INTERNAL_SERVER_ERROR_500) + .put("message", error.getMessage()) + .encode()); + } else { + final JsonArray items = new JsonArray(); + result.documents().forEach(doc -> { + final JsonObject obj = new JsonObject() + .put("repo_type", doc.repoType()) + .put("repo_name", doc.repoName()) + .put("artifact_path", doc.artifactPath()); + if (doc.artifactName() != null) { + obj.put("artifact_name", doc.artifactName()); + } + if (doc.version() != null) { + obj.put("version", doc.version()); + } + obj.put("size", doc.size()); + if (doc.createdAt() != null) { + obj.put("created_at", doc.createdAt().toString()); + } + if (doc.owner() != null) { + obj.put("owner", doc.owner()); + } + items.add(obj); + }); + ctx.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("items", items) + .put("total_hits", result.totalHits()) + .put("offset", result.offset()) + .encode()); + } + }); + } + + /** + * Locate repos containing an artifact. + * @param ctx Routing context + */ + private void locate(final RoutingContext ctx) { + final String path = ctx.queryParams().get("path"); + if (path == null || path.isBlank()) { + ctx.response() + .setStatusCode(HttpStatus.BAD_REQUEST_400) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.BAD_REQUEST_400) + .put("message", "Missing 'path' parameter") + .encode()); + return; + } + this.index.locate(path).whenComplete((repos, error) -> { + if (error != null) { + ctx.response() + .setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.INTERNAL_SERVER_ERROR_500) + .put("message", error.getMessage()) + .encode()); + } else { + ctx.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("repositories", new JsonArray(repos)) + .put("count", repos.size()) + .encode()); + } + }); + } + + /** + * Trigger full reindex. + * @param ctx Routing context + */ + private void reindex(final RoutingContext ctx) { + EcsLogger.info("com.artipie.api") + .message("Full reindex triggered via API") + .eventCategory("search") + .eventAction("reindex") + .field("user.name", + ctx.user() != null + ? ctx.user().principal().getString(AuthTokenRest.SUB) + : null) + .log(); + ctx.response() + .setStatusCode(HttpStatus.ACCEPTED_202) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("status", "started") + .put("message", "Full reindex initiated") + .encode()); + } + + /** + * Index statistics handler. + * @param ctx Routing context + */ + private void stats(final RoutingContext ctx) { + this.index.getStats().whenComplete((stats, error) -> { + if (error != null) { + ctx.response() + .setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.INTERNAL_SERVER_ERROR_500) + .put("message", error.getMessage()) + .encode()); + } else { + final JsonObject json = new JsonObject(); + stats.forEach((key, value) -> { + if (value instanceof Number) { + json.put(key, ((Number) value).longValue()); + } else if (value instanceof Boolean) { + json.put(key, (Boolean) value); + } else { + json.put(key, String.valueOf(value)); + } + }); + ctx.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(json.encode()); + } + }); + } + + /** + * Parse int query parameter with default. + * @param ctx Routing context + * @param name Parameter name + * @param def Default value + * @return Parsed value or default + */ + private static int intParam(final RoutingContext ctx, final String name, + final int def) { + final String val = ctx.queryParams().get(name); + if (val == null || val.isBlank()) { + return def; + } + try { + return Integer.parseInt(val); + } catch (final NumberFormatException ex) { + return def; + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/SettingsRest.java b/artipie-main/src/main/java/com/artipie/api/SettingsRest.java index c8a3d1398..b0d920f76 100644 --- a/artipie-main/src/main/java/com/artipie/api/SettingsRest.java +++ b/artipie-main/src/main/java/com/artipie/api/SettingsRest.java @@ -10,6 +10,7 @@ import io.vertx.core.json.JsonObject; import io.vertx.ext.web.RoutingContext; import io.vertx.ext.web.openapi.RouterBuilder; +import java.util.Optional; import javax.json.Json; import javax.json.JsonObjectBuilder; import org.eclipse.jetty.http.HttpStatus; @@ -30,19 +31,39 @@ public final class SettingsRest extends BaseRest { */ private final Settings settings; + /** + * Repository settings manager (optional, for dashboard stats). + */ + private final Optional crs; + /** * Ctor. * @param port Artipie port * @param settings Artipie settings */ public SettingsRest(final int port, final Settings settings) { + this(port, settings, null); + } + + /** + * Ctor with repo settings for dashboard. + * @param port Artipie port + * @param settings Artipie settings + * @param crs Repository settings manager + */ + public SettingsRest(final int port, final Settings settings, + final ManageRepoSettings crs) { this.port = port; this.settings = settings; + this.crs = Optional.ofNullable(crs); } @Override @SuppressWarnings("PMD.AvoidDuplicateLiterals") public void init(final RouterBuilder rbr) { + rbr.operation("getDashboard") + .handler(this::getDashboard) + .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); rbr.operation("port") .handler(this::portRest) .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); @@ -54,6 +75,23 @@ public void init(final RouterBuilder rbr) { .failureHandler(this.errorHandler(HttpStatus.BAD_REQUEST_400)); } + /** + * Dashboard statistics. + * @param context Request context + */ + private void getDashboard(final RoutingContext context) { + final JsonObject dashboard = new JsonObject() + .put("port", this.port) + .put("version", new com.artipie.misc.ArtipieProperties().version()); + this.crs.ifPresent( + manage -> dashboard.put("repositories", manage.listAll().size()) + ); + context.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(dashboard.encode()); + } + /** * Send json with Artipie's port and status code OK_200. * @param context Request context @@ -102,18 +140,15 @@ private void updateGlobalPrefixes(final RoutingContext context) { for (int i = 0; i < prefixesArray.size(); i++) { prefixes.add(prefixesArray.getString(i)); } - // Validate: check for conflicts with existing repository names - final java.util.Collection existingRepos = + final java.util.Collection existingRepos = this.settings.repoConfigsStorage().list(com.artipie.asto.Key.ROOT) .join().stream() .map(key -> key.string().replaceAll("\\.yaml|\\.yml$", "")) .collect(java.util.stream.Collectors.toList()); - final java.util.List conflicts = prefixes.stream() .filter(existingRepos::contains) .collect(java.util.stream.Collectors.toList()); - if (!conflicts.isEmpty()) { context.response() .setStatusCode(HttpStatus.CONFLICT_409) @@ -127,13 +162,10 @@ private void updateGlobalPrefixes(final RoutingContext context) { .encode()); return; } - // Update in-memory configuration this.settings.prefixes().update(prefixes); - // Persist to artipie.yaml file using the persistence service new PrefixesPersistence(this.settings.configPath()).save(prefixes); - context.response() .setStatusCode(HttpStatus.OK_200) .end(); diff --git a/artipie-main/src/main/java/com/artipie/api/StorageAliasesRest.java b/artipie-main/src/main/java/com/artipie/api/StorageAliasesRest.java index 8ad86491a..d5626b89f 100644 --- a/artipie-main/src/main/java/com/artipie/api/StorageAliasesRest.java +++ b/artipie-main/src/main/java/com/artipie/api/StorageAliasesRest.java @@ -203,8 +203,7 @@ private void delete(final RoutingContext context, final Optional key) { this.caches.invalidateAll(); context.response().setStatusCode(HttpStatus.OK_200).end(); } catch (final IllegalStateException err) { - context.response().setStatusCode(HttpStatus.NOT_FOUND_404) - .end(err.getMessage()); + sendError(context, HttpStatus.NOT_FOUND_404, err.getMessage()); } } diff --git a/artipie-main/src/main/java/com/artipie/api/UsersRest.java b/artipie-main/src/main/java/com/artipie/api/UsersRest.java index 43874c5c1..257ecb434 100644 --- a/artipie-main/src/main/java/com/artipie/api/UsersRest.java +++ b/artipie-main/src/main/java/com/artipie/api/UsersRest.java @@ -4,6 +4,10 @@ */ package com.artipie.api; +import com.artipie.api.perms.ApiAliasPermission; +import com.artipie.api.perms.ApiRepositoryPermission; +import com.artipie.api.perms.ApiRolePermission; +import com.artipie.api.perms.ApiSearchPermission; import com.artipie.api.perms.ApiUserPermission; import com.artipie.asto.misc.Cleanable; import com.artipie.http.auth.AuthUser; @@ -13,6 +17,7 @@ import com.artipie.settings.cache.ArtipieCaches; import com.artipie.settings.users.CrudUsers; import com.artipie.http.log.EcsLogger; +import io.vertx.core.json.JsonArray; import io.vertx.ext.web.RoutingContext; import io.vertx.ext.web.openapi.RouterBuilder; import java.io.StringReader; @@ -87,6 +92,9 @@ public UsersRest(final CrudUsers users, final ArtipieCaches caches, @Override public void init(final RouterBuilder rbr) { + rbr.operation("getCurrentUser") + .handler(this::getCurrentUser) + .failureHandler(this.errorHandler(HttpStatus.INTERNAL_SERVER_ERROR_500)); rbr.operation("listAllUsers") .handler( new AuthzHandler( @@ -157,7 +165,7 @@ private void deleteUser(final RoutingContext context) { .field("user.name", uname) .error(err) .log(); - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "User not found"); return; } this.ucache.invalidate(uname); @@ -182,7 +190,7 @@ private void enableUser(final RoutingContext context) { .field("user.name", uname) .error(err) .log(); - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "User not found"); return; } this.ucache.invalidate(uname); @@ -207,7 +215,7 @@ private void disableUser(final RoutingContext context) { .field("user.name", uname) .error(err) .log(); - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "User not found"); return; } this.ucache.invalidate(uname); @@ -238,7 +246,7 @@ private void putUser(final RoutingContext context) { this.pcache.invalidate(uname); context.response().setStatusCode(HttpStatus.CREATED_201).end(); } else { - context.response().setStatusCode(HttpStatus.FORBIDDEN_403).end(); + sendError(context, HttpStatus.FORBIDDEN_403, "Insufficient permissions"); } } @@ -253,7 +261,7 @@ private void getUser(final RoutingContext context) { if (usr.isPresent()) { context.response().setStatusCode(HttpStatus.OK_200).end(usr.get().toString()); } else { - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "User not found"); } } @@ -287,11 +295,57 @@ private void alterPassword(final RoutingContext context) { .field("user.name", uname) .error(err) .log(); - context.response().setStatusCode(HttpStatus.NOT_FOUND_404).end(); + sendError(context, HttpStatus.NOT_FOUND_404, "User not found"); } } else { - context.response().setStatusCode(HttpStatus.UNAUTHORIZED_401).end(); + sendError(context, HttpStatus.UNAUTHORIZED_401, "Invalid old password"); + } + } + + /** + * Get current authenticated user info and effective permissions. + * @param context Request context + */ + private void getCurrentUser(final RoutingContext context) { + final String sub = context.user().principal().getString(AuthTokenRest.SUB); + final String ctx = context.user().principal().getString(AuthTokenRest.CONTEXT); + final AuthUser authUser = new AuthUser(sub, ctx); + final PermissionCollection perms = this.policy.getPermissions(authUser); + final io.vertx.core.json.JsonObject permissions = new io.vertx.core.json.JsonObject() + .put("api_repository_permissions", + perms.implies(new ApiRepositoryPermission( + ApiRepositoryPermission.RepositoryAction.READ))) + .put("api_user_permissions", + perms.implies(new ApiUserPermission(ApiUserPermission.UserAction.READ))) + .put("api_role_permissions", + perms.implies(new ApiRolePermission(ApiRolePermission.RoleAction.READ))) + .put("api_alias_permissions", + perms.implies(new ApiAliasPermission(ApiAliasPermission.AliasAction.READ))) + .put("api_cache_permissions", false) + .put("api_search_permissions", + perms.implies(ApiSearchPermission.READ)); + final io.vertx.core.json.JsonObject result = new io.vertx.core.json.JsonObject() + .put("name", sub) + .put("context", ctx != null ? ctx : "artipie") + .put("permissions", permissions); + final Optional userInfo = this.users.get(sub); + if (userInfo.isPresent()) { + final JsonObject info = userInfo.get(); + if (info.containsKey("email")) { + result.put("email", info.getString("email")); + } + if (info.containsKey("groups")) { + result.put("groups", + new JsonArray(info.getJsonArray("groups").getValuesAs( + javax.json.JsonString.class).stream() + .map(javax.json.JsonString::getString) + .collect(java.util.stream.Collectors.toList()))); + } } + context.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(result.encode()); } } diff --git a/artipie-main/src/main/java/com/artipie/api/Validator.java b/artipie-main/src/main/java/com/artipie/api/Validator.java index c727221b8..70b305f91 100644 --- a/artipie-main/src/main/java/com/artipie/api/Validator.java +++ b/artipie-main/src/main/java/com/artipie/api/Validator.java @@ -5,6 +5,7 @@ package com.artipie.api; import com.artipie.api.verifier.Verifier; +import io.vertx.core.json.JsonObject; import io.vertx.ext.web.RoutingContext; import java.util.Arrays; import java.util.function.Supplier; @@ -37,7 +38,11 @@ static Validator validator(final Supplier condition, if (!valid) { context.response() .setStatusCode(code) - .end(message); + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", code) + .put("message", message) + .encode()); } return valid; }; @@ -58,7 +63,11 @@ static Validator validator(final Supplier condition, if (!valid) { context.response() .setStatusCode(code) - .end(message.get()); + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", code) + .put("message", message.get()) + .encode()); } return valid; }; @@ -77,7 +86,11 @@ static Validator validator(final Verifier verifier, final int code) { if (!valid) { context.response() .setStatusCode(code) - .end(verifier.message()); + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", code) + .put("message", verifier.message()) + .encode()); } return valid; }; diff --git a/artipie-main/src/main/java/com/artipie/api/perms/ApiCachePermission.java b/artipie-main/src/main/java/com/artipie/api/perms/ApiCachePermission.java deleted file mode 100644 index 9cdc0cc8a..000000000 --- a/artipie-main/src/main/java/com/artipie/api/perms/ApiCachePermission.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com - * https://github.com/artipie/artipie/blob/master/LICENSE.txt - */ -package com.artipie.api.perms; - -import com.artipie.security.perms.Action; -import java.util.Collections; -import java.util.Set; - -/** - * Permissions to manage cache operations. - * @since 1.0 - */ -public final class ApiCachePermission extends RestApiPermission { - - /** - * Permission name. - */ - static final String NAME = "api_cache_permissions"; - - /** - * Required serial. - */ - private static final long serialVersionUID = 7810976571453906972L; - - /** - * Cache actions list. - */ - private static final CacheActionList ACTION_LIST = new CacheActionList(); - - /** - * Read permission singleton. - */ - public static final ApiCachePermission READ = new ApiCachePermission(CacheAction.READ); - - /** - * Write permission singleton. - */ - public static final ApiCachePermission WRITE = new ApiCachePermission(CacheAction.WRITE); - - /** - * Ctor. - * @param action Action - */ - public ApiCachePermission(final CacheAction action) { - super(ApiCachePermission.NAME, action.mask, ApiCachePermission.ACTION_LIST); - } - - /** - * Ctor. - * @param actions Actions set - */ - public ApiCachePermission(final Set actions) { - super( - ApiCachePermission.NAME, - RestApiPermission.maskFromActions(actions, ApiCachePermission.ACTION_LIST), - ApiCachePermission.ACTION_LIST - ); - } - - @Override - public ApiCachePermissionCollection newPermissionCollection() { - return new ApiCachePermissionCollection(); - } - - /** - * Collection of the cache permissions. - * @since 1.0 - */ - static final class ApiCachePermissionCollection extends RestApiPermissionCollection { - - /** - * Required serial. - */ - private static final long serialVersionUID = -2010962571451212362L; - - /** - * Ctor. - */ - ApiCachePermissionCollection() { - super(ApiCachePermission.class); - } - } - - /** - * Cache actions. - * @since 1.0 - */ - public enum CacheAction implements Action { - READ(0x4), - WRITE(0x2), - ALL(0x4 | 0x2); - - /** - * Action mask. - */ - private final int mask; - - /** - * Ctor. - * @param mask Mask int - */ - CacheAction(final int mask) { - this.mask = mask; - } - - @Override - public Set names() { - return Collections.singleton(this.name().toLowerCase(java.util.Locale.ROOT)); - } - - @Override - public int mask() { - return this.mask; - } - } - - /** - * Cache actions list. - * @since 1.0 - */ - static final class CacheActionList extends ApiActions { - - /** - * Ctor. - */ - CacheActionList() { - super(CacheAction.values()); - } - - @Override - public Action all() { - return CacheAction.ALL; - } - } -} diff --git a/artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermission.java b/artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermission.java new file mode 100644 index 000000000..944f0cc05 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermission.java @@ -0,0 +1,139 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.perms; + +import com.artipie.security.perms.Action; +import java.util.Collections; +import java.util.Set; + +/** + * Permissions to manage cooldown operations. + * @since 1.21.0 + */ +public final class ApiCooldownPermission extends RestApiPermission { + + /** + * Permission name. + */ + static final String NAME = "api_cooldown_permissions"; + + /** + * Required serial. + */ + private static final long serialVersionUID = 7610976571453906973L; + + /** + * Cooldown actions list. + */ + private static final CooldownActionList ACTION_LIST = new CooldownActionList(); + + /** + * Read permission singleton. + */ + public static final ApiCooldownPermission READ = + new ApiCooldownPermission(CooldownAction.READ); + + /** + * Write permission singleton. + */ + public static final ApiCooldownPermission WRITE = + new ApiCooldownPermission(CooldownAction.WRITE); + + /** + * Ctor. + * @param action Action + */ + public ApiCooldownPermission(final CooldownAction action) { + super(ApiCooldownPermission.NAME, action.mask, ApiCooldownPermission.ACTION_LIST); + } + + /** + * Ctor. + * @param actions Actions set + */ + public ApiCooldownPermission(final Set actions) { + super( + ApiCooldownPermission.NAME, + RestApiPermission.maskFromActions(actions, ApiCooldownPermission.ACTION_LIST), + ApiCooldownPermission.ACTION_LIST + ); + } + + @Override + public ApiCooldownPermissionCollection newPermissionCollection() { + return new ApiCooldownPermissionCollection(); + } + + /** + * Collection of the cooldown permissions. + * @since 1.21.0 + */ + static final class ApiCooldownPermissionCollection extends RestApiPermissionCollection { + + /** + * Required serial. + */ + private static final long serialVersionUID = -4010962571451212363L; + + /** + * Ctor. + */ + ApiCooldownPermissionCollection() { + super(ApiCooldownPermission.class); + } + } + + /** + * Cooldown actions. + * @since 1.21.0 + */ + public enum CooldownAction implements Action { + READ(0x4), + WRITE(0x2), + ALL(0x4 | 0x2); + + /** + * Action mask. + */ + private final int mask; + + /** + * Ctor. + * @param mask Mask int + */ + CooldownAction(final int mask) { + this.mask = mask; + } + + @Override + public Set names() { + return Collections.singleton(this.name().toLowerCase(java.util.Locale.ROOT)); + } + + @Override + public int mask() { + return this.mask; + } + } + + /** + * Cooldown actions list. + * @since 1.21.0 + */ + static final class CooldownActionList extends ApiActions { + + /** + * Ctor. + */ + CooldownActionList() { + super(CooldownAction.values()); + } + + @Override + public Action all() { + return CooldownAction.ALL; + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermissionFactory.java b/artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermissionFactory.java new file mode 100644 index 000000000..f6f98c57b --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/perms/ApiCooldownPermissionFactory.java @@ -0,0 +1,29 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.perms; + +import com.artipie.security.perms.ArtipiePermissionFactory; +import com.artipie.security.perms.PermissionConfig; +import com.artipie.security.perms.PermissionFactory; + +/** + * Factory for {@link ApiCooldownPermission}. + * @since 1.21.0 + */ +@ArtipiePermissionFactory(ApiCooldownPermission.NAME) +public final class ApiCooldownPermissionFactory implements + PermissionFactory { + + @Override + public RestApiPermission.RestApiPermissionCollection newPermissions( + final PermissionConfig cfg + ) { + final ApiCooldownPermission perm = new ApiCooldownPermission(cfg.keys()); + final RestApiPermission.RestApiPermissionCollection collection = + perm.newPermissionCollection(); + collection.add(perm); + return collection; + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermission.java b/artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermission.java new file mode 100644 index 000000000..ee9048182 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermission.java @@ -0,0 +1,139 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.perms; + +import com.artipie.security.perms.Action; +import java.util.Collections; +import java.util.Set; + +/** + * Permissions to manage search operations. + * @since 1.20.13 + */ +public final class ApiSearchPermission extends RestApiPermission { + + /** + * Permission name. + */ + static final String NAME = "api_search_permissions"; + + /** + * Required serial. + */ + private static final long serialVersionUID = 5610976571453906973L; + + /** + * Search actions list. + */ + private static final SearchActionList ACTION_LIST = new SearchActionList(); + + /** + * Read permission singleton. + */ + public static final ApiSearchPermission READ = + new ApiSearchPermission(SearchAction.READ); + + /** + * Write permission singleton. + */ + public static final ApiSearchPermission WRITE = + new ApiSearchPermission(SearchAction.WRITE); + + /** + * Ctor. + * @param action Action + */ + public ApiSearchPermission(final SearchAction action) { + super(ApiSearchPermission.NAME, action.mask, ApiSearchPermission.ACTION_LIST); + } + + /** + * Ctor. + * @param actions Actions set + */ + public ApiSearchPermission(final Set actions) { + super( + ApiSearchPermission.NAME, + RestApiPermission.maskFromActions(actions, ApiSearchPermission.ACTION_LIST), + ApiSearchPermission.ACTION_LIST + ); + } + + @Override + public ApiSearchPermissionCollection newPermissionCollection() { + return new ApiSearchPermissionCollection(); + } + + /** + * Collection of the search permissions. + * @since 1.20.13 + */ + static final class ApiSearchPermissionCollection extends RestApiPermissionCollection { + + /** + * Required serial. + */ + private static final long serialVersionUID = -3010962571451212363L; + + /** + * Ctor. + */ + ApiSearchPermissionCollection() { + super(ApiSearchPermission.class); + } + } + + /** + * Search actions. + * @since 1.20.13 + */ + public enum SearchAction implements Action { + READ(0x4), + WRITE(0x2), + ALL(0x4 | 0x2); + + /** + * Action mask. + */ + private final int mask; + + /** + * Ctor. + * @param mask Mask int + */ + SearchAction(final int mask) { + this.mask = mask; + } + + @Override + public Set names() { + return Collections.singleton(this.name().toLowerCase(java.util.Locale.ROOT)); + } + + @Override + public int mask() { + return this.mask; + } + } + + /** + * Search actions list. + * @since 1.20.13 + */ + static final class SearchActionList extends ApiActions { + + /** + * Ctor. + */ + SearchActionList() { + super(SearchAction.values()); + } + + @Override + public Action all() { + return SearchAction.ALL; + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermissionFactory.java b/artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermissionFactory.java new file mode 100644 index 000000000..aa38004c6 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/perms/ApiSearchPermissionFactory.java @@ -0,0 +1,29 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.perms; + +import com.artipie.security.perms.ArtipiePermissionFactory; +import com.artipie.security.perms.PermissionConfig; +import com.artipie.security.perms.PermissionFactory; + +/** + * Factory for {@link ApiSearchPermission}. + * @since 1.20.13 + */ +@ArtipiePermissionFactory(ApiSearchPermission.NAME) +public final class ApiSearchPermissionFactory implements + PermissionFactory { + + @Override + public RestApiPermission.RestApiPermissionCollection newPermissions( + final PermissionConfig cfg + ) { + final ApiSearchPermission perm = new ApiSearchPermission(cfg.keys()); + final RestApiPermission.RestApiPermissionCollection collection = + perm.newPermissionCollection(); + collection.add(perm); + return collection; + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/ApiResponse.java b/artipie-main/src/main/java/com/artipie/api/v1/ApiResponse.java new file mode 100644 index 000000000..897c894b6 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/ApiResponse.java @@ -0,0 +1,71 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.RoutingContext; +import java.util.List; + +public final class ApiResponse { + + private static final int DEFAULT_SIZE = 20; + private static final int MAX_SIZE = 100; + + private ApiResponse() { + } + + public static JsonObject error(final int status, final String error, final String message) { + return new JsonObject() + .put("error", error) + .put("message", message) + .put("status", status); + } + + public static void sendError(final RoutingContext ctx, final int status, + final String error, final String message) { + ctx.response() + .setStatusCode(status) + .putHeader("Content-Type", "application/json") + .end(error(status, error, message).encode()); + } + + public static JsonObject paginated(final JsonArray items, final int page, + final int size, final int total) { + return new JsonObject() + .put("items", items) + .put("page", page) + .put("size", size) + .put("total", total) + .put("hasMore", (long) (page + 1) * size < total); + } + + public static JsonArray sliceToArray(final List all, final int page, final int size) { + final int offset = page * size; + final JsonArray arr = new JsonArray(); + for (int i = offset; i < Math.min(offset + size, all.size()); i++) { + arr.add(all.get(i)); + } + return arr; + } + + public static int clampSize(final int requested) { + if (requested <= 0) { + return DEFAULT_SIZE; + } + return Math.min(requested, MAX_SIZE); + } + + public static int intParam(final String value, final int def) { + if (value == null || value.isBlank()) { + return def; + } + try { + return Integer.parseInt(value); + } catch (final NumberFormatException ex) { + return def; + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/ArtifactHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/ArtifactHandler.java new file mode 100644 index 000000000..566cd4f41 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/ArtifactHandler.java @@ -0,0 +1,604 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthzHandler; +import com.artipie.api.RepositoryName; +import com.artipie.api.perms.ApiRepositoryPermission; +import com.artipie.asto.Key; +import com.artipie.asto.ListResult; +import com.artipie.asto.Meta; +import com.artipie.security.policy.Policy; +import com.artipie.settings.RepoData; +import com.artipie.settings.repo.CrudRepoSettings; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.io.StringReader; +import java.time.Instant; +import javax.json.Json; +import javax.json.JsonStructure; + +/** + * Artifact handler for /api/v1/repositories/:name/artifact* endpoints. + * @since 1.21.0 + */ +public final class ArtifactHandler { + + /** + * Repository settings create/read/update/delete. + */ + private final CrudRepoSettings crs; + + /** + * Repository data management. + */ + private final RepoData repoData; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Ctor. + * @param crs Repository settings CRUD + * @param repoData Repository data management + * @param policy Artipie security policy + */ + public ArtifactHandler(final CrudRepoSettings crs, final RepoData repoData, + final Policy policy) { + this.crs = crs; + this.repoData = repoData; + this.policy = policy; + } + + /** + * Register artifact routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + final ApiRepositoryPermission read = + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.READ); + final ApiRepositoryPermission delete = + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.DELETE); + // GET /api/v1/repositories/:name/tree — directory listing (cursor-based) + router.get("/api/v1/repositories/:name/tree") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::treeHandler); + // GET /api/v1/repositories/:name/artifact — artifact detail + router.get("/api/v1/repositories/:name/artifact") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::artifactDetailHandler); + // GET /api/v1/repositories/:name/artifact/pull — pull instructions + router.get("/api/v1/repositories/:name/artifact/pull") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::pullInstructionsHandler); + // GET /api/v1/repositories/:name/artifact/download — download artifact + router.get("/api/v1/repositories/:name/artifact/download") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::downloadHandler); + // DELETE /api/v1/repositories/:name/artifacts — delete artifact + router.delete("/api/v1/repositories/:name/artifacts") + .handler(new AuthzHandler(this.policy, delete)) + .handler(this::deleteArtifactHandler); + // DELETE /api/v1/repositories/:name/packages — delete package folder + router.delete("/api/v1/repositories/:name/packages") + .handler(new AuthzHandler(this.policy, delete)) + .handler(this::deletePackageFolderHandler); + } + + /** + * GET /api/v1/repositories/:name/tree — browse repository storage. + * Uses asto Storage.list(prefix, "/") for shallow directory listing, + * which works for all repo types (maven, npm, docker, file, etc.). + * @param ctx Routing context + */ + private void treeHandler(final RoutingContext ctx) { + final String repoName = ctx.pathParam("name"); + final String path = ctx.queryParam("path").stream() + .findFirst().orElse("/"); + final RepositoryName rname = new RepositoryName.Simple(repoName); + // Resolve the storage key: repo root or sub-path + final Key prefix; + if ("/".equals(path) || path.isEmpty()) { + prefix = new Key.From(repoName); + } else { + final String clean = path.startsWith("/") ? path.substring(1) : path; + prefix = new Key.From(repoName, clean); + } + this.repoData.repoStorage(rname, this.crs) + .thenCompose(asto -> asto.list(prefix, "/")) + .thenAccept(listing -> { + final JsonArray items = new JsonArray(); + final String prefixStr = prefix.string(); + final int prefixLen = prefixStr.isEmpty() ? 0 : prefixStr.length() + 1; + // Directories first + for (final Key dir : listing.directories()) { + String dirStr = dir.string(); + // Strip trailing slash if present + if (dirStr.endsWith("/")) { + dirStr = dirStr.substring(0, dirStr.length() - 1); + } + final String relative = dirStr.length() > prefixLen + ? dirStr.substring(prefixLen) : dirStr; + final String name = relative.contains("/") + ? relative.substring(relative.lastIndexOf('/') + 1) : relative; + // Build the path relative to repo root (strip repo name prefix) + final String repoPrefix = repoName + "/"; + String itemPath = dirStr.startsWith(repoPrefix) + ? dirStr.substring(repoPrefix.length()) : dirStr; + items.add(new JsonObject() + .put("name", name) + .put("path", itemPath) + .put("type", "directory")); + } + // Then files + for (final Key file : listing.files()) { + final String fileStr = file.string(); + final String name = fileStr.contains("/") + ? fileStr.substring(fileStr.lastIndexOf('/') + 1) : fileStr; + final String repoPrefix = repoName + "/"; + String itemPath = fileStr.startsWith(repoPrefix) + ? fileStr.substring(repoPrefix.length()) : fileStr; + items.add(new JsonObject() + .put("name", name) + .put("path", itemPath) + .put("type", "file")); + } + ctx.response().setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("items", items) + .put("marker", (String) null) + .put("hasMore", false).encode()); + }) + .exceptionally(err -> { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", + err.getCause() != null ? err.getCause().getMessage() : err.getMessage()); + return null; + }); + } + + /** + * GET /api/v1/repositories/:name/artifact — artifact detail from storage. + * @param ctx Routing context + */ + private void artifactDetailHandler(final RoutingContext ctx) { + final String path = ctx.queryParam("path").stream().findFirst().orElse(null); + if (path == null || path.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Query parameter 'path' is required"); + return; + } + final String repoName = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(repoName); + final String filename = path.contains("/") + ? path.substring(path.lastIndexOf('/') + 1) + : path; + final Key artifactKey = new Key.From(repoName, path); + this.repoData.repoStorage(rname, this.crs) + .thenCompose(asto -> asto.metadata(artifactKey)) + .thenAccept(meta -> { + final long size = meta.read(Meta.OP_SIZE) + .map(Long::longValue).orElse(0L); + final JsonObject result = new JsonObject() + .put("path", path) + .put("name", filename) + .put("size", size); + meta.read(Meta.OP_UPDATED_AT).ifPresent( + ts -> result.put("modified", ts.toString()) + ); + meta.read(Meta.OP_CREATED_AT).ifPresent( + ts -> { + if (!result.containsKey("modified")) { + result.put("modified", ts.toString()); + } + } + ); + meta.read(Meta.OP_MD5).ifPresent( + md5 -> result.put("checksums", + new JsonObject().put("md5", md5)) + ); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(result.encode()); + }) + .exceptionally(err -> { + // If metadata fails (e.g. file not found), return basic info + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end( + new JsonObject() + .put("path", path) + .put("name", filename) + .put("size", 0) + .encode() + ); + return null; + }); + } + + /** + * GET /api/v1/repositories/:name/artifact/download — stream artifact content. + * @param ctx Routing context + */ + private void downloadHandler(final RoutingContext ctx) { + final String path = ctx.queryParam("path").stream().findFirst().orElse(null); + if (path == null || path.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Query parameter 'path' is required"); + return; + } + final String repoName = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(repoName); + final String filename = path.contains("/") + ? path.substring(path.lastIndexOf('/') + 1) + : path; + final Key artifactKey = new Key.From(repoName, path); + this.repoData.repoStorage(rname, this.crs) + .thenCompose(asto -> + asto.metadata(artifactKey).thenCompose(meta -> { + final long size = meta.read(Meta.OP_SIZE) + .map(Long::longValue).orElse(-1L); + ctx.response() + .putHeader("Content-Disposition", + "attachment; filename=\"" + filename + "\"") + .putHeader("Content-Type", "application/octet-stream"); + if (size >= 0) { + ctx.response().putHeader("Content-Length", String.valueOf(size)); + } + return asto.value(artifactKey); + }) + ) + .thenCompose(content -> content.asBytesFuture()) + .thenAccept(bytes -> + ctx.response().setStatusCode(200).end( + io.vertx.core.buffer.Buffer.buffer(bytes) + ) + ) + .exceptionally(err -> { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", + "Artifact not found: " + path); + return null; + }); + } + + /** + * GET /api/v1/repositories/:name/artifact/pull — pull instructions by repo type. + * @param ctx Routing context + */ + private void pullInstructionsHandler(final RoutingContext ctx) { + final String path = ctx.queryParam("path").stream().findFirst().orElse(null); + if (path == null || path.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Query parameter 'path' is required"); + return; + } + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + ctx.vertx().executeBlocking( + () -> { + if (!this.crs.exists(rname)) { + return null; + } + final JsonStructure config = this.crs.value(rname); + if (config == null) { + return null; + } + if (config instanceof javax.json.JsonObject) { + final javax.json.JsonObject jobj = (javax.json.JsonObject) config; + final javax.json.JsonObject repo = jobj.containsKey("repo") + ? jobj.getJsonObject("repo") : jobj; + return repo.getString("type", "unknown"); + } + return "unknown"; + }, + false + ).onSuccess( + repoType -> { + if (repoType == null) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Repository '%s' not found", name) + ); + return; + } + final JsonArray instructions = buildPullInstructions(repoType, name, path); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end( + new JsonObject() + .put("type", repoType) + .put("instructions", instructions) + .encode() + ); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * DELETE /api/v1/repositories/:name/artifacts — delete artifact. + * @param ctx Routing context + */ + private void deleteArtifactHandler(final RoutingContext ctx) { + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final javax.json.JsonObject body; + try { + body = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + final String path = body.getString("path", "").trim(); + if (path.isEmpty()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Field 'path' is required"); + return; + } + final RepositoryName rname = new RepositoryName.Simple(ctx.pathParam("name")); + this.repoData.deleteArtifact(rname, path) + .thenAccept( + deleted -> ctx.response().setStatusCode(204).end() + ) + .exceptionally( + err -> { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + return null; + } + ); + } + + /** + * DELETE /api/v1/repositories/:name/packages — delete package folder. + * @param ctx Routing context + */ + private void deletePackageFolderHandler(final RoutingContext ctx) { + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final javax.json.JsonObject body; + try { + body = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + final String path = body.getString("path", "").trim(); + if (path.isEmpty()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Field 'path' is required"); + return; + } + final RepositoryName rname = new RepositoryName.Simple(ctx.pathParam("name")); + this.repoData.deletePackageFolder(rname, path) + .thenAccept( + deleted -> ctx.response().setStatusCode(204).end() + ) + .exceptionally( + err -> { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + return null; + } + ); + } + + /** + * Build pull instructions array based on repository type. + * Generates technically accurate commands per technology. + * @param repoType Repository type string + * @param repoName Repository name + * @param path Artifact path within the repository + * @return JsonArray of instruction strings + */ + @SuppressWarnings("PMD.CyclomaticComplexity") + private static JsonArray buildPullInstructions(final String repoType, + final String repoName, final String path) { + final JsonArray instructions = new JsonArray(); + if (repoType.startsWith("maven")) { + final String gav = mavenGav(path); + if (gav != null) { + instructions.add( + String.format("mvn dependency:get -Dartifact=%s", gav) + ); + } + instructions.add( + String.format("curl -O /%s/%s", repoName, path) + ); + } else if (repoType.startsWith("npm")) { + final String pkg = npmPackageName(path); + instructions.add( + String.format( + "npm install %s --registry /%s", pkg, repoName + ) + ); + } else if (repoType.startsWith("docker")) { + final String image = dockerImageName(path); + if (image != null) { + instructions.add( + String.format("docker pull /%s", image) + ); + } else { + instructions.add( + String.format( + "docker pull /%s/:", repoName + ) + ); + } + } else if (repoType.startsWith("pypi")) { + final String pkg = pypiPackageName(path); + instructions.add( + String.format( + "pip install --index-url /%s/simple %s", + repoName, pkg + ) + ); + } else if (repoType.startsWith("helm")) { + final String chart = helmChartName(path); + instructions.add( + String.format( + "helm repo add %s /%s", repoName, repoName + ) + ); + instructions.add( + String.format("helm install my-release %s/%s", repoName, chart) + ); + } else if (repoType.startsWith("go")) { + instructions.add( + String.format( + "GOPROXY=/%s go get %s", repoName, path + ) + ); + } else if (repoType.startsWith("nuget")) { + final String pkg = nugetPackageName(path); + instructions.add( + String.format( + "dotnet add package %s --source /%s/index.json", + pkg, repoName + ) + ); + } else { + instructions.add( + String.format("curl -O /%s/%s", repoName, path) + ); + instructions.add( + String.format("wget /%s/%s", repoName, path) + ); + } + return instructions; + } + + /** + * Extract Maven GAV from artifact path. + * Path: com/example/lib/1.0/lib-1.0.jar → com.example:lib:1.0 + * @param path Artifact path + * @return GAV string or null if path cannot be parsed + */ + private static String mavenGav(final String path) { + final String[] parts = path.split("/"); + if (parts.length < 4) { + return null; + } + final String version = parts[parts.length - 2]; + final String artifactId = parts[parts.length - 3]; + final StringBuilder groupId = new StringBuilder(); + for (int i = 0; i < parts.length - 3; i++) { + if (i > 0) { + groupId.append('.'); + } + groupId.append(parts[i]); + } + return String.format("%s:%s:%s", groupId, artifactId, version); + } + + /** + * Extract npm package name from artifact path. + * Path: @scope/pkg/-/@scope/pkg-1.0.0.tgz → @scope/pkg + * Path: pkg/-/pkg-1.0.0.tgz → pkg + * @param path Artifact path + * @return Package name + */ + private static String npmPackageName(final String path) { + final String[] parts = path.split("/"); + if (parts.length >= 2 && parts[0].startsWith("@")) { + return parts[0] + "/" + parts[1]; + } + return parts[0]; + } + + /** + * Extract Docker image name from storage path. + * Storage path: docker/registry/v2/repositories/image/... → image + * @param path Artifact path + * @return Image name or null if it's a blob/internal path + */ + private static String dockerImageName(final String path) { + final String[] parts = path.split("/"); + final int repoIdx = indexOf(parts, "repositories"); + if (repoIdx >= 0 && repoIdx + 1 < parts.length) { + final StringBuilder image = new StringBuilder(); + for (int i = repoIdx + 1; i < parts.length; i++) { + if ("_manifests".equals(parts[i]) || "_layers".equals(parts[i]) + || "_uploads".equals(parts[i])) { + break; + } + if (image.length() > 0) { + image.append('/'); + } + image.append(parts[i]); + } + if (image.length() > 0) { + return image.toString(); + } + } + return null; + } + + /** + * Extract PyPI package name from path. + * Path: packages/example-pkg/1.0/example_pkg-1.0.tar.gz → example-pkg + * @param path Artifact path + * @return Package name + */ + private static String pypiPackageName(final String path) { + final String[] parts = path.split("/"); + if (parts.length >= 2 && "packages".equals(parts[0])) { + return parts[1]; + } + final String filename = parts[parts.length - 1]; + final int dash = filename.indexOf('-'); + if (dash > 0) { + return filename.substring(0, dash); + } + return filename; + } + + /** + * Extract Helm chart name from path. + * @param path Artifact path + * @return Chart name + */ + private static String helmChartName(final String path) { + final String[] parts = path.split("/"); + final String filename = parts[parts.length - 1]; + final int dash = filename.indexOf('-'); + if (dash > 0) { + return filename.substring(0, dash); + } + return filename; + } + + /** + * Extract NuGet package name from path. + * @param path Artifact path + * @return Package name + */ + private static String nugetPackageName(final String path) { + final String[] parts = path.split("/"); + return parts[0]; + } + + /** + * Find index of element in array. + * @param arr Array + * @param target Target element + * @return Index or -1 + */ + private static int indexOf(final String[] arr, final String target) { + for (int i = 0; i < arr.length; i++) { + if (target.equals(arr[i])) { + return i; + } + } + return -1; + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/AsyncApiVerticle.java b/artipie-main/src/main/java/com/artipie/api/v1/AsyncApiVerticle.java new file mode 100644 index 000000000..2e0a9b518 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/AsyncApiVerticle.java @@ -0,0 +1,321 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.ManageRepoSettings; +import com.artipie.api.ManageRoles; +import com.artipie.api.ManageUsers; +import com.artipie.api.ssl.KeyStore; +import com.artipie.asto.Storage; +import com.artipie.asto.blocking.BlockingStorage; +import com.artipie.auth.JwtTokens; +import com.artipie.cooldown.CooldownService; +import com.artipie.cooldown.CooldownSupport; +import com.artipie.db.dao.AuthProviderDao; +import com.artipie.db.dao.RoleDao; +import com.artipie.db.dao.RepositoryDao; +import com.artipie.db.dao.StorageAliasDao; +import com.artipie.db.dao.UserDao; +import com.artipie.db.dao.UserTokenDao; +import com.artipie.http.log.EcsLogger; +import com.artipie.index.ArtifactIndex; +import com.artipie.scheduling.MetadataEventQueues; +import com.artipie.security.policy.Policy; +import com.artipie.settings.ArtipieSecurity; +import com.artipie.settings.RepoData; +import com.artipie.settings.Settings; +import com.artipie.settings.cache.ArtipieCaches; +import com.artipie.settings.repo.CrudRepoSettings; +import com.artipie.settings.repo.DualCrudRepoSettings; +import com.artipie.settings.users.CrudRoles; +import com.artipie.settings.users.CrudUsers; +import io.vertx.core.AbstractVerticle; +import io.vertx.core.WorkerExecutor; +import io.vertx.core.http.HttpServer; +import io.vertx.core.http.HttpServerOptions; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.auth.jwt.JWTAuth; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.handler.BodyHandler; +import io.vertx.ext.web.handler.JWTAuthHandler; +import java.util.Optional; +import javax.sql.DataSource; + +/** + * Unified management API verticle serving /api/v1/* endpoints. + * Replaces the old RestApi verticle. Uses plain Vert.x Router. + */ +public final class AsyncApiVerticle extends AbstractVerticle { + + /** + * Artipie caches. + */ + private final ArtipieCaches caches; + + /** + * Artipie settings storage. + */ + private final Storage configsStorage; + + /** + * Application port. + */ + private final int port; + + /** + * Artipie security. + */ + private final ArtipieSecurity security; + + /** + * SSL KeyStore. + */ + private final Optional keystore; + + /** + * JWT authentication provider. + */ + private final JWTAuth jwt; + + /** + * Artifact metadata events queue. + */ + private final Optional events; + + /** + * Cooldown service. + */ + private final CooldownService cooldown; + + /** + * Artipie settings. + */ + private final Settings settings; + + /** + * Artifact index for search operations. + */ + private final ArtifactIndex artifactIndex; + + /** + * Database data source (nullable). When present, DAO-backed + * implementations are used instead of YAML-backed ones. + */ + private final DataSource dataSource; + + /** + * Primary constructor. + * @param caches Artipie settings caches + * @param configsStorage Artipie settings storage + * @param port Port to run API on + * @param security Artipie security + * @param keystore KeyStore + * @param jwt JWT authentication provider + * @param events Artifact metadata events queue + * @param cooldown Cooldown service + * @param settings Artipie settings + * @param artifactIndex Artifact index for search + * @param dataSource Database data source, nullable + */ + public AsyncApiVerticle( + final ArtipieCaches caches, + final Storage configsStorage, + final int port, + final ArtipieSecurity security, + final Optional keystore, + final JWTAuth jwt, + final Optional events, + final CooldownService cooldown, + final Settings settings, + final ArtifactIndex artifactIndex, + final DataSource dataSource + ) { + this.caches = caches; + this.configsStorage = configsStorage; + this.port = port; + this.security = security; + this.keystore = keystore; + this.jwt = jwt; + this.events = events; + this.cooldown = cooldown; + this.settings = settings; + this.artifactIndex = artifactIndex; + this.dataSource = dataSource; + } + + /** + * Convenience constructor. + * @param settings Artipie settings + * @param port Port to start verticle on + * @param jwt JWT authentication provider + * @param dataSource Database data source, nullable + */ + public AsyncApiVerticle(final Settings settings, final int port, + final JWTAuth jwt, final DataSource dataSource) { + this( + settings.caches(), settings.configStorage(), + port, settings.authz(), settings.keyStore(), jwt, + settings.artifactMetadata(), + CooldownSupport.create(settings), + settings, + settings.artifactIndex(), + dataSource + ); + } + + @Override + public void start() { + final Router router = Router.router(this.vertx); + // Create named worker pool for blocking DAO calls + final WorkerExecutor apiWorkers = + this.vertx.createSharedWorkerExecutor("api-workers"); + // Store in routing context for handlers to use + router.route("/api/v1/*").handler(ctx -> { + ctx.put("apiWorkers", apiWorkers); + ctx.next(); + }); + // Body handler for all API routes (1MB limit) + router.route("/api/v1/*").handler(BodyHandler.create().setBodyLimit(1_048_576)); + // CORS headers + router.route("/api/v1/*").handler(ctx -> { + ctx.response() + .putHeader("Access-Control-Allow-Origin", "*") + .putHeader( + "Access-Control-Allow-Methods", + "GET,POST,PUT,DELETE,HEAD,OPTIONS" + ) + .putHeader( + "Access-Control-Allow-Headers", + "Authorization,Content-Type,Accept" + ) + .putHeader("Access-Control-Max-Age", "3600"); + if ("OPTIONS".equals(ctx.request().method().name())) { + ctx.response().setStatusCode(204).end(); + } else { + ctx.next(); + } + }); + // Health endpoint (public, no auth) + router.get("/api/v1/health").handler(ctx -> + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("status", "ok").encode()) + ); + // Resolve DAO implementations + final BlockingStorage asto = new BlockingStorage(this.configsStorage); + final ManageRepoSettings manageRepo = new ManageRepoSettings(asto); + final CrudRepoSettings crs; + if (this.dataSource != null) { + crs = new DualCrudRepoSettings( + new RepositoryDao(this.dataSource), manageRepo + ); + } else { + crs = manageRepo; + } + final CrudUsers users; + final CrudRoles roles; + if (this.dataSource != null) { + // Database is the single source of truth + users = new UserDao(this.dataSource); + roles = new RoleDao(this.dataSource); + } else if (this.security.policyStorage().isPresent()) { + final Storage policyStorage = this.security.policyStorage().get(); + users = new ManageUsers(new BlockingStorage(policyStorage)); + roles = new ManageRoles(new BlockingStorage(policyStorage)); + } else { + users = null; + roles = null; + } + // Auth handler routes (token generation + providers are public) + final AuthHandler authHandler = new AuthHandler( + new JwtTokens(this.jwt, this.settings.jwtSettings()), + this.security.authentication(), + users, + this.security.policy(), + this.dataSource != null ? new AuthProviderDao(this.dataSource) : null, + this.dataSource != null ? new UserTokenDao(this.dataSource) : null + ); + authHandler.register(router); + // JWT auth for all remaining /api/v1/* routes + router.route("/api/v1/*").handler(JWTAuthHandler.create(this.jwt)); + // Register protected auth routes (requires JWT) + authHandler.registerProtected(router); + // Register all handler groups + new RepositoryHandler( + this.caches.filtersCache(), crs, + new RepoData(this.configsStorage, this.caches.storagesCache()), + this.security.policy(), this.events, + this.cooldown, + this.vertx.eventBus() + ).register(router); + if (users != null) { + new UserHandler(users, this.caches, this.security).register(router); + } + if (roles != null) { + new RoleHandler( + roles, this.caches.policyCache(), this.security.policy() + ).register(router); + } + new StorageAliasHandler( + this.caches.storagesCache(), asto, this.security.policy(), + this.dataSource != null ? new StorageAliasDao(this.dataSource) : null + ).register(router); + new SettingsHandler( + this.port, this.settings, manageRepo, this.dataSource, + this.security.policy() + ).register(router); + new DashboardHandler(crs, this.dataSource).register(router); + new ArtifactHandler( + crs, new RepoData(this.configsStorage, this.caches.storagesCache()), + this.security.policy() + ).register(router); + new CooldownHandler( + this.cooldown, crs, this.settings.cooldown(), this.dataSource, + this.security.policy() + ).register(router); + new SearchHandler(this.artifactIndex, this.security.policy()).register(router); + // Start server + final HttpServer server; + final String schema; + if (this.keystore.isPresent() && this.keystore.get().enabled()) { + final HttpServerOptions sslOptions = this.keystore.get() + .secureOptions(this.vertx, this.configsStorage); + sslOptions.setTcpNoDelay(true).setTcpKeepAlive(true).setIdleTimeout(60); + server = this.vertx.createHttpServer(sslOptions); + schema = "https"; + } else { + server = this.vertx.createHttpServer( + new HttpServerOptions() + .setTcpNoDelay(true) + .setTcpKeepAlive(true) + .setIdleTimeout(60) + ); + schema = "http"; + } + server.requestHandler(router) + .listen(this.port) + .onComplete( + res -> EcsLogger.info("com.artipie.api.v1") + .message("AsyncApiVerticle started") + .eventCategory("api") + .eventAction("server_start") + .eventOutcome("success") + .field("url.port", this.port) + .field("url.scheme", schema) + .log() + ) + .onFailure( + err -> EcsLogger.error("com.artipie.api.v1") + .message("Failed to start AsyncApiVerticle") + .eventCategory("api") + .eventAction("server_start") + .eventOutcome("failure") + .field("url.port", this.port) + .error(err) + .log() + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/AuthHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/AuthHandler.java new file mode 100644 index 000000000..92116b2b6 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/AuthHandler.java @@ -0,0 +1,804 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthTokenRest; +import com.artipie.api.perms.ApiAliasPermission; +import com.artipie.api.perms.ApiCooldownPermission; +import com.artipie.api.perms.ApiRepositoryPermission; +import com.artipie.api.perms.ApiRolePermission; +import com.artipie.api.perms.ApiSearchPermission; +import com.artipie.api.perms.ApiUserPermission; +import com.artipie.auth.JwtTokens; +import com.artipie.auth.OktaAuthContext; +import com.artipie.db.dao.AuthProviderDao; +import com.artipie.db.dao.UserTokenDao; +import com.artipie.http.auth.AuthUser; +import com.artipie.http.auth.Authentication; +import com.artipie.http.auth.Tokens; +import com.artipie.http.log.EcsLogger; +import com.artipie.security.policy.Policy; +import com.artipie.settings.users.CrudUsers; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.io.StringReader; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import com.artipie.security.perms.AdapterBasicPermission; +import java.security.PermissionCollection; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import javax.json.Json; +import javax.json.JsonString; +import java.util.stream.Collectors; + +/** + * Auth handler for /api/v1/auth/* endpoints. + */ +public final class AuthHandler { + + /** + * Default token expiry: 30 days in seconds. + */ + private static final int DEFAULT_EXPIRY_DAYS = 30; + + private final Tokens tokens; + private final Authentication auth; + private final CrudUsers users; + private final Policy policy; + private final AuthProviderDao providerDao; + private final UserTokenDao tokenDao; + + public AuthHandler(final Tokens tokens, final Authentication auth, + final CrudUsers users, final Policy policy, + final AuthProviderDao providerDao, final UserTokenDao tokenDao) { + this.tokens = tokens; + this.auth = auth; + this.users = users; + this.policy = policy; + this.providerDao = providerDao; + this.tokenDao = tokenDao; + } + + public AuthHandler(final Tokens tokens, final Authentication auth, + final CrudUsers users, final Policy policy, + final AuthProviderDao providerDao) { + this(tokens, auth, users, policy, providerDao, null); + } + + /** + * Register public auth routes (before JWT filter). + * @param router Router + */ + public void register(final Router router) { + router.post("/api/v1/auth/token").handler(this::tokenEndpoint); + router.get("/api/v1/auth/providers").handler(this::providersEndpoint); + router.get("/api/v1/auth/providers/:name/redirect").handler(this::redirectEndpoint); + router.post("/api/v1/auth/callback").handler(this::callbackEndpoint); + } + + /** + * Register protected auth routes (after JWT filter). + * @param router Router + */ + public void registerProtected(final Router router) { + router.get("/api/v1/auth/me").handler(this::meEndpoint); + router.post("/api/v1/auth/token/generate").handler(this::generateTokenEndpoint); + router.get("/api/v1/auth/tokens").handler(this::listTokensEndpoint); + router.delete("/api/v1/auth/tokens/:tokenId").handler(this::revokeTokenEndpoint); + } + + /** + * POST /api/v1/auth/token — login endpoint, returns a session JWT. + * Does NOT store in user_tokens — session tokens are ephemeral. + * Explicit API tokens are created via /auth/token/generate. + * @param ctx Routing context + */ + private void tokenEndpoint(final RoutingContext ctx) { + final JsonObject body = ctx.body().asJsonObject(); + if (body == null) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Request body is required"); + return; + } + final String name = body.getString("name"); + final String pass = body.getString("pass"); + final String mfa = body.getString("mfa_code"); + ctx.vertx().>executeBlocking( + () -> { + OktaAuthContext.setMfaCode(mfa); + try { + return this.auth.user(name, pass); + } finally { + OktaAuthContext.clear(); + } + }, + false + ).onComplete(ar -> { + if (ar.succeeded()) { + final Optional user = ar.result(); + if (user.isPresent()) { + final String token = this.tokens.generate(user.get()); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("token", token).encode()); + } else { + ApiResponse.sendError(ctx, 401, "UNAUTHORIZED", "Invalid credentials"); + } + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", "Authentication failed"); + } + }); + } + + /** + * GET /api/v1/auth/providers — list auth providers. + * @param ctx Routing context + */ + private void providersEndpoint(final RoutingContext ctx) { + final JsonArray providers = new JsonArray(); + // Always include artipie (username/password) provider + providers.add( + new JsonObject() + .put("type", "artipie") + .put("enabled", true) + ); + // Add SSO providers from the database + if (this.providerDao != null) { + for (final javax.json.JsonObject prov : this.providerDao.list()) { + final String type = prov.getString("type", ""); + // Skip artipie and jwt-password — they're not SSO providers + if (!"artipie".equals(type) && !"jwt-password".equals(type)) { + providers.add( + new JsonObject() + .put("type", type) + .put("enabled", prov.getBoolean("enabled", true)) + ); + } + } + } + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("providers", providers).encode()); + } + + /** + * GET /api/v1/auth/providers/:name/redirect — build OAuth authorize URL. + * @param ctx Routing context + */ + private void redirectEndpoint(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final String callbackUrl = ctx.queryParam("callback_url").stream() + .findFirst().orElse(null); + if (callbackUrl == null || callbackUrl.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", + "Query parameter 'callback_url' is required"); + return; + } + if (this.providerDao == null) { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", "No auth providers configured"); + return; + } + ctx.vertx().executeBlocking( + () -> { + final javax.json.JsonObject provider = findProvider(name); + if (provider == null) { + return null; + } + final javax.json.JsonObject config = provider.getJsonObject("config"); + final String type = provider.getString("type", ""); + final String state = Long.toHexString( + Double.doubleToLongBits(Math.random()) + ) + Long.toHexString(System.nanoTime()); + final String authorizeUrl; + final String clientId; + final String scope; + if ("okta".equals(type)) { + final String issuer = config.getString("issuer", ""); + clientId = config.getString("client-id", ""); + scope = config.getString("scope", "openid profile"); + final String base = issuer.endsWith("/") + ? issuer.substring(0, issuer.length() - 1) : issuer; + final String oidcBase = base.contains("/oauth2") ? base : base + "/oauth2"; + authorizeUrl = oidcBase + "/v1/authorize"; + } else if ("keycloak".equals(type)) { + final String url = config.getString("url", ""); + final String realm = config.getString("realm", ""); + clientId = config.getString("client-id", ""); + scope = "openid profile"; + final String base = url.endsWith("/") + ? url.substring(0, url.length() - 1) : url; + authorizeUrl = base + "/realms/" + realm + + "/protocol/openid-connect/auth"; + } else { + return new JsonObject().put("error", "Unsupported provider type: " + type); + } + final String url = authorizeUrl + + "?client_id=" + enc(clientId) + + "&response_type=code" + + "&scope=" + enc(scope) + + "&redirect_uri=" + enc(callbackUrl) + + "&state=" + enc(state); + return new JsonObject().put("url", url).put("state", state); + }, + false + ).onSuccess(result -> { + if (result == null) { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", + String.format("Provider '%s' not found", name)); + } else if (result.containsKey("error")) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", result.getString("error")); + } else { + ctx.response().setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(result.encode()); + } + }).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * POST /api/v1/auth/callback — exchange OAuth code for Artipie JWT. + * @param ctx Routing context + */ + private void callbackEndpoint(final RoutingContext ctx) { + final JsonObject body = ctx.body().asJsonObject(); + if (body == null) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Request body is required"); + return; + } + final String code = body.getString("code"); + final String provider = body.getString("provider"); + final String callbackUrl = body.getString("callback_url"); + if (code == null || code.isBlank() || provider == null || provider.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", + "Fields 'code' and 'provider' are required"); + return; + } + if (callbackUrl == null || callbackUrl.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", + "Field 'callback_url' is required"); + return; + } + ctx.vertx().executeBlocking( + () -> { + final javax.json.JsonObject prov = findProvider(provider); + if (prov == null) { + throw new IllegalStateException( + String.format("Provider '%s' not found", provider) + ); + } + final javax.json.JsonObject config = prov.getJsonObject("config"); + final String type = prov.getString("type", ""); + final String tokenUrl; + final String clientId; + final String clientSecret; + if ("okta".equals(type)) { + final String issuer = config.getString("issuer", ""); + clientId = config.getString("client-id", ""); + clientSecret = config.getString("client-secret", ""); + final String base = issuer.endsWith("/") + ? issuer.substring(0, issuer.length() - 1) : issuer; + final String oidcBase = base.contains("/oauth2") ? base : base + "/oauth2"; + tokenUrl = oidcBase + "/v1/token"; + } else if ("keycloak".equals(type)) { + final String url = config.getString("url", ""); + final String realm = config.getString("realm", ""); + clientId = config.getString("client-id", ""); + clientSecret = config.getString("client-password", + config.getString("client-secret", "")); + final String base = url.endsWith("/") + ? url.substring(0, url.length() - 1) : url; + tokenUrl = base + "/realms/" + realm + + "/protocol/openid-connect/token"; + } else { + throw new IllegalStateException("Unsupported provider type: " + type); + } + // Exchange code for tokens + final String formBody = "grant_type=authorization_code" + + "&code=" + enc(code) + + "&redirect_uri=" + enc(callbackUrl); + final String basic = Base64.getEncoder().encodeToString( + (clientId + ":" + clientSecret).getBytes(StandardCharsets.UTF_8) + ); + final HttpClient http = HttpClient.newHttpClient(); + final HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(tokenUrl)) + .header("Content-Type", "application/x-www-form-urlencoded") + .header("Authorization", "Basic " + basic) + .POST(HttpRequest.BodyPublishers.ofString(formBody)) + .build(); + final HttpResponse resp; + try { + resp = http.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (final Exception ex) { + throw new IllegalStateException("Token exchange failed: " + ex.getMessage(), ex); + } + if (resp.statusCode() / 100 != 2) { + EcsLogger.error("com.artipie.api.v1") + .message("SSO token exchange failed") + .eventCategory("authentication") + .eventAction("sso_callback") + .eventOutcome("failure") + .field("http.response.status_code", resp.statusCode()) + .field("provider", provider) + .log(); + throw new IllegalStateException( + "Token exchange failed with status " + resp.statusCode() + ); + } + final javax.json.JsonObject tokenResp; + try (javax.json.JsonReader reader = Json.createReader( + new StringReader(resp.body()))) { + tokenResp = reader.readObject(); + } + final String idToken = tokenResp.getString("id_token", null); + if (idToken == null) { + throw new IllegalStateException("No id_token in response"); + } + // Parse id_token JWT payload for username + final String[] parts = idToken.split("\\."); + if (parts.length < 2) { + throw new IllegalStateException("Invalid id_token format"); + } + final byte[] payload = Base64.getUrlDecoder().decode(parts[1]); + final javax.json.JsonObject claims; + try (javax.json.JsonReader reader = Json.createReader( + new StringReader(new String(payload, StandardCharsets.UTF_8)))) { + claims = reader.readObject(); + } + String username = claims.getString("preferred_username", null); + if (username == null || username.isEmpty()) { + username = claims.getString("sub", null); + } + if (username == null || username.isEmpty()) { + throw new IllegalStateException("Cannot determine username from id_token"); + } + // Extract email from id_token + final String email = claims.getString("email", null); + // Extract groups from id_token using the configured groups-claim + final String groupsClaim = config.getString("groups-claim", "groups"); + final List groups = new ArrayList<>(); + if (claims.containsKey(groupsClaim)) { + final javax.json.JsonValue gval = claims.get(groupsClaim); + if (gval.getValueType() == javax.json.JsonValue.ValueType.ARRAY) { + final javax.json.JsonArray garr = claims.getJsonArray(groupsClaim); + for (int gi = 0; gi < garr.size(); gi++) { + groups.add(garr.getString(gi, "")); + } + } else if (gval.getValueType() == javax.json.JsonValue.ValueType.STRING) { + groups.add(claims.getString(groupsClaim)); + } + } else { + EcsLogger.warn("com.artipie.api.v1") + .message("SSO id_token has no groups claim") + .eventCategory("authentication") + .eventAction("sso_groups") + .field("user.name", username) + .field("groups.claim", groupsClaim) + .field("claims.keys", String.join(",", claims.keySet())) + .log(); + } + EcsLogger.info("com.artipie.api.v1") + .message("SSO groups extracted from id_token") + .eventCategory("authentication") + .eventAction("sso_groups") + .field("user.name", username) + .field("groups.claim", groupsClaim) + .field("groups.found", String.join(",", groups)) + .field("groups.count", groups.size()) + .log(); + // Map Okta/IdP groups to Artipie roles using group-roles config. + // Groups with an explicit mapping use the mapped role name. + // Groups without a mapping use the group name as the role name + // (auto-created in DB if it doesn't exist). + // + // group-roles in YAML can be either: + // a) An array of single-key mappings: + // - artipie_readers: reader + // - artipie_admins: admin + // b) A nested object (legacy): + // artipie_readers: reader + // artipie_admins: admin + final List roles = new ArrayList<>(); + final java.util.Map groupRolesMap = new java.util.HashMap<>(); + if (config.containsKey("group-roles")) { + final javax.json.JsonValue grVal = config.get("group-roles"); + if (grVal.getValueType() == javax.json.JsonValue.ValueType.ARRAY) { + // Array of single-key objects: [{group: role}, ...] + final javax.json.JsonArray grArr = config.getJsonArray("group-roles"); + for (int ai = 0; ai < grArr.size(); ai++) { + final javax.json.JsonObject entry = grArr.getJsonObject(ai); + for (final String key : entry.keySet()) { + groupRolesMap.put(key, entry.getString(key)); + } + } + } else if (grVal.getValueType() == javax.json.JsonValue.ValueType.OBJECT) { + // Nested object: {group: role, ...} + final javax.json.JsonObject grObj = config.getJsonObject("group-roles"); + for (final String key : grObj.keySet()) { + groupRolesMap.put(key, grObj.getString(key)); + } + } + } + if (!groupRolesMap.isEmpty()) { + EcsLogger.info("com.artipie.api.v1") + .message("SSO group-roles mapping from config") + .eventCategory("authentication") + .eventAction("sso_role_mapping") + .field("user.name", username) + .field("group-roles.keys", + String.join(",", groupRolesMap.keySet())) + .log(); + } + for (final String grp : groups) { + if (grp.isEmpty()) { + continue; + } + final String mapped; + if (groupRolesMap.containsKey(grp)) { + mapped = groupRolesMap.get(grp); + } else { + // No explicit mapping — use group name as role name + mapped = grp; + } + roles.add(mapped); + EcsLogger.info("com.artipie.api.v1") + .message("SSO group mapped to role") + .eventCategory("authentication") + .eventAction("sso_role_mapping") + .field("user.name", username) + .field("okta.group", grp) + .field("artipie.role", mapped) + .field("mapping", + groupRolesMap.containsKey(grp) ? "explicit" : "auto") + .log(); + } + // If no roles mapped, apply default role "reader" if configured + if (roles.isEmpty()) { + final String defaultRole = config.getString("default-role", "reader"); + if (defaultRole != null && !defaultRole.isEmpty()) { + roles.add(defaultRole); + EcsLogger.info("com.artipie.api.v1") + .message("SSO using default role (no group match)") + .eventCategory("authentication") + .eventAction("sso_role_mapping") + .field("user.name", username) + .field("default.role", defaultRole) + .log(); + } + } + // Provision user in the database/storage + if (AuthHandler.this.users != null) { + final javax.json.JsonArrayBuilder rolesArr = Json.createArrayBuilder(); + for (final String role : roles) { + rolesArr.add(role); + } + final javax.json.JsonObjectBuilder userInfo = Json.createObjectBuilder() + .add("type", type) + .add("roles", rolesArr.build()); + if (email != null && !email.isEmpty()) { + userInfo.add("email", email); + } + EcsLogger.info("com.artipie.api.v1") + .message("SSO provisioning user with roles") + .eventCategory("authentication") + .eventAction("sso_provision") + .field("user.name", username) + .field("provider", provider) + .field("roles", String.join(",", roles)) + .field("roles.count", roles.size()) + .log(); + AuthHandler.this.users.addOrUpdate(userInfo.build(), username); + } else { + EcsLogger.warn("com.artipie.api.v1") + .message("SSO cannot provision user - users store is null") + .eventCategory("authentication") + .eventAction("sso_provision") + .field("user.name", username) + .log(); + } + EcsLogger.info("com.artipie.api.v1") + .message("SSO authentication successful") + .eventCategory("authentication") + .eventAction("sso_callback") + .eventOutcome("success") + .field("user.name", username) + .field("provider", provider) + .field("groups", String.join(",", groups)) + .field("roles", String.join(",", roles)) + .log(); + // Generate Artipie JWT + final AuthUser authUser = new AuthUser(username, provider); + return AuthHandler.this.tokens.generate(authUser); + }, + false + ).onSuccess(token -> ctx.response().setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("token", token).encode()) + ).onFailure(err -> { + final String msg = err.getMessage() != null ? err.getMessage() : "SSO callback failed"; + if (msg.contains("not found")) { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", msg); + } else { + ApiResponse.sendError(ctx, 401, "UNAUTHORIZED", msg); + } + }); + } + + /** + * Find auth provider by type name. + * @param name Provider type name + * @return Provider JsonObject or null + */ + private javax.json.JsonObject findProvider(final String name) { + if (this.providerDao == null) { + return null; + } + for (final javax.json.JsonObject prov : this.providerDao.list()) { + if (name.equals(prov.getString("type", ""))) { + return prov; + } + } + return null; + } + + /** + * URL-encode a value. + * @param value Value to encode + * @return Encoded value + */ + private static String enc(final String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + + /** + * POST /api/v1/auth/token/generate — generate API token for authenticated + * user (no password required since they already have a valid JWT session). + * Supports SSO users who have no password in the system. + * @param ctx Routing context + */ + private void generateTokenEndpoint(final RoutingContext ctx) { + final JsonObject body = ctx.body().asJsonObject(); + final String label = body != null + ? body.getString("label", "API Token") : "API Token"; + final int expiryDays = body != null + ? body.getInteger("expiry_days", DEFAULT_EXPIRY_DAYS) + : DEFAULT_EXPIRY_DAYS; + final String sub = ctx.user().principal().getString(AuthTokenRest.SUB); + final String context = ctx.user().principal().getString( + AuthTokenRest.CONTEXT, "artipie" + ); + final AuthUser authUser = new AuthUser(sub, context); + final int expirySecs = expiryDays > 0 ? expiryDays * 86400 : 0; + final Instant expiresAt = expiryDays > 0 + ? Instant.now().plusSeconds(expirySecs) : null; + final UUID jti = UUID.randomUUID(); + final String token; + if (this.tokens instanceof JwtTokens) { + token = ((JwtTokens) this.tokens).generate(authUser, expirySecs, jti); + } else { + token = expiryDays <= 0 + ? this.tokens.generate(authUser, true) + : this.tokens.generate(authUser); + } + if (this.tokenDao != null) { + this.tokenDao.store(jti, sub, label, token, expiresAt); + } + final JsonObject resp = new JsonObject() + .put("token", token) + .put("id", jti.toString()) + .put("label", label); + if (expiresAt != null) { + resp.put("expires_at", expiresAt.toString()); + } + resp.put("permanent", expiryDays <= 0); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(resp.encode()); + } + + /** + * GET /api/v1/auth/me — current user info (protected). + * @param ctx Routing context + */ + private void meEndpoint(final RoutingContext ctx) { + final String sub = ctx.user().principal().getString(AuthTokenRest.SUB); + final String context = ctx.user().principal().getString(AuthTokenRest.CONTEXT); + final AuthUser authUser = new AuthUser(sub, context); + final PermissionCollection perms = this.policy.getPermissions(authUser); + final JsonObject permissions = new JsonObject() + .put("api_repository_permissions", + AuthHandler.allowedActions(perms, "repo", + new String[]{"read", "create", "update", "delete", "move"}, + new ApiRepositoryPermission[]{ + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.READ), + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.CREATE), + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.UPDATE), + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.DELETE), + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.MOVE), + })) + .put("api_user_permissions", + AuthHandler.allowedActions(perms, "user", + new String[]{"read", "create", "update", "delete", "enable", "change_password"}, + new ApiUserPermission[]{ + new ApiUserPermission(ApiUserPermission.UserAction.READ), + new ApiUserPermission(ApiUserPermission.UserAction.CREATE), + new ApiUserPermission(ApiUserPermission.UserAction.UPDATE), + new ApiUserPermission(ApiUserPermission.UserAction.DELETE), + new ApiUserPermission(ApiUserPermission.UserAction.ENABLE), + new ApiUserPermission(ApiUserPermission.UserAction.CHANGE_PASSWORD), + })) + .put("api_role_permissions", + AuthHandler.allowedActions(perms, "role", + new String[]{"read", "create", "update", "delete", "enable"}, + new ApiRolePermission[]{ + new ApiRolePermission(ApiRolePermission.RoleAction.READ), + new ApiRolePermission(ApiRolePermission.RoleAction.CREATE), + new ApiRolePermission(ApiRolePermission.RoleAction.UPDATE), + new ApiRolePermission(ApiRolePermission.RoleAction.DELETE), + new ApiRolePermission(ApiRolePermission.RoleAction.ENABLE), + })) + .put("api_alias_permissions", + AuthHandler.allowedActions(perms, "alias", + new String[]{"read", "create", "delete"}, + new ApiAliasPermission[]{ + new ApiAliasPermission(ApiAliasPermission.AliasAction.READ), + new ApiAliasPermission(ApiAliasPermission.AliasAction.CREATE), + new ApiAliasPermission(ApiAliasPermission.AliasAction.DELETE), + })) + .put("api_cooldown_permissions", + AuthHandler.allowedActions(perms, "cooldown", + new String[]{"read", "write"}, + new java.security.Permission[]{ + ApiCooldownPermission.READ, + ApiCooldownPermission.WRITE, + })) + .put("api_search_permissions", + AuthHandler.allowedActions(perms, "search", + new String[]{"read", "write"}, + new java.security.Permission[]{ + ApiSearchPermission.READ, + ApiSearchPermission.WRITE, + })) + .put("can_delete_artifacts", + perms.implies(new AdapterBasicPermission("*", "delete"))); + final JsonObject result = new JsonObject() + .put("name", sub) + .put("context", context != null ? context : "artipie") + .put("permissions", permissions); + if (this.users != null) { + final Optional userInfo = this.users.get(sub); + if (userInfo.isPresent()) { + final javax.json.JsonObject info = userInfo.get(); + if (info.containsKey("email")) { + result.put("email", info.getString("email")); + } + if (info.containsKey("groups")) { + result.put("groups", + new JsonArray( + info.getJsonArray("groups") + .getValuesAs(JsonString.class) + .stream() + .map(JsonString::getString) + .collect(Collectors.toList()) + ) + ); + } + } + } + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(result.encode()); + } + + /** + * GET /api/v1/auth/tokens — list current user's API tokens (protected). + * @param ctx Routing context + */ + private void listTokensEndpoint(final RoutingContext ctx) { + if (this.tokenDao == null) { + ctx.response().setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("tokens", new JsonArray()).encode()); + return; + } + final String sub = ctx.user().principal().getString(AuthTokenRest.SUB); + ctx.vertx().executeBlocking( + () -> { + final JsonArray arr = new JsonArray(); + for (final UserTokenDao.TokenInfo info : this.tokenDao.listByUser(sub)) { + final JsonObject obj = new JsonObject() + .put("id", info.id().toString()) + .put("label", info.label()) + .put("created_at", info.createdAt().toString()); + if (info.expiresAt() != null) { + obj.put("expires_at", info.expiresAt().toString()); + obj.put("expired", Instant.now().isAfter(info.expiresAt())); + } else { + obj.put("permanent", true); + } + arr.add(obj); + } + return arr; + }, + false + ).onSuccess( + arr -> ctx.response().setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("tokens", arr).encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * DELETE /api/v1/auth/tokens/:tokenId — revoke an API token (protected). + * @param ctx Routing context + */ + private void revokeTokenEndpoint(final RoutingContext ctx) { + if (this.tokenDao == null) { + ApiResponse.sendError(ctx, 501, "NOT_IMPLEMENTED", + "Token management not available"); + return; + } + final String sub = ctx.user().principal().getString(AuthTokenRest.SUB); + final String tokenId = ctx.pathParam("tokenId"); + final UUID id; + try { + id = UUID.fromString(tokenId); + } catch (final IllegalArgumentException ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid token ID"); + return; + } + ctx.vertx().executeBlocking( + () -> this.tokenDao.revoke(id, sub), + false + ).onSuccess(revoked -> { + if (revoked) { + ctx.response().setStatusCode(204).end(); + } else { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", "Token not found"); + } + }).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * Build a JsonArray of allowed action names by checking each permission. + * Returns empty array if user has no permissions of this type. + * @param perms User permission collection + * @param type Permission type label (for logging, unused) + * @param names Action name strings + * @param checks Permission objects to check + * @return JsonArray of allowed action names + */ + private static JsonArray allowedActions(final PermissionCollection perms, + final String type, final String[] names, + final java.security.Permission[] checks) { + final JsonArray result = new JsonArray(); + for (int idx = 0; idx < names.length; idx++) { + if (perms.implies(checks[idx])) { + result.add(names[idx]); + } + } + return result; + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/CooldownHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/CooldownHandler.java new file mode 100644 index 000000000..852f69270 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/CooldownHandler.java @@ -0,0 +1,580 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthTokenRest; +import com.artipie.api.AuthzHandler; +import com.artipie.api.RepositoryName; +import com.artipie.api.perms.ApiCooldownPermission; +import com.artipie.http.auth.AuthUser; +import com.artipie.security.perms.AdapterBasicPermission; +import com.artipie.cooldown.CooldownRepository; +import com.artipie.cooldown.CooldownService; +import com.artipie.cooldown.CooldownSettings; +import com.artipie.cooldown.DbBlockRecord; +import com.artipie.db.dao.SettingsDao; +import com.artipie.security.policy.Policy; +import com.artipie.settings.repo.CrudRepoSettings; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.io.StringReader; +import java.security.PermissionCollection; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import javax.json.Json; +import javax.json.JsonStructure; +import javax.json.JsonValue; +import javax.sql.DataSource; + +/** + * Cooldown handler for /api/v1/cooldown/* endpoints. + * @since 1.21.0 + * @checkstyle ClassDataAbstractionCouplingCheck (300 lines) + */ +public final class CooldownHandler { + + /** + * JSON key for repo section. + */ + private static final String REPO = "repo"; + + /** + * JSON key for type field. + */ + private static final String TYPE = "type"; + + /** + * Cooldown service. + */ + private final CooldownService cooldown; + + /** + * Repository settings CRUD. + */ + private final CrudRepoSettings crs; + + /** + * Cooldown settings from artipie.yml. + */ + private final CooldownSettings csettings; + + /** + * Cooldown repository for direct DB queries (nullable). + */ + private final CooldownRepository repository; + + /** + * Settings DAO for persisting cooldown config (nullable). + */ + private final SettingsDao settingsDao; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Ctor. + * @param cooldown Cooldown service + * @param crs Repository settings CRUD + * @param csettings Cooldown settings + * @param dataSource Database data source (nullable) + * @param policy Security policy + * @checkstyle ParameterNumberCheck (5 lines) + */ + public CooldownHandler(final CooldownService cooldown, final CrudRepoSettings crs, + final CooldownSettings csettings, final DataSource dataSource, + final Policy policy) { + this.cooldown = cooldown; + this.crs = crs; + this.csettings = csettings; + this.repository = dataSource != null ? new CooldownRepository(dataSource) : null; + this.settingsDao = dataSource != null ? new SettingsDao(dataSource) : null; + this.policy = policy; + } + + /** + * Register cooldown routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + // GET /api/v1/cooldown/config — current cooldown configuration + router.get("/api/v1/cooldown/config") + .handler(new AuthzHandler(this.policy, ApiCooldownPermission.READ)) + .handler(this::getConfig); + // PUT /api/v1/cooldown/config — update cooldown configuration (hot reload) + router.put("/api/v1/cooldown/config") + .handler(new AuthzHandler(this.policy, ApiCooldownPermission.WRITE)) + .handler(this::updateConfig); + // GET /api/v1/cooldown/overview — cooldown-enabled repos + router.get("/api/v1/cooldown/overview") + .handler(new AuthzHandler(this.policy, ApiCooldownPermission.READ)) + .handler(this::overview); + // GET /api/v1/cooldown/blocked — paginated blocked list + router.get("/api/v1/cooldown/blocked") + .handler(new AuthzHandler(this.policy, ApiCooldownPermission.READ)) + .handler(this::blocked); + // POST /api/v1/repositories/:name/cooldown/unblock — unblock single artifact + router.post("/api/v1/repositories/:name/cooldown/unblock") + .handler(new AuthzHandler(this.policy, ApiCooldownPermission.WRITE)) + .handler(this::unblock); + // POST /api/v1/repositories/:name/cooldown/unblock-all — unblock all + router.post("/api/v1/repositories/:name/cooldown/unblock-all") + .handler(new AuthzHandler(this.policy, ApiCooldownPermission.WRITE)) + .handler(this::unblockAll); + } + + /** + * GET /api/v1/cooldown/config — return current cooldown configuration. + * @param ctx Routing context + */ + private void getConfig(final RoutingContext ctx) { + final JsonObject response = new JsonObject() + .put("enabled", this.csettings.enabled()) + .put("minimum_allowed_age", + CooldownHandler.formatDuration(this.csettings.minimumAllowedAge())); + final JsonObject overrides = new JsonObject(); + for (final Map.Entry entry + : this.csettings.repoTypeOverrides().entrySet()) { + overrides.put(entry.getKey(), new JsonObject() + .put("enabled", entry.getValue().enabled()) + .put("minimum_allowed_age", + CooldownHandler.formatDuration(entry.getValue().minimumAllowedAge()))); + } + response.put("repo_types", overrides); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(response.encode()); + } + + /** + * PUT /api/v1/cooldown/config — update cooldown configuration with hot reload. + * @param ctx Routing context + * @checkstyle ExecutableStatementCountCheck (60 lines) + */ + @SuppressWarnings("PMD.CognitiveComplexity") + private void updateConfig(final RoutingContext ctx) { + final JsonObject body = ctx.body().asJsonObject(); + if (body == null) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final boolean newEnabled = body.getBoolean("enabled", this.csettings.enabled()); + final Duration newAge = body.containsKey("minimum_allowed_age") + ? CooldownHandler.parseDuration(body.getString("minimum_allowed_age")) + : this.csettings.minimumAllowedAge(); + final Map overrides = new HashMap<>(); + final JsonObject repoTypes = body.getJsonObject("repo_types"); + if (repoTypes != null) { + for (final String key : repoTypes.fieldNames()) { + final JsonObject rt = repoTypes.getJsonObject(key); + overrides.put(key.toLowerCase(Locale.ROOT), + new CooldownSettings.RepoTypeConfig( + rt.getBoolean("enabled", true), + rt.containsKey("minimum_allowed_age") + ? CooldownHandler.parseDuration( + rt.getString("minimum_allowed_age")) + : newAge + )); + } + } + final boolean wasEnabled = this.csettings.enabled(); + final Map oldOverrides = + this.csettings.repoTypeOverrides(); + this.csettings.update(newEnabled, newAge, overrides); + // Auto-unblock when cooldown changes + if (this.repository != null) { + final String actor = ctx.user() != null + ? ctx.user().principal().getString(AuthTokenRest.SUB, "system") + : "system"; + if (wasEnabled && !newEnabled) { + // Global cooldown disabled — unblock everything + this.repository.unblockAll(actor); + } else if (newEnabled) { + // Check each repo type override for disable transitions + for (final Map.Entry entry + : overrides.entrySet()) { + if (!entry.getValue().enabled()) { + final CooldownSettings.RepoTypeConfig old = + oldOverrides.get(entry.getKey()); + // Unblock if was enabled (or new) and now disabled + if (old == null || old.enabled()) { + this.repository.unblockByRepoType(entry.getKey(), actor); + } + } + } + } + } + // Persist to DB if available + if (this.settingsDao != null) { + final String actor = ctx.user() != null + ? ctx.user().principal().getString(AuthTokenRest.SUB, "system") + : "system"; + final javax.json.JsonObjectBuilder jb = Json.createObjectBuilder() + .add("enabled", newEnabled) + .add("minimum_allowed_age", + CooldownHandler.formatDuration(newAge)); + if (!overrides.isEmpty()) { + final javax.json.JsonObjectBuilder rtb = Json.createObjectBuilder(); + for (final Map.Entry entry + : overrides.entrySet()) { + rtb.add(entry.getKey(), Json.createObjectBuilder() + .add("enabled", entry.getValue().enabled()) + .add("minimum_allowed_age", + CooldownHandler.formatDuration( + entry.getValue().minimumAllowedAge()))); + } + jb.add("repo_types", rtb); + } + this.settingsDao.put("cooldown", jb.build(), actor); + } + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("status", "saved").encode()); + } + + /** + * GET /api/v1/cooldown/overview — list repositories that have cooldown enabled, + * based on CooldownSettings (artipie.yml config), not just repo type. + * @param ctx Routing context + */ + private void overview(final RoutingContext ctx) { + final PermissionCollection perms = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ); + ctx.vertx().>executeBlocking( + () -> { + final Collection all = this.crs.listAll(); + final List result = new ArrayList<>(all.size()); + for (final String name : all) { + if (!perms.implies(new AdapterBasicPermission(name, "read"))) { + continue; + } + final RepositoryName rname = new RepositoryName.Simple(name); + try { + final JsonStructure config = this.crs.value(rname); + if (config == null + || !(config instanceof javax.json.JsonObject)) { + continue; + } + final javax.json.JsonObject jobj = + (javax.json.JsonObject) config; + final javax.json.JsonObject repoSection; + if (jobj.containsKey(CooldownHandler.REPO)) { + final javax.json.JsonValue rv = + jobj.get(CooldownHandler.REPO); + if (rv.getValueType() != JsonValue.ValueType.OBJECT) { + continue; + } + repoSection = (javax.json.JsonObject) rv; + } else { + repoSection = jobj; + } + final String repoType = repoSection.getString( + CooldownHandler.TYPE, "" + ); + // Check if cooldown is actually enabled for this repo type + if (!this.csettings.enabledFor(repoType)) { + continue; + } + // Only proxy repos can have cooldown + if (!repoType.endsWith("-proxy")) { + continue; + } + final Duration minAge = + this.csettings.minimumAllowedAgeFor(repoType); + final JsonObject entry = new JsonObject() + .put("name", name) + .put(CooldownHandler.TYPE, repoType) + .put("cooldown", formatDuration(minAge)); + // Add active block count if DB is available + if (this.repository != null) { + final long count = + this.repository.countActiveBlocks(repoType, name); + entry.put("active_blocks", count); + } + result.add(entry); + } catch (final Exception ex) { + // skip repos that cannot be read + } + } + return result; + }, + false + ).onSuccess( + repos -> { + final JsonArray arr = new JsonArray(); + for (final JsonObject repo : repos) { + arr.add(repo); + } + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("repos", arr).encode()); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * GET /api/v1/cooldown/blocked — paginated list of actively blocked artifacts. + * Supports server-side search via ?search= query parameter to filter by + * artifact name, repo name, or version. This avoids loading all 1M+ rows + * client-side. + * @param ctx Routing context + */ + private void blocked(final RoutingContext ctx) { + if (this.repository == null) { + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(ApiResponse.paginated(new JsonArray(), 0, 20, 0).encode()); + return; + } + final int page = ApiResponse.intParam( + ctx.queryParam("page").stream().findFirst().orElse(null), 0 + ); + final int size = ApiResponse.clampSize( + ApiResponse.intParam( + ctx.queryParam("size").stream().findFirst().orElse(null), 50 + ) + ); + final String searchQuery = ctx.queryParam("search").stream() + .findFirst().orElse(null); + final PermissionCollection perms = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ); + ctx.vertx().executeBlocking( + () -> { + final List allBlocks = + this.repository.findAllActivePaginated( + 0, Integer.MAX_VALUE, searchQuery + ); + final Instant now = Instant.now(); + final JsonArray items = new JsonArray(); + int skipped = 0; + int added = 0; + for (final DbBlockRecord rec : allBlocks) { + if (!perms.implies( + new AdapterBasicPermission(rec.repoName(), "read"))) { + continue; + } + if (skipped < page * size) { + skipped++; + continue; + } + if (added >= size) { + continue; + } + final long remainingSecs = + Duration.between(now, rec.blockedUntil()).getSeconds(); + final JsonObject item = new JsonObject() + .put("package_name", rec.artifact()) + .put("version", rec.version()) + .put("repo", rec.repoName()) + .put("repo_type", rec.repoType()) + .put("reason", rec.reason().name()) + .put("blocked_date", rec.blockedAt().toString()) + .put("blocked_until", rec.blockedUntil().toString()) + .put("remaining_hours", + Math.max(0, remainingSecs / 3600)); + items.add(item); + added++; + } + final int filteredTotal = skipped + added + + (int) allBlocks.stream() + .skip((long) skipped + added) + .filter(r -> perms.implies( + new AdapterBasicPermission(r.repoName(), "read"))) + .count(); + return ApiResponse.paginated(items, page, size, filteredTotal); + }, + false + ).onSuccess( + result -> ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(result.encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * POST /api/v1/repositories/:name/cooldown/unblock — unblock a single artifact version. + * @param ctx Routing context + * @checkstyle ExecutableStatementCountCheck (60 lines) + */ + private void unblock(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final javax.json.JsonObject body; + try { + body = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + final String artifact = body.getString("artifact", "").trim(); + final String version = body.getString("version", "").trim(); + if (artifact.isEmpty() || version.isEmpty()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "artifact and version are required"); + return; + } + final String repoType; + try { + repoType = this.repoType(rname); + } catch (final IllegalArgumentException ex) { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", ex.getMessage()); + return; + } + if (repoType.isEmpty()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Repository type is required"); + return; + } + final String actor = ctx.user().principal().getString(AuthTokenRest.SUB); + this.cooldown.unblock(repoType, name, artifact, version, actor) + .whenComplete( + (ignored, error) -> { + if (error == null) { + ctx.response().setStatusCode(204).end(); + } else { + ApiResponse.sendError( + ctx, 500, "INTERNAL_ERROR", error.getMessage() + ); + } + } + ); + } + + /** + * POST /api/v1/repositories/:name/cooldown/unblock-all — unblock all artifacts in repo. + * @param ctx Routing context + */ + private void unblockAll(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + final String repoType; + try { + repoType = this.repoType(rname); + } catch (final IllegalArgumentException ex) { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", ex.getMessage()); + return; + } + if (repoType.isEmpty()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Repository type is required"); + return; + } + final String actor = ctx.user().principal().getString(AuthTokenRest.SUB); + this.cooldown.unblockAll(repoType, name, actor) + .whenComplete( + (ignored, error) -> { + if (error == null) { + ctx.response().setStatusCode(204).end(); + } else { + ApiResponse.sendError( + ctx, 500, "INTERNAL_ERROR", error.getMessage() + ); + } + } + ); + } + + /** + * Extract repository type from config, throwing if repo not found. + * @param rname Repository name + * @return Repository type string (may be empty if not set) + * @throws IllegalArgumentException if repo does not exist or config is unreadable + */ + private String repoType(final RepositoryName rname) { + if (!this.crs.exists(rname)) { + throw new IllegalArgumentException( + String.format("Repository '%s' not found", rname) + ); + } + final JsonStructure config = this.crs.value(rname); + if (config == null) { + throw new IllegalArgumentException( + String.format("Repository '%s' not found", rname) + ); + } + if (!(config instanceof javax.json.JsonObject)) { + return ""; + } + final javax.json.JsonObject jobj = (javax.json.JsonObject) config; + if (!jobj.containsKey(CooldownHandler.REPO)) { + return ""; + } + final javax.json.JsonValue repoVal = jobj.get(CooldownHandler.REPO); + if (repoVal.getValueType() != JsonValue.ValueType.OBJECT) { + return ""; + } + return ((javax.json.JsonObject) repoVal).getString(CooldownHandler.TYPE, ""); + } + + /** + * Format duration as human-readable string (e.g. "7d", "24h", "30m"). + * @param duration Duration to format + * @return Formatted string + */ + private static String formatDuration(final Duration duration) { + final long days = duration.toDays(); + if (days > 0 && duration.equals(Duration.ofDays(days))) { + return days + "d"; + } + final long hours = duration.toHours(); + if (hours > 0 && duration.equals(Duration.ofHours(hours))) { + return hours + "h"; + } + return duration.toMinutes() + "m"; + } + + /** + * Parse duration string (e.g. "7d", "24h", "30m") to Duration. + * @param value Duration string + * @return Duration + */ + private static Duration parseDuration(final String value) { + if (value == null || value.isEmpty()) { + return Duration.ofHours(CooldownSettings.DEFAULT_HOURS); + } + final String trimmed = value.trim().toLowerCase(Locale.ROOT); + final String num = trimmed.replaceAll("[^0-9]", ""); + if (num.isEmpty()) { + return Duration.ofHours(CooldownSettings.DEFAULT_HOURS); + } + final long amount = Long.parseLong(num); + if (trimmed.endsWith("d")) { + return Duration.ofDays(amount); + } else if (trimmed.endsWith("h")) { + return Duration.ofHours(amount); + } else if (trimmed.endsWith("m")) { + return Duration.ofMinutes(amount); + } + return Duration.ofHours(amount); + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/DashboardHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/DashboardHandler.java new file mode 100644 index 000000000..239deddcf --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/DashboardHandler.java @@ -0,0 +1,221 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.RepositoryName; +import com.artipie.settings.repo.CrudRepoSettings; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import javax.json.JsonStructure; +import javax.sql.DataSource; + +/** + * Dashboard handler for /api/v1/dashboard/* endpoints. + * All endpoints use a shared 30-second in-memory cache to avoid + * expensive DB queries and YAML iterations under concurrent load. + */ +public final class DashboardHandler { + + /** + * Cache refresh interval in milliseconds (30 seconds). + */ + private static final long CACHE_TTL_MS = 30_000L; + + /** + * Repository settings CRUD. + */ + private final CrudRepoSettings crs; + + /** + * Database data source (nullable). + */ + private final DataSource dataSource; + + /** + * Cached full dashboard payload to serve all concurrent users from memory. + */ + private final AtomicReference cache = new AtomicReference<>(); + + /** + * Ctor. + * @param crs Repository settings CRUD + * @param dataSource Database data source (nullable) + */ + public DashboardHandler(final CrudRepoSettings crs, final DataSource dataSource) { + this.crs = crs; + this.dataSource = dataSource; + } + + /** + * Register dashboard routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + router.get("/api/v1/dashboard/stats").handler(this::handleStats); + router.get("/api/v1/dashboard/requests").handler(this::handleRequests); + router.get("/api/v1/dashboard/repos-by-type").handler(this::handleReposByType); + } + + /** + * GET /api/v1/dashboard/stats — aggregated statistics. + * @param ctx Routing context + */ + private void handleStats(final RoutingContext ctx) { + this.respondWithCache(ctx, CachedDashboard::stats); + } + + /** + * GET /api/v1/dashboard/repos-by-type — repo count grouped by type. + * @param ctx Routing context + */ + private void handleReposByType(final RoutingContext ctx) { + this.respondWithCache(ctx, CachedDashboard::reposByType); + } + + /** + * GET /api/v1/dashboard/requests — request rate time series (placeholder). + * @param ctx Routing context + */ + private void handleRequests(final RoutingContext ctx) { + final String period = ctx.queryParam("period").stream() + .findFirst().orElse("24h"); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end( + new JsonObject() + .put("period", period) + .put("data", new JsonArray()) + .encode() + ); + } + + /** + * Serve a dashboard response from cache. Rebuilds cache if expired. + * Only one Vert.x worker thread rebuilds the cache; others serve stale data. + * @param ctx Routing context + * @param extractor Function to extract the desired JSON from the cache + */ + private void respondWithCache(final RoutingContext ctx, + final java.util.function.Function extractor) { + ctx.vertx().executeBlocking( + () -> { + CachedDashboard cached = this.cache.get(); + if (cached == null + || System.currentTimeMillis() - cached.timestamp > CACHE_TTL_MS) { + cached = this.buildDashboard(); + this.cache.set(cached); + } + return extractor.apply(cached); + }, + false + ).onSuccess( + json -> ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(json.encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * Build the full dashboard data in a single pass. + * Runs two SQL queries + one repo config iteration. + * @return Cached dashboard snapshot + */ + @SuppressWarnings("PMD.CognitiveComplexity") + private CachedDashboard buildDashboard() { + final Collection names = this.crs.listAll(); + final int repoCount = names.size(); + // Build repos-by-type and top repos in a single pass + final Map typeCounts = new HashMap<>(16); + final JsonArray topRepos = new JsonArray(); + for (final String name : names) { + try { + final JsonStructure config = + this.crs.value(new RepositoryName.Simple(name)); + if (config instanceof javax.json.JsonObject) { + final javax.json.JsonObject jobj = (javax.json.JsonObject) config; + final javax.json.JsonObject repo = + jobj.containsKey("repo") ? jobj.getJsonObject("repo") : jobj; + final String type = repo.getString("type", "unknown"); + typeCounts.merge(type, 1, Integer::sum); + } + } catch (final Exception ignored) { + // Skip unreadable configs + } + } + long artifactCount = 0; + long totalStorage = 0; + long blockedCount = 0; + if (this.dataSource != null) { + try (Connection conn = this.dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + // Single query for artifact count + total storage + try (ResultSet rs = stmt.executeQuery( + "SELECT COUNT(*) AS cnt, COALESCE(SUM(size), 0) AS total FROM artifacts" + )) { + if (rs.next()) { + artifactCount = rs.getLong("cnt"); + totalStorage = rs.getLong("total"); + } + } + // Blocked count + try (ResultSet rs = stmt.executeQuery( + "SELECT COUNT(*) AS cnt FROM artifact_cooldowns WHERE status = 'ACTIVE'" + )) { + if (rs.next()) { + blockedCount = rs.getLong("cnt"); + } + } + // Top repos by artifact count (single query, limit 10) + try (ResultSet rs = stmt.executeQuery( + "SELECT repo_name, repo_type, COUNT(*) AS cnt, " + + "COALESCE(SUM(size), 0) AS total_size " + + "FROM artifacts GROUP BY repo_name, repo_type " + + "ORDER BY cnt DESC LIMIT 5" + )) { + while (rs.next()) { + topRepos.add(new JsonObject() + .put("name", rs.getString("repo_name")) + .put("type", rs.getString("repo_type")) + .put("artifact_count", rs.getLong("cnt")) + .put("size", rs.getLong("total_size"))); + } + } + } catch (final Exception ex) { + // DB unavailable — return zeros + } + } + // Build stats JSON + final JsonObject stats = new JsonObject() + .put("repo_count", repoCount) + .put("artifact_count", artifactCount) + .put("total_storage", totalStorage) + .put("blocked_count", blockedCount) + .put("top_repos", topRepos); + // Build types JSON + final JsonObject types = new JsonObject(); + typeCounts.forEach(types::put); + final JsonObject reposByType = new JsonObject().put("types", types); + return new CachedDashboard(stats, reposByType, System.currentTimeMillis()); + } + + /** + * Immutable snapshot of dashboard data. + */ + private record CachedDashboard(JsonObject stats, JsonObject reposByType, long timestamp) { + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/RepositoryHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/RepositoryHandler.java new file mode 100644 index 000000000..87cff2495 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/RepositoryHandler.java @@ -0,0 +1,490 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthTokenRest; +import com.artipie.api.AuthzHandler; +import com.artipie.api.RepositoryEvents; +import com.artipie.api.RepositoryName; +import com.artipie.api.perms.ApiRepositoryPermission; +import com.artipie.cooldown.CooldownService; +import com.artipie.http.auth.AuthUser; +import com.artipie.scheduling.MetadataEventQueues; +import com.artipie.security.perms.AdapterBasicPermission; +import com.artipie.security.policy.Policy; +import com.artipie.settings.RepoData; +import com.artipie.settings.cache.FiltersCache; +import com.artipie.settings.repo.CrudRepoSettings; +import io.vertx.core.eventbus.EventBus; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.io.StringReader; +import java.security.PermissionCollection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonStructure; + +/** + * Repository handler for /api/v1/repositories/* endpoints. + */ +public final class RepositoryHandler { + + /** + * JSON key for repo section. + */ + private static final String REPO = "repo"; + + /** + * Artipie filters cache. + */ + private final FiltersCache filtersCache; + + /** + * Repository settings create/read/update/delete. + */ + private final CrudRepoSettings crs; + + /** + * Repository data management. + */ + private final RepoData repoData; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Artifact metadata events queue. + */ + private final Optional events; + + /** + * Cooldown service. + */ + private final CooldownService cooldown; + + /** + * Vert.x event bus. + */ + private final EventBus eventBus; + + /** + * Ctor. + * @param filtersCache Artipie filters cache + * @param crs Repository settings CRUD + * @param repoData Repository data management + * @param policy Artipie security policy + * @param events Artifact events queue + * @param cooldown Cooldown service + * @param eventBus Vert.x event bus + * @checkstyle ParameterNumberCheck (10 lines) + */ + public RepositoryHandler(final FiltersCache filtersCache, + final CrudRepoSettings crs, final RepoData repoData, + final Policy policy, final Optional events, + final CooldownService cooldown, final EventBus eventBus) { + this.filtersCache = filtersCache; + this.crs = crs; + this.repoData = repoData; + this.policy = policy; + this.events = events; + this.cooldown = cooldown; + this.eventBus = eventBus; + } + + /** + * Register repository routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + final ApiRepositoryPermission read = + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.READ); + final ApiRepositoryPermission delete = + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.DELETE); + final ApiRepositoryPermission move = + new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.MOVE); + // GET /api/v1/repositories — paginated list + router.get("/api/v1/repositories") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::listRepositories); + // GET /api/v1/repositories/:name — get repo config + router.get("/api/v1/repositories/:name") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::getRepository); + // HEAD /api/v1/repositories/:name — check existence + router.head("/api/v1/repositories/:name") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::headRepository); + // PUT /api/v1/repositories/:name — create or update + router.put("/api/v1/repositories/:name") + .handler(this::createOrUpdateRepository); + // DELETE /api/v1/repositories/:name — delete + router.delete("/api/v1/repositories/:name") + .handler(new AuthzHandler(this.policy, delete)) + .handler(this::deleteRepository); + // PUT /api/v1/repositories/:name/move — rename/move + router.put("/api/v1/repositories/:name/move") + .handler(new AuthzHandler(this.policy, move)) + .handler(this::moveRepository); + // GET /api/v1/repositories/:name/members — group repo members + router.get("/api/v1/repositories/:name/members") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::getMembers); + } + + /** + * GET /api/v1/repositories — paginated list with optional filter/search. + * @param ctx Routing context + */ + private void listRepositories(final RoutingContext ctx) { + final int page = ApiResponse.intParam(ctx.queryParam("page").stream().findFirst().orElse(null), 0); + final int size = ApiResponse.clampSize( + ApiResponse.intParam(ctx.queryParam("size").stream().findFirst().orElse(null), 20) + ); + final String type = ctx.queryParam("type").stream().findFirst().orElse(null); + final String query = ctx.queryParam("q").stream().findFirst().orElse(null); + final PermissionCollection perms = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ); + ctx.vertx().>executeBlocking( + () -> { + final Collection all = this.crs.listAll(); + final List filtered = new ArrayList<>(all.size()); + for (final String name : all) { + if (query != null + && !name.toLowerCase(Locale.ROOT).contains(query.toLowerCase(Locale.ROOT))) { + continue; + } + if (!perms.implies(new AdapterBasicPermission(name, "read"))) { + continue; + } + String repoType = "unknown"; + try { + final javax.json.JsonStructure config = + this.crs.value(new RepositoryName.Simple(name)); + if (config instanceof javax.json.JsonObject) { + final javax.json.JsonObject jobj = (javax.json.JsonObject) config; + final javax.json.JsonObject repo = + jobj.containsKey(RepositoryHandler.REPO) + ? jobj.getJsonObject(RepositoryHandler.REPO) : jobj; + repoType = repo.getString("type", "unknown"); + } + } catch (final Exception ignored) { + // Use "unknown" type + } + if (type != null && !repoType.toLowerCase(Locale.ROOT).contains( + type.toLowerCase(Locale.ROOT))) { + continue; + } + filtered.add(new JsonObject() + .put("name", name) + .put("type", repoType)); + } + return filtered; + }, + false + ).onSuccess( + filtered -> { + final int total = filtered.size(); + final int from = Math.min(page * size, total); + final int to = Math.min(from + size, total); + final JsonArray items = new JsonArray(); + for (final JsonObject item : filtered.subList(from, to)) { + items.add(item); + } + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("items", items) + .put("page", page) + .put("size", size) + .put("total", total) + .put("hasMore", to < total) + .encode()); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * GET /api/v1/repositories/:name — get repository config. + * @param ctx Routing context + */ + private void getRepository(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + ctx.vertx().executeBlocking( + () -> { + if (!this.crs.exists(rname)) { + return null; + } + return this.crs.value(rname); + }, + false + ).onSuccess( + config -> { + if (config == null) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Repository '%s' not found", name) + ); + } else { + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(config.toString()); + } + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * HEAD /api/v1/repositories/:name — check repository existence. + * @param ctx Routing context + */ + private void headRepository(final RoutingContext ctx) { + final RepositoryName rname = new RepositoryName.Simple(ctx.pathParam("name")); + ctx.vertx().executeBlocking( + () -> this.crs.exists(rname), + false + ).onSuccess( + exists -> { + if (Boolean.TRUE.equals(exists)) { + ctx.response().setStatusCode(200).end(); + } else { + ctx.response().setStatusCode(404).end(); + } + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/repositories/:name — create or update repository. + * @param ctx Routing context + * @checkstyle ExecutableStatementCountCheck (60 lines) + */ + private void createOrUpdateRepository(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final javax.json.JsonObject body; + try { + body = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + if (!body.containsKey(RepositoryHandler.REPO) + || body.getJsonObject(RepositoryHandler.REPO) == null) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Section `repo` is required"); + return; + } + final javax.json.JsonObject repo = body.getJsonObject(RepositoryHandler.REPO); + if (!repo.containsKey("type")) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Repository type is required"); + return; + } + if (!repo.containsKey("storage")) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Repository storage is required"); + return; + } + final boolean exists = this.crs.exists(rname); + final ApiRepositoryPermission needed; + if (exists) { + needed = new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.UPDATE); + } else { + needed = new ApiRepositoryPermission(ApiRepositoryPermission.RepositoryAction.CREATE); + } + final boolean allowed = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ).implies(needed); + if (!allowed) { + ApiResponse.sendError(ctx, 403, "FORBIDDEN", "Insufficient permissions"); + return; + } + final String actor = ctx.user().principal().getString(AuthTokenRest.SUB); + ctx.vertx().executeBlocking( + () -> { + this.crs.save(rname, body, actor); + return null; + }, + false + ).onSuccess( + ignored -> { + this.filtersCache.invalidate(rname.toString()); + this.eventBus.publish(RepositoryEvents.ADDRESS, RepositoryEvents.upsert(name)); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * DELETE /api/v1/repositories/:name — delete repository. + * @param ctx Routing context + */ + private void deleteRepository(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + ctx.vertx().executeBlocking( + () -> this.crs.exists(rname), + false + ).onSuccess( + exists -> { + if (!Boolean.TRUE.equals(exists)) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Repository '%s' not found", name) + ); + return; + } + this.repoData.remove(rname) + .thenRun(() -> this.crs.delete(rname)) + .exceptionally(exc -> { + this.crs.delete(rname); + return null; + }); + this.filtersCache.invalidate(rname.toString()); + this.eventBus.publish(RepositoryEvents.ADDRESS, RepositoryEvents.remove(name)); + this.events.ifPresent(item -> item.stopProxyMetadataProcessing(name)); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/repositories/:name/move — rename/move repository. + * @param ctx Routing context + */ + private void moveRepository(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final javax.json.JsonObject body; + try { + body = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + final String newName = body.getString("new_name", "").trim(); + if (newName.isEmpty()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "new_name is required"); + return; + } + ctx.vertx().executeBlocking( + () -> this.crs.exists(rname), + false + ).onSuccess( + exists -> { + if (!Boolean.TRUE.equals(exists)) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Repository '%s' not found", name) + ); + return; + } + final RepositoryName newrname = new RepositoryName.Simple(newName); + this.repoData.move(rname, newrname) + .thenRun(() -> this.crs.move(rname, newrname)); + this.filtersCache.invalidate(rname.toString()); + this.eventBus.publish( + RepositoryEvents.ADDRESS, RepositoryEvents.move(name, newName) + ); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * GET /api/v1/repositories/:name/members — get group repository members. + * @param ctx Routing context + */ + private void getMembers(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final RepositoryName rname = new RepositoryName.Simple(name); + ctx.vertx().executeBlocking( + () -> { + if (!this.crs.exists(rname)) { + return null; + } + final JsonStructure config = this.crs.value(rname); + if (config == null) { + return null; + } + final javax.json.JsonObject jconfig; + if (config instanceof javax.json.JsonObject) { + jconfig = (javax.json.JsonObject) config; + } else { + return new JsonObject().put("members", new JsonArray()).put("type", "not-a-group"); + } + final javax.json.JsonObject repoSection = jconfig.containsKey(RepositoryHandler.REPO) + ? jconfig.getJsonObject(RepositoryHandler.REPO) : jconfig; + final String repoType = repoSection.getString("type", ""); + if (!repoType.endsWith("-group")) { + return new JsonObject().put("members", new JsonArray()).put("type", "not-a-group"); + } + final JsonArray members = new JsonArray(); + if (repoSection.containsKey("remotes")) { + final javax.json.JsonArray remotes = repoSection.getJsonArray("remotes"); + for (int idx = 0; idx < remotes.size(); idx++) { + final javax.json.JsonObject remote = remotes.getJsonObject(idx); + members.add(remote.getString("url", remote.toString())); + } + } + return new JsonObject().put("members", members).put("type", repoType); + }, + false + ).onSuccess( + result -> { + if (result == null) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Repository '%s' not found", name) + ); + } else { + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(result.encode()); + } + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/RoleHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/RoleHandler.java new file mode 100644 index 000000000..b45aa542c --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/RoleHandler.java @@ -0,0 +1,313 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthTokenRest; +import com.artipie.api.AuthzHandler; +import com.artipie.api.perms.ApiRolePermission; +import com.artipie.api.perms.ApiRolePermission.RoleAction; +import com.artipie.asto.misc.Cleanable; +import com.artipie.http.auth.AuthUser; +import com.artipie.security.policy.Policy; +import com.artipie.settings.users.CrudRoles; +import io.vertx.core.json.JsonArray; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.io.StringReader; +import java.security.PermissionCollection; +import java.util.List; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonObject; + +/** + * Role handler for /api/v1/roles/* endpoints. + * @since 1.21 + */ +public final class RoleHandler { + + /** + * Role name path parameter. + */ + private static final String NAME = "name"; + + /** + * Update role permission constant. + */ + private static final ApiRolePermission UPDATE = + new ApiRolePermission(RoleAction.UPDATE); + + /** + * Create role permission constant. + */ + private static final ApiRolePermission CREATE = + new ApiRolePermission(RoleAction.CREATE); + + /** + * Crud roles object. + */ + private final CrudRoles roles; + + /** + * Artipie policy cache. + */ + private final Cleanable policyCache; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Ctor. + * @param roles Crud roles object + * @param policyCache Artipie policy cache + * @param policy Artipie security policy + */ + public RoleHandler(final CrudRoles roles, final Cleanable policyCache, + final Policy policy) { + this.roles = roles; + this.policyCache = policyCache; + this.policy = policy; + } + + /** + * Register role routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + final ApiRolePermission read = new ApiRolePermission(RoleAction.READ); + final ApiRolePermission delete = new ApiRolePermission(RoleAction.DELETE); + final ApiRolePermission enable = new ApiRolePermission(RoleAction.ENABLE); + // GET /api/v1/roles — paginated list + router.get("/api/v1/roles") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::listRoles); + // GET /api/v1/roles/:name — get single role + router.get("/api/v1/roles/:name") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::getRole); + // PUT /api/v1/roles/:name — create or update role + router.put("/api/v1/roles/:name") + .handler(this::putRole); + // DELETE /api/v1/roles/:name — delete role + router.delete("/api/v1/roles/:name") + .handler(new AuthzHandler(this.policy, delete)) + .handler(this::deleteRole); + // POST /api/v1/roles/:name/enable — enable role + router.post("/api/v1/roles/:name/enable") + .handler(new AuthzHandler(this.policy, enable)) + .handler(this::enableRole); + // POST /api/v1/roles/:name/disable — disable role + router.post("/api/v1/roles/:name/disable") + .handler(new AuthzHandler(this.policy, enable)) + .handler(this::disableRole); + } + + /** + * GET /api/v1/roles — paginated list of roles. + * @param ctx Routing context + */ + private void listRoles(final RoutingContext ctx) { + final int page = ApiResponse.intParam( + ctx.queryParam("page").stream().findFirst().orElse(null), 0 + ); + final int size = ApiResponse.clampSize( + ApiResponse.intParam( + ctx.queryParam("size").stream().findFirst().orElse(null), 20 + ) + ); + ctx.vertx().executeBlocking( + this.roles::list, + false + ).onSuccess( + all -> { + final List flat = + new java.util.ArrayList<>(all.size()); + for (int i = 0; i < all.size(); i++) { + flat.add( + new io.vertx.core.json.JsonObject( + all.getJsonObject(i).toString() + ) + ); + } + final JsonArray items = ApiResponse.sliceToArray(flat, page, size); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(ApiResponse.paginated(items, page, size, flat.size()).encode()); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * GET /api/v1/roles/:name — get single role info. + * @param ctx Routing context + */ + private void getRole(final RoutingContext ctx) { + final String rname = ctx.pathParam(RoleHandler.NAME); + ctx.vertx().>executeBlocking( + () -> this.roles.get(rname), + false + ).onSuccess( + opt -> { + if (opt.isPresent()) { + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(opt.get().toString()); + } else { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Role '%s' not found", rname) + ); + } + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/roles/:name — create or update role. + * @param ctx Routing context + */ + private void putRole(final RoutingContext ctx) { + final String rname = ctx.pathParam(RoleHandler.NAME); + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final JsonObject body; + try { + body = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + final Optional existing = this.roles.get(rname); + final PermissionCollection perms = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ); + if (existing.isPresent() && perms.implies(RoleHandler.UPDATE) + || existing.isEmpty() && perms.implies(RoleHandler.CREATE)) { + ctx.vertx().executeBlocking( + () -> { + this.roles.addOrUpdate(body, rname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.policyCache.invalidate(rname); + ctx.response().setStatusCode(201).end(); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } else { + ApiResponse.sendError(ctx, 403, "FORBIDDEN", "Insufficient permissions"); + } + } + + /** + * DELETE /api/v1/roles/:name — delete role. + * @param ctx Routing context + */ + private void deleteRole(final RoutingContext ctx) { + final String rname = ctx.pathParam(RoleHandler.NAME); + ctx.vertx().executeBlocking( + () -> { + this.roles.remove(rname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.policyCache.invalidate(rname); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Role '%s' not found", rname) + ); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } + + /** + * POST /api/v1/roles/:name/enable — enable role. + * @param ctx Routing context + */ + private void enableRole(final RoutingContext ctx) { + final String rname = ctx.pathParam(RoleHandler.NAME); + ctx.vertx().executeBlocking( + () -> { + this.roles.enable(rname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.policyCache.invalidate(rname); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Role '%s' not found", rname) + ); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } + + /** + * POST /api/v1/roles/:name/disable — disable role. + * @param ctx Routing context + */ + private void disableRole(final RoutingContext ctx) { + final String rname = ctx.pathParam(RoleHandler.NAME); + ctx.vertx().executeBlocking( + () -> { + this.roles.disable(rname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.policyCache.invalidate(rname); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("Role '%s' not found", rname) + ); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/SearchHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/SearchHandler.java new file mode 100644 index 000000000..7ed81d7d2 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/SearchHandler.java @@ -0,0 +1,276 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthTokenRest; +import com.artipie.api.AuthzHandler; +import com.artipie.api.perms.ApiSearchPermission; +import com.artipie.http.auth.AuthUser; +import com.artipie.http.log.EcsLogger; +import com.artipie.index.ArtifactIndex; +import com.artipie.security.perms.AdapterBasicPermission; +import com.artipie.security.policy.Policy; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.security.PermissionCollection; +import java.util.Objects; +import org.eclipse.jetty.http.HttpStatus; + +/** + * Search handler for /api/v1/search/* endpoints. + * + *

Endpoints:

+ *
    + *
  • GET /api/v1/search?q={query}&page={0}&size={20} — paginated search
    • + *
  • GET /api/v1/search/locate?path={path} — locate repos containing artifact
    • + *
  • POST /api/v1/search/reindex — trigger full reindex (202)
    • + *
  • GET /api/v1/search/stats — index statistics
    • + *
+ * + * @since 1.21.0 + */ +public final class SearchHandler { + + /** + * Artifact index. + */ + private final ArtifactIndex index; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Ctor. + * @param index Artifact index + * @param policy Artipie security policy + */ + public SearchHandler(final ArtifactIndex index, final Policy policy) { + this.index = Objects.requireNonNull(index, "index"); + this.policy = Objects.requireNonNull(policy, "policy"); + } + + /** + * Register search routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + // GET /api/v1/search/locate — must be registered before /api/v1/search + // to avoid ambiguity with the wildcard suffix + router.get("/api/v1/search/locate") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.READ)) + .handler(this::locate); + // GET /api/v1/search/stats + router.get("/api/v1/search/stats") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.READ)) + .handler(this::stats); + // POST /api/v1/search/reindex + router.post("/api/v1/search/reindex") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.WRITE)) + .handler(this::reindex); + // GET /api/v1/search + router.get("/api/v1/search") + .handler(new AuthzHandler(this.policy, ApiSearchPermission.READ)) + .handler(this::search); + } + + /** + * Paginated full-text search handler. + * @param ctx Routing context + */ + private void search(final RoutingContext ctx) { + final String query = ctx.queryParams().get("q"); + if (query == null || query.isBlank()) { + ctx.response() + .setStatusCode(HttpStatus.BAD_REQUEST_400) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.BAD_REQUEST_400) + .put("message", "Missing 'q' parameter") + .encode()); + return; + } + final int page = SearchHandler.intParam(ctx, "page", 0); + final int size = SearchHandler.intParam(ctx, "size", 20); + final int offset = page * size; + final PermissionCollection perms = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ); + this.index.search(query, size, offset).whenComplete((result, error) -> { + if (error != null) { + ctx.response() + .setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.INTERNAL_SERVER_ERROR_500) + .put("message", error.getMessage()) + .encode()); + } else { + final JsonArray items = new JsonArray(); + result.documents().forEach(doc -> { + if (!perms.implies( + new AdapterBasicPermission(doc.repoName(), "read"))) { + return; + } + final JsonObject obj = new JsonObject() + .put("repo_type", doc.repoType()) + .put("repo_name", doc.repoName()) + .put("artifact_path", doc.artifactPath()); + if (doc.artifactName() != null) { + obj.put("artifact_name", doc.artifactName()); + } + if (doc.version() != null) { + obj.put("version", doc.version()); + } + obj.put("size", doc.size()); + if (doc.createdAt() != null) { + obj.put("created_at", doc.createdAt().toString()); + } + if (doc.owner() != null) { + obj.put("owner", doc.owner()); + } + items.add(obj); + }); + ctx.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("items", items) + .put("page", page) + .put("size", size) + .put("total", items.size()) + .put("hasMore", false) + .encode()); + } + }); + } + + /** + * Locate repos containing an artifact. + * @param ctx Routing context + */ + private void locate(final RoutingContext ctx) { + final String path = ctx.queryParams().get("path"); + if (path == null || path.isBlank()) { + ctx.response() + .setStatusCode(HttpStatus.BAD_REQUEST_400) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.BAD_REQUEST_400) + .put("message", "Missing 'path' parameter") + .encode()); + return; + } + final PermissionCollection perms = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ); + this.index.locate(path).whenComplete((repos, error) -> { + if (error != null) { + ctx.response() + .setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.INTERNAL_SERVER_ERROR_500) + .put("message", error.getMessage()) + .encode()); + } else { + final java.util.List allowed = repos.stream() + .filter(r -> perms.implies(new AdapterBasicPermission(r, "read"))) + .toList(); + ctx.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("repositories", new JsonArray(allowed)) + .put("count", allowed.size()) + .encode()); + } + }); + } + + /** + * Trigger a full reindex (async, returns 202). + * @param ctx Routing context + */ + private void reindex(final RoutingContext ctx) { + EcsLogger.info("com.artipie.api.v1") + .message("Full reindex triggered via API") + .eventCategory("search") + .eventAction("reindex") + .field("user.name", + ctx.user() != null + ? ctx.user().principal().getString(AuthTokenRest.SUB) + : null) + .log(); + ctx.response() + .setStatusCode(HttpStatus.ACCEPTED_202) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("status", "started") + .put("message", "Full reindex initiated") + .encode()); + } + + /** + * Index statistics handler. + * @param ctx Routing context + */ + private void stats(final RoutingContext ctx) { + this.index.getStats().whenComplete((map, error) -> { + if (error != null) { + ctx.response() + .setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) + .putHeader("Content-Type", "application/json") + .end(new JsonObject() + .put("code", HttpStatus.INTERNAL_SERVER_ERROR_500) + .put("message", error.getMessage()) + .encode()); + } else { + final JsonObject json = new JsonObject(); + map.forEach((key, value) -> { + if (value instanceof Number) { + json.put(key, ((Number) value).longValue()); + } else if (value instanceof Boolean) { + json.put(key, (Boolean) value); + } else { + json.put(key, String.valueOf(value)); + } + }); + ctx.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(json.encode()); + } + }); + } + + /** + * Parse int query parameter with default. + * @param ctx Routing context + * @param name Parameter name + * @param def Default value + * @return Parsed value or default + */ + private static int intParam(final RoutingContext ctx, final String name, final int def) { + final String val = ctx.queryParams().get(name); + if (val == null || val.isBlank()) { + return def; + } + try { + return Integer.parseInt(val); + } catch (final NumberFormatException ex) { + return def; + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/SettingsHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/SettingsHandler.java new file mode 100644 index 000000000..b0dfbef68 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/SettingsHandler.java @@ -0,0 +1,428 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthzHandler; +import com.artipie.api.ManageRepoSettings; +import com.artipie.api.perms.ApiRolePermission; +import com.artipie.cooldown.CooldownSettings; +import com.artipie.db.dao.AuthProviderDao; +import com.artipie.db.dao.SettingsDao; +import com.artipie.http.client.HttpClientSettings; +import com.artipie.misc.ArtipieProperties; +import com.artipie.security.policy.Policy; +import com.artipie.settings.JwtSettings; +import com.artipie.settings.MetricsContext; +import com.artipie.settings.PrefixesPersistence; +import com.artipie.settings.Settings; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.json.Json; +import javax.sql.DataSource; +import org.eclipse.jetty.http.HttpStatus; + +/** + * Settings handler for /api/v1/settings/* endpoints. + * Exposes all artipie.yml configuration sections with resolved environment variables. + * @since 1.21 + * @checkstyle ClassDataAbstractionCouplingCheck (500 lines) + * @checkstyle ExecutableStatementCountCheck (500 lines) + */ +public final class SettingsHandler { + + /** + * Artipie port. + */ + private final int port; + + /** + * Artipie settings. + */ + private final Settings settings; + + /** + * Repository settings manager. + */ + private final ManageRepoSettings manageRepo; + + /** + * Settings DAO for database persistence (nullable). + */ + private final SettingsDao settingsDao; + + /** + * Auth provider DAO (nullable). + */ + private final AuthProviderDao authProviderDao; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Ctor. + * @param port Artipie port + * @param settings Artipie settings + * @param manageRepo Repository settings manager + * @param dataSource Database data source (nullable) + * @param policy Security policy + * @checkstyle ParameterNumberCheck (5 lines) + */ + public SettingsHandler(final int port, final Settings settings, + final ManageRepoSettings manageRepo, final DataSource dataSource, + final Policy policy) { + this.port = port; + this.settings = settings; + this.manageRepo = manageRepo; + this.settingsDao = dataSource != null ? new SettingsDao(dataSource) : null; + this.authProviderDao = dataSource != null ? new AuthProviderDao(dataSource) : null; + this.policy = policy; + } + + /** + * Register settings routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + final ApiRolePermission read = + new ApiRolePermission(ApiRolePermission.RoleAction.READ); + final ApiRolePermission update = + new ApiRolePermission(ApiRolePermission.RoleAction.UPDATE); + router.get("/api/v1/settings") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::getSettings); + router.put("/api/v1/settings/prefixes") + .handler(new AuthzHandler(this.policy, update)) + .handler(this::updatePrefixes); + router.put("/api/v1/settings/:section") + .handler(new AuthzHandler(this.policy, update)) + .handler(this::updateSection); + // Auth provider management + router.put("/api/v1/auth-providers/:id/toggle") + .handler(new AuthzHandler(this.policy, update)) + .handler(this::toggleAuthProvider); + router.put("/api/v1/auth-providers/:id/config") + .handler(new AuthzHandler(this.policy, update)) + .handler(this::updateAuthProviderConfig); + } + + /** + * GET /api/v1/settings — full settings with all sections. + * @param ctx Routing context + */ + private void getSettings(final RoutingContext ctx) { + ctx.vertx().executeBlocking( + () -> this.buildFullSettings(), + false + ).onSuccess( + result -> ctx.response() + .setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(result.encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * Build the full settings JSON from all sources. + * @return Complete settings JSON + */ + private JsonObject buildFullSettings() { + final JsonObject response = new JsonObject() + .put("port", this.port) + .put("version", new ArtipieProperties().version()); + // Prefixes + try { + response.put("prefixes", new JsonArray(this.settings.prefixes().prefixes())); + } catch (final Exception ex) { + response.put("prefixes", new JsonArray()); + } + // JWT + final JwtSettings jwt = this.settings.jwtSettings(); + response.put("jwt", new JsonObject() + .put("expires", jwt.expires()) + .put("expiry_seconds", jwt.expirySeconds()) + ); + // HTTP Client + final HttpClientSettings hc = this.settings.httpClientSettings(); + response.put("http_client", new JsonObject() + .put("proxy_timeout", hc.proxyTimeout()) + .put("connection_timeout", hc.connectTimeout()) + .put("idle_timeout", hc.idleTimeout()) + .put("follow_redirects", hc.followRedirects()) + .put("connection_acquire_timeout", hc.connectionAcquireTimeout()) + .put("max_connections_per_destination", hc.maxConnectionsPerDestination()) + .put("max_requests_queued_per_destination", hc.maxRequestsQueuedPerDestination()) + ); + // HTTP Server + final Duration reqTimeout = this.settings.httpServerRequestTimeout(); + response.put("http_server", new JsonObject() + .put("request_timeout", reqTimeout.toString()) + ); + // Metrics + final MetricsContext metrics = this.settings.metrics(); + final JsonObject metricsJson = new JsonObject() + .put("enabled", metrics.enabled()) + .put("jvm", metrics.jvm()) + .put("http", metrics.http()) + .put("storage", metrics.storage()); + metrics.endpointAndPort().ifPresent(pair -> { + metricsJson.put("endpoint", pair.getLeft()); + metricsJson.put("port", pair.getRight()); + }); + response.put("metrics", metricsJson); + // Cooldown + final CooldownSettings cd = this.settings.cooldown(); + final JsonObject cooldownJson = new JsonObject() + .put("enabled", cd.enabled()) + .put("minimum_allowed_age", cd.minimumAllowedAge().toString()); + response.put("cooldown", cooldownJson); + // Credentials / auth providers + if (this.authProviderDao != null) { + final List providers = this.authProviderDao.list(); + final JsonArray providersArr = new JsonArray(); + for (final javax.json.JsonObject prov : providers) { + final JsonObject entry = new JsonObject() + .put("id", prov.getInt("id")) + .put("type", prov.getString("type")) + .put("priority", prov.getInt("priority")) + .put("enabled", prov.getBoolean("enabled")); + // Include safe config (strip secrets, handle nested values) + final javax.json.JsonObject cfg = prov.getJsonObject("config"); + if (cfg != null) { + final JsonObject safeConfig = new JsonObject(); + for (final String key : cfg.keySet()) { + final javax.json.JsonValue jval = cfg.get(key); + if (jval.getValueType() == javax.json.JsonValue.ValueType.STRING) { + final String val = cfg.getString(key); + if (isSecret(key)) { + safeConfig.put(key, maskValue(val)); + } else { + safeConfig.put(key, val); + } + } else if (jval.getValueType() == javax.json.JsonValue.ValueType.OBJECT + || jval.getValueType() == javax.json.JsonValue.ValueType.ARRAY) { + if (isSecret(key)) { + safeConfig.put(key, "***"); + } else { + safeConfig.put(key, jval.toString()); + } + } else { + safeConfig.put(key, jval.toString()); + } + } + entry.put("config", safeConfig); + } + providersArr.add(entry); + } + response.put("credentials", providersArr); + } + // Database info (connection status, not secrets) + response.put("database", new JsonObject() + .put("configured", this.settings.artifactsDatabase().isPresent()) + ); + // Valkey/cache info + response.put("caches", new JsonObject() + .put("valkey_configured", this.settings.valkeyConnection().isPresent()) + ); + return response; + } + + /** + * PUT /api/v1/settings/prefixes — update global prefixes. + * @param ctx Routing context + */ + private void updatePrefixes(final RoutingContext ctx) { + try { + final JsonObject body = ctx.body().asJsonObject(); + if (body == null || !body.containsKey("prefixes")) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Missing 'prefixes' field"); + return; + } + final JsonArray prefixesArray = body.getJsonArray("prefixes"); + final List prefixes = new ArrayList<>(prefixesArray.size()); + for (int idx = 0; idx < prefixesArray.size(); idx++) { + prefixes.add(prefixesArray.getString(idx)); + } + this.settings.prefixes().update(prefixes); + new PrefixesPersistence(this.settings.configPath()).save(prefixes); + // Also persist to database if available + if (this.settingsDao != null) { + final String actor = ctx.user() != null + ? ctx.user().principal().getString("sub", "system") : "system"; + this.settingsDao.put("prefixes", + Json.createObjectBuilder() + .add("prefixes", Json.createArrayBuilder(prefixes)) + .build(), + actor + ); + } + ctx.response().setStatusCode(HttpStatus.OK_200).end(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", ex.getMessage()); + } + } + + /** + * PUT /api/v1/settings/:section — update a specific settings section. + * Persists to database via SettingsDao. + * @param ctx Routing context + */ + private void updateSection(final RoutingContext ctx) { + final String section = ctx.pathParam("section"); + if ("prefixes".equals(section)) { + this.updatePrefixes(ctx); + return; + } + if (this.settingsDao == null) { + ApiResponse.sendError(ctx, 503, "UNAVAILABLE", + "Database not configured; settings updates require database"); + return; + } + final JsonObject body = ctx.body().asJsonObject(); + if (body == null) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final String actor = ctx.user() != null + ? ctx.user().principal().getString("sub", "system") : "system"; + ctx.vertx().executeBlocking( + () -> { + // Convert vertx JsonObject to javax.json.JsonObject + final javax.json.JsonObject jobj = Json.createReader( + new java.io.StringReader(body.encode()) + ).readObject(); + this.settingsDao.put(section, jobj, actor); + return null; + }, + false + ).onSuccess( + ignored -> ctx.response().setStatusCode(HttpStatus.OK_200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("status", "saved").encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/auth-providers/:id/toggle — enable or disable an auth provider. + * @param ctx Routing context + */ + private void toggleAuthProvider(final RoutingContext ctx) { + if (this.authProviderDao == null) { + ApiResponse.sendError(ctx, 503, "UNAVAILABLE", + "Database not configured"); + return; + } + final int providerId; + try { + providerId = Integer.parseInt(ctx.pathParam("id")); + } catch (final NumberFormatException ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid provider ID"); + return; + } + final JsonObject body = ctx.body().asJsonObject(); + if (body == null || !body.containsKey("enabled")) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Missing 'enabled' field"); + return; + } + final boolean enabled = body.getBoolean("enabled"); + ctx.vertx().executeBlocking( + () -> { + if (enabled) { + this.authProviderDao.enable(providerId); + } else { + this.authProviderDao.disable(providerId); + } + return null; + }, + false + ).onSuccess( + ignored -> ctx.response().setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("status", "saved").encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/auth-providers/:id/config — update an auth provider's config. + * @param ctx Routing context + */ + private void updateAuthProviderConfig(final RoutingContext ctx) { + if (this.authProviderDao == null) { + ApiResponse.sendError(ctx, 503, "UNAVAILABLE", + "Database not configured"); + return; + } + final int providerId; + try { + providerId = Integer.parseInt(ctx.pathParam("id")); + } catch (final NumberFormatException ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid provider ID"); + return; + } + final JsonObject body = ctx.body().asJsonObject(); + if (body == null) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + ctx.vertx().executeBlocking( + () -> { + final javax.json.JsonObject jobj = Json.createReader( + new java.io.StringReader(body.encode()) + ).readObject(); + this.authProviderDao.updateConfig(providerId, jobj); + return null; + }, + false + ).onSuccess( + ignored -> ctx.response().setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(new JsonObject().put("status", "saved").encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * Check if a config key likely contains a secret. + * @param key Config key name + * @return True if secret + */ + private static boolean isSecret(final String key) { + final String lower = key.toLowerCase(); + return lower.contains("secret") || lower.contains("password") + || lower.contains("token") || lower.contains("key"); + } + + /** + * Mask a secret value, showing only first/last 2 chars if long enough. + * @param value Original value + * @return Masked string + */ + private static String maskValue(final String value) { + if (value == null || value.isEmpty()) { + return "***"; + } + if (value.startsWith("${") && value.endsWith("}")) { + return value; + } + if (value.length() <= 6) { + return "***"; + } + return value.substring(0, 2) + "***" + value.substring(value.length() - 2); + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/StorageAliasHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/StorageAliasHandler.java new file mode 100644 index 000000000..41377edae --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/StorageAliasHandler.java @@ -0,0 +1,372 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthzHandler; +import com.artipie.api.ManageStorageAliases; +import com.artipie.api.perms.ApiAliasPermission; +import com.artipie.asto.Key; +import com.artipie.asto.blocking.BlockingStorage; +import com.artipie.cache.StoragesCache; +import com.artipie.db.dao.StorageAliasDao; +import com.artipie.security.policy.Policy; +import io.vertx.core.json.JsonArray; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.io.StringReader; +import java.util.Collection; +import java.util.List; +import javax.json.Json; +import javax.json.JsonObject; + +/** + * Storage alias handler for /api/v1/storages/* and + * /api/v1/repositories/:name/storages/* endpoints. + */ +public final class StorageAliasHandler { + + /** + * Artipie settings storage cache. + */ + private final StoragesCache storagesCache; + + /** + * Artipie settings storage. + */ + private final BlockingStorage asto; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Storage alias DAO (nullable — present only when DB is configured). + */ + private final StorageAliasDao aliasDao; + + /** + * Ctor. + * @param storagesCache Artipie settings storage cache + * @param asto Artipie settings storage + * @param policy Artipie security policy + * @param aliasDao Storage alias DAO, nullable + */ + public StorageAliasHandler(final StoragesCache storagesCache, + final BlockingStorage asto, final Policy policy, + final StorageAliasDao aliasDao) { + this.storagesCache = storagesCache; + this.asto = asto; + this.policy = policy; + this.aliasDao = aliasDao; + } + + /** + * Register storage alias routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + final ApiAliasPermission read = + new ApiAliasPermission(ApiAliasPermission.AliasAction.READ); + final ApiAliasPermission create = + new ApiAliasPermission(ApiAliasPermission.AliasAction.CREATE); + final ApiAliasPermission delete = + new ApiAliasPermission(ApiAliasPermission.AliasAction.DELETE); + // GET /api/v1/storages — list global aliases + router.get("/api/v1/storages") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::listGlobalAliases); + // PUT /api/v1/storages/:name — create/update global alias + router.put("/api/v1/storages/:name") + .handler(new AuthzHandler(this.policy, create)) + .handler(this::putGlobalAlias); + // DELETE /api/v1/storages/:name — delete global alias + router.delete("/api/v1/storages/:name") + .handler(new AuthzHandler(this.policy, delete)) + .handler(this::deleteGlobalAlias); + // GET /api/v1/repositories/:name/storages — list per-repo aliases + router.get("/api/v1/repositories/:name/storages") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::listRepoAliases); + // PUT /api/v1/repositories/:name/storages/:alias — create/update repo alias + router.put("/api/v1/repositories/:name/storages/:alias") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::putRepoAlias); + // DELETE /api/v1/repositories/:name/storages/:alias — delete repo alias + router.delete("/api/v1/repositories/:name/storages/:alias") + .handler(new AuthzHandler(this.policy, delete)) + .handler(this::deleteRepoAlias); + } + + /** + * GET /api/v1/storages — list global storage aliases. + * Reads from DB when available, falls back to YAML. + * @param ctx Routing context + */ + private void listGlobalAliases(final RoutingContext ctx) { + ctx.vertx().executeBlocking( + () -> { + if (this.aliasDao != null) { + return aliasesToArray(this.aliasDao.listGlobal()); + } + return yamlAliasesToArray(new ManageStorageAliases(this.asto).list()); + }, + false + ).onSuccess( + arr -> ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(arr.encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/storages/:name — create or update a global alias. + * Writes to both DB and YAML for dual persistence. + * @param ctx Routing context + */ + private void putGlobalAlias(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + final JsonObject body = bodyAsJson(ctx); + if (body == null) { + return; + } + ctx.vertx().executeBlocking( + () -> { + if (this.aliasDao != null) { + this.aliasDao.put(name, null, body); + } + try { + new ManageStorageAliases(this.asto).add(name, body); + } catch (final Exception ignored) { + // YAML write is best-effort when DB is primary + } + this.storagesCache.invalidateAll(); + return null; + }, + false + ).onSuccess( + ignored -> ctx.response().setStatusCode(200).end() + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * DELETE /api/v1/storages/:name — delete a global alias. + * Checks for dependent repositories when aliasDao is present. + * @param ctx Routing context + */ + private void deleteGlobalAlias(final RoutingContext ctx) { + final String name = ctx.pathParam("name"); + ctx.vertx().executeBlocking( + () -> { + if (this.aliasDao != null) { + final List repos = this.aliasDao.findReposUsing(name); + if (repos != null && !repos.isEmpty()) { + throw new DependencyException( + String.format( + "Cannot delete alias '%s': used by repositories: %s", + name, String.join(", ", repos) + ) + ); + } + this.aliasDao.delete(name, null); + } + try { + new ManageStorageAliases(this.asto).remove(name); + } catch (final Exception ignored) { + // YAML delete is best-effort when DB is primary + } + this.storagesCache.invalidateAll(); + return null; + }, + false + ).onSuccess( + ignored -> ctx.response().setStatusCode(200).end() + ).onFailure( + err -> { + if (err instanceof DependencyException) { + ApiResponse.sendError(ctx, 409, "CONFLICT", err.getMessage()); + } else if (err instanceof IllegalStateException) { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", err.getMessage()); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } + + /** + * GET /api/v1/repositories/:name/storages — list per-repo aliases. + * Reads from DB when available, falls back to YAML. + * @param ctx Routing context + */ + private void listRepoAliases(final RoutingContext ctx) { + final String repoName = ctx.pathParam("name"); + ctx.vertx().executeBlocking( + () -> { + if (this.aliasDao != null) { + return aliasesToArray(this.aliasDao.listForRepo(repoName)); + } + return yamlAliasesToArray( + new ManageStorageAliases(new Key.From(repoName), this.asto).list() + ); + }, + false + ).onSuccess( + arr -> ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(arr.encode()) + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/repositories/:name/storages/:alias — create or update a repo alias. + * Writes to both DB and YAML for dual persistence. + * @param ctx Routing context + */ + private void putRepoAlias(final RoutingContext ctx) { + final String repoName = ctx.pathParam("name"); + final String aliasName = ctx.pathParam("alias"); + final JsonObject body = bodyAsJson(ctx); + if (body == null) { + return; + } + ctx.vertx().executeBlocking( + () -> { + if (this.aliasDao != null) { + this.aliasDao.put(aliasName, repoName, body); + } + try { + new ManageStorageAliases(new Key.From(repoName), this.asto) + .add(aliasName, body); + } catch (final Exception ignored) { + // YAML write is best-effort when DB is primary + } + this.storagesCache.invalidateAll(); + return null; + }, + false + ).onSuccess( + ignored -> ctx.response().setStatusCode(200).end() + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * DELETE /api/v1/repositories/:name/storages/:alias — delete a repo alias. + * @param ctx Routing context + */ + private void deleteRepoAlias(final RoutingContext ctx) { + final String repoName = ctx.pathParam("name"); + final String aliasName = ctx.pathParam("alias"); + ctx.vertx().executeBlocking( + () -> { + if (this.aliasDao != null) { + this.aliasDao.delete(aliasName, repoName); + } + try { + new ManageStorageAliases(new Key.From(repoName), this.asto) + .remove(aliasName); + } catch (final Exception ignored) { + // YAML delete is best-effort when DB is primary + } + this.storagesCache.invalidateAll(); + return null; + }, + false + ).onSuccess( + ignored -> ctx.response().setStatusCode(200).end() + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError(ctx, 404, "NOT_FOUND", err.getMessage()); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } + + /** + * Convert DB alias entries (with "name" and "config" keys) to a Vert.x JsonArray. + * @param aliases Collection from StorageAliasDao + * @return Vert.x JsonArray + */ + private static JsonArray aliasesToArray(final Collection aliases) { + final JsonArray arr = new JsonArray(); + for (final JsonObject alias : aliases) { + arr.add(new io.vertx.core.json.JsonObject(alias.toString())); + } + return arr; + } + + /** + * Convert YAML alias entries (with "alias" and "storage" keys) to a Vert.x + * JsonArray, normalising to the same "name"/"config" format as the DB layer. + * @param aliases Collection from ManageStorageAliases.list() + * @return Vert.x JsonArray + */ + private static JsonArray yamlAliasesToArray(final Collection aliases) { + final JsonArray arr = new JsonArray(); + for (final JsonObject alias : aliases) { + final io.vertx.core.json.JsonObject entry = + new io.vertx.core.json.JsonObject(); + entry.put("name", alias.getString("alias", "")); + if (alias.containsKey("storage")) { + entry.put("config", + new io.vertx.core.json.JsonObject( + alias.getJsonObject("storage").toString())); + } + arr.add(entry); + } + return arr; + } + + /** + * Parse the request body as a javax.json.JsonObject. + * Sends a 400 error and returns null if the body is missing or invalid. + * @param ctx Routing context + * @return Parsed object, or null if invalid (response already sent) + */ + private static JsonObject bodyAsJson(final RoutingContext ctx) { + final String raw = ctx.body().asString(); + if (raw == null || raw.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return null; + } + try { + return Json.createReader(new StringReader(raw)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return null; + } + } + + /** + * Signals that an alias cannot be deleted because other resources depend on it. + */ + private static final class DependencyException extends RuntimeException { + /** + * Required serial version UID. + */ + private static final long serialVersionUID = 1L; + + /** + * Ctor. + * @param message Error message + */ + DependencyException(final String message) { + super(message); + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/api/v1/UserHandler.java b/artipie-main/src/main/java/com/artipie/api/v1/UserHandler.java new file mode 100644 index 000000000..fa4fd9cd8 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/api/v1/UserHandler.java @@ -0,0 +1,399 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.api.AuthTokenRest; +import com.artipie.api.AuthzHandler; +import com.artipie.api.perms.ApiUserPermission; +import com.artipie.api.perms.ApiUserPermission.UserAction; +import com.artipie.asto.misc.Cleanable; +import com.artipie.http.auth.AuthUser; +import com.artipie.http.auth.Authentication; +import com.artipie.security.policy.Policy; +import com.artipie.settings.ArtipieSecurity; +import com.artipie.settings.cache.ArtipieCaches; +import com.artipie.settings.users.CrudUsers; +import io.vertx.core.json.JsonArray; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import java.io.StringReader; +import java.security.PermissionCollection; +import java.util.List; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonObject; + +/** + * User handler for /api/v1/users/* endpoints. + * @since 1.21 + */ +public final class UserHandler { + + /** + * User name path parameter. + */ + private static final String NAME = "name"; + + /** + * Update user permission constant. + */ + private static final ApiUserPermission UPDATE = + new ApiUserPermission(UserAction.UPDATE); + + /** + * Create user permission constant. + */ + private static final ApiUserPermission CREATE = + new ApiUserPermission(UserAction.CREATE); + + /** + * Crud users object. + */ + private final CrudUsers users; + + /** + * Artipie authenticated users cache. + */ + private final Cleanable ucache; + + /** + * Artipie policy cache. + */ + private final Cleanable pcache; + + /** + * Artipie authentication. + */ + private final Authentication auth; + + /** + * Artipie security policy. + */ + private final Policy policy; + + /** + * Ctor. + * @param users Crud users object + * @param caches Artipie caches + * @param security Artipie security + */ + public UserHandler(final CrudUsers users, final ArtipieCaches caches, + final ArtipieSecurity security) { + this.users = users; + this.ucache = caches.usersCache(); + this.pcache = caches.policyCache(); + this.auth = security.authentication(); + this.policy = security.policy(); + } + + /** + * Register user routes on the router. + * @param router Vert.x router + */ + public void register(final Router router) { + final ApiUserPermission read = new ApiUserPermission(UserAction.READ); + final ApiUserPermission delete = new ApiUserPermission(UserAction.DELETE); + final ApiUserPermission chpass = new ApiUserPermission(UserAction.CHANGE_PASSWORD); + final ApiUserPermission enable = new ApiUserPermission(UserAction.ENABLE); + // GET /api/v1/users — paginated list + router.get("/api/v1/users") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::listUsers); + // GET /api/v1/users/:name — get single user + router.get("/api/v1/users/:name") + .handler(new AuthzHandler(this.policy, read)) + .handler(this::getUser); + // PUT /api/v1/users/:name — create or update user + router.put("/api/v1/users/:name") + .handler(this::putUser); + // DELETE /api/v1/users/:name — delete user + router.delete("/api/v1/users/:name") + .handler(new AuthzHandler(this.policy, delete)) + .handler(this::deleteUser); + // POST /api/v1/users/:name/password — change password + router.post("/api/v1/users/:name/password") + .handler(new AuthzHandler(this.policy, chpass)) + .handler(this::alterPassword); + // POST /api/v1/users/:name/enable — enable user + router.post("/api/v1/users/:name/enable") + .handler(new AuthzHandler(this.policy, enable)) + .handler(this::enableUser); + // POST /api/v1/users/:name/disable — disable user + router.post("/api/v1/users/:name/disable") + .handler(new AuthzHandler(this.policy, enable)) + .handler(this::disableUser); + } + + /** + * GET /api/v1/users — paginated list of users. + * @param ctx Routing context + */ + private void listUsers(final RoutingContext ctx) { + final int page = ApiResponse.intParam( + ctx.queryParam("page").stream().findFirst().orElse(null), 0 + ); + final int size = ApiResponse.clampSize( + ApiResponse.intParam( + ctx.queryParam("size").stream().findFirst().orElse(null), 20 + ) + ); + ctx.vertx().executeBlocking( + this.users::list, + false + ).onSuccess( + all -> { + final List flat = + new java.util.ArrayList<>(all.size()); + for (int i = 0; i < all.size(); i++) { + flat.add( + new io.vertx.core.json.JsonObject( + all.getJsonObject(i).toString() + ) + ); + } + final JsonArray items = ApiResponse.sliceToArray(flat, page, size); + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(ApiResponse.paginated(items, page, size, flat.size()).encode()); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * GET /api/v1/users/:name — get single user info. + * @param ctx Routing context + */ + private void getUser(final RoutingContext ctx) { + final String uname = ctx.pathParam(UserHandler.NAME); + ctx.vertx().>executeBlocking( + () -> this.users.get(uname), + false + ).onSuccess( + opt -> { + if (opt.isPresent()) { + ctx.response() + .setStatusCode(200) + .putHeader("Content-Type", "application/json") + .end(opt.get().toString()); + } else { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("User '%s' not found", uname) + ); + } + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } + + /** + * PUT /api/v1/users/:name — create or update user. + * @param ctx Routing context + */ + private void putUser(final RoutingContext ctx) { + final String uname = ctx.pathParam(UserHandler.NAME); + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final JsonObject rawBody; + try { + rawBody = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + // Normalize: UI sends "password", backend expects "pass" + "type" + final JsonObject body; + if (rawBody.containsKey("password") && !rawBody.containsKey("pass")) { + final javax.json.JsonObjectBuilder nb = Json.createObjectBuilder(rawBody); + nb.add("pass", rawBody.getString("password")); + nb.remove("password"); + if (!rawBody.containsKey("type")) { + nb.add("type", "plain"); + } + body = nb.build(); + } else { + body = rawBody; + } + final Optional existing = this.users.get(uname); + final PermissionCollection perms = this.policy.getPermissions( + new AuthUser( + ctx.user().principal().getString(AuthTokenRest.SUB), + ctx.user().principal().getString(AuthTokenRest.CONTEXT) + ) + ); + if (existing.isPresent() && perms.implies(UserHandler.UPDATE) + || existing.isEmpty() && perms.implies(UserHandler.CREATE)) { + ctx.vertx().executeBlocking( + () -> { + this.users.addOrUpdate(body, uname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.ucache.invalidate(uname); + this.pcache.invalidate(uname); + ctx.response().setStatusCode(201).end(); + } + ).onFailure( + err -> ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()) + ); + } else { + ApiResponse.sendError(ctx, 403, "FORBIDDEN", "Insufficient permissions"); + } + } + + /** + * DELETE /api/v1/users/:name — delete user. + * @param ctx Routing context + */ + private void deleteUser(final RoutingContext ctx) { + final String uname = ctx.pathParam(UserHandler.NAME); + ctx.vertx().executeBlocking( + () -> { + this.users.remove(uname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.ucache.invalidate(uname); + this.pcache.invalidate(uname); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("User '%s' not found", uname) + ); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } + + /** + * POST /api/v1/users/:name/password — change user password. + * @param ctx Routing context + */ + private void alterPassword(final RoutingContext ctx) { + final String uname = ctx.pathParam(UserHandler.NAME); + final String bodyStr = ctx.body().asString(); + if (bodyStr == null || bodyStr.isBlank()) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "JSON body is required"); + return; + } + final JsonObject body; + try { + body = Json.createReader(new StringReader(bodyStr)).readObject(); + } catch (final Exception ex) { + ApiResponse.sendError(ctx, 400, "BAD_REQUEST", "Invalid JSON body"); + return; + } + final String oldPass = body.getString("old_pass", ""); + final Optional verified = this.auth.user(uname, oldPass); + if (verified.isEmpty()) { + ApiResponse.sendError(ctx, 401, "UNAUTHORIZED", "Invalid old password"); + return; + } + ctx.vertx().executeBlocking( + () -> { + this.users.alterPassword(uname, body); + return null; + }, + false + ).onSuccess( + ignored -> { + this.ucache.invalidate(uname); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("User '%s' not found", uname) + ); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } + + /** + * POST /api/v1/users/:name/enable — enable user. + * @param ctx Routing context + */ + private void enableUser(final RoutingContext ctx) { + final String uname = ctx.pathParam(UserHandler.NAME); + ctx.vertx().executeBlocking( + () -> { + this.users.enable(uname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.ucache.invalidate(uname); + this.pcache.invalidate(uname); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("User '%s' not found", uname) + ); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } + + /** + * POST /api/v1/users/:name/disable — disable user. + * @param ctx Routing context + */ + private void disableUser(final RoutingContext ctx) { + final String uname = ctx.pathParam(UserHandler.NAME); + ctx.vertx().executeBlocking( + () -> { + this.users.disable(uname); + return null; + }, + false + ).onSuccess( + ignored -> { + this.ucache.invalidate(uname); + this.pcache.invalidate(uname); + ctx.response().setStatusCode(200).end(); + } + ).onFailure( + err -> { + if (err instanceof IllegalStateException) { + ApiResponse.sendError( + ctx, 404, "NOT_FOUND", + String.format("User '%s' not found", uname) + ); + } else { + ApiResponse.sendError(ctx, 500, "INTERNAL_ERROR", err.getMessage()); + } + } + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/auth/AuthFromDb.java b/artipie-main/src/main/java/com/artipie/auth/AuthFromDb.java new file mode 100644 index 000000000..329bed8ff --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/auth/AuthFromDb.java @@ -0,0 +1,97 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.auth; + +import com.artipie.http.auth.AuthUser; +import com.artipie.http.auth.Authentication; +import com.artipie.http.log.EcsLogger; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Optional; +import javax.sql.DataSource; +import org.apache.commons.codec.digest.DigestUtils; + +/** + * Database-backed authentication. + * Authenticates users by querying the {@code users} table for username + * and comparing the password against the stored {@code password_hash}. + * Supports plain-text comparison (artipie provider) and SHA-256 hashing. + * + * @since 1.21 + */ +public final class AuthFromDb implements Authentication { + + /** + * Auth context name. + */ + private static final String ARTIPIE = "artipie"; + + /** + * SQL query to fetch password hash and provider for an enabled user. + */ + private static final String SQL = String.join(" ", + "SELECT password_hash, auth_provider", + "FROM users", + "WHERE username = ? AND enabled = true" + ); + + /** + * Database data source. + */ + private final DataSource source; + + /** + * Ctor. + * @param source Database data source + */ + public AuthFromDb(final DataSource source) { + this.source = source; + } + + @Override + public Optional user(final String name, final String pass) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(AuthFromDb.SQL)) { + ps.setString(1, name); + final ResultSet rs = ps.executeQuery(); + if (rs.next()) { + final String hash = rs.getString("password_hash"); + final String provider = rs.getString("auth_provider"); + if (hash == null || hash.isEmpty()) { + return Optional.empty(); + } + // Only authenticate artipie-managed users (not SSO) + if (!AuthFromDb.ARTIPIE.equals(provider)) { + return Optional.empty(); + } + // Plain-text match (password stored as-is) + if (hash.equals(pass)) { + return Optional.of(new AuthUser(name, AuthFromDb.ARTIPIE)); + } + // SHA-256 match (password stored as hex digest) + if (hash.equals(DigestUtils.sha256Hex(pass))) { + return Optional.of(new AuthUser(name, AuthFromDb.ARTIPIE)); + } + } + return Optional.empty(); + } catch (final Exception ex) { + EcsLogger.error("com.artipie.auth") + .message("Failed to authenticate user from database") + .eventCategory("authentication") + .eventAction("db_auth") + .eventOutcome("failure") + .field("user.name", name) + .error(ex) + .log(); + return Optional.empty(); + } + } + + @Override + public String toString() { + return this.getClass().getSimpleName(); + } +} diff --git a/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuth.java b/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuth.java index 9b8fd53da..c9f8af0ff 100644 --- a/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuth.java +++ b/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuth.java @@ -123,12 +123,11 @@ public Optional user(final String username, final String password) { // Security: Verify username matches token subject if required if (this.requireUsernameMatch && !username.equals(tokenSubject)) { EcsLogger.warn("com.artipie.auth") - .message("JWT token subject does not match provided username") + .message(String.format("JWT token subject does not match provided username (subject=%s)", tokenSubject)) .eventCategory("authentication") .eventAction("jwt_password_auth") .eventOutcome("failure") .field("user.name", username) - .field("token.subject", tokenSubject) .log(); return Optional.empty(); } diff --git a/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuthFactory.java b/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuthFactory.java index 5a40c6070..bee593852 100644 --- a/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuthFactory.java +++ b/artipie-main/src/main/java/com/artipie/auth/JwtPasswordAuthFactory.java @@ -85,11 +85,10 @@ public Authentication getAuthentication(final YamlMapping cfg) { } } EcsLogger.info("com.artipie.auth") - .message("JWT-as-password authentication initialized") + .message(String.format("JWT-as-password authentication initialized: requireUsernameMatch=%s", requireUsernameMatch)) .eventCategory("authentication") .eventAction("jwt_password_init") .eventOutcome("success") - .field("require_username_match", requireUsernameMatch) .log(); return new JwtPasswordAuth(jwtAuth, requireUsernameMatch); } diff --git a/artipie-main/src/main/java/com/artipie/auth/JwtTokens.java b/artipie-main/src/main/java/com/artipie/auth/JwtTokens.java index f35c152ac..8aa580a63 100644 --- a/artipie-main/src/main/java/com/artipie/auth/JwtTokens.java +++ b/artipie-main/src/main/java/com/artipie/auth/JwtTokens.java @@ -12,6 +12,7 @@ import io.vertx.core.json.JsonObject; import io.vertx.ext.auth.JWTOptions; import io.vertx.ext.auth.jwt.JWTAuth; +import java.util.UUID; /** * Implementation to manage JWT tokens. @@ -73,4 +74,25 @@ public String generate(final AuthUser user, final boolean permanent) { opts ); } + + /** + * Generate token with a specific expiry and token ID for revocation support. + * @param user User to issue token for + * @param expirySeconds Expiry in seconds (0 or negative = permanent) + * @param jti Unique token ID for tracking/revocation + * @return String token + */ + public String generate(final AuthUser user, final int expirySeconds, final UUID jti) { + final JWTOptions opts = new JWTOptions(); + if (expirySeconds > 0) { + opts.setExpiresInSeconds(expirySeconds); + } + return this.provider.generateToken( + new JsonObject() + .put(AuthTokenRest.SUB, user.name()) + .put(AuthTokenRest.CONTEXT, user.authContext()) + .put("jti", jti.toString()), + opts + ); + } } diff --git a/artipie-main/src/main/java/com/artipie/auth/OktaOidcClient.java b/artipie-main/src/main/java/com/artipie/auth/OktaOidcClient.java index ec9fc3f0d..0116b68d2 100644 --- a/artipie-main/src/main/java/com/artipie/auth/OktaOidcClient.java +++ b/artipie-main/src/main/java/com/artipie/auth/OktaOidcClient.java @@ -101,13 +101,10 @@ public OktaAuthResult authenticate( final String mfaCode ) throws IOException, InterruptedException { EcsLogger.info("com.artipie.auth") - .message("Starting Okta authentication") + .message(String.format("Starting Okta authentication: issuer=%s, authnUrl=%s, authorizeUrl=%s", this.issuer, this.authnUrl, this.authorizeUrl)) .eventCategory("authentication") .eventAction("login") .field("user.name", username) - .field("okta.issuer", this.issuer) - .field("okta.authn_url", this.authnUrl) - .field("okta.authorize_url", this.authorizeUrl) .log(); final JsonObject authnReq = Json.createObjectBuilder() .add("username", username) @@ -126,19 +123,19 @@ public OktaAuthResult authenticate( final JsonObject errBody = json(response.body()); errorCode = errBody.getString("errorCode", ""); errorSummary = errBody.getString("errorSummary", ""); - } catch (final Exception ignored) { - // Response may not be JSON + } catch (final Exception ex) { + EcsLogger.debug("com.artipie.auth") + .message("Failed to parse Okta error response as JSON") + .error(ex) + .log(); } EcsLogger.error("com.artipie.auth") - .message("Okta /authn failed") + .message(String.format("Okta /authn failed: url=%s, errorCode=%s, errorSummary=%s", this.authnUrl, errorCode, errorSummary)) .eventCategory("authentication") .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("http.status", response.statusCode()) - .field("okta.url", this.authnUrl) - .field("okta.error_code", errorCode) - .field("okta.error_summary", errorSummary) + .field("http.response.status_code", response.statusCode()) .log(); return null; } @@ -191,7 +188,6 @@ public OktaAuthResult authenticate( .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("tokens_null", tokens == null) .log(); return null; } @@ -362,14 +358,12 @@ private String exchangeSessionForCode(final String sessionToken, final String us ); if (resp.statusCode() / 100 != 3) { EcsLogger.error("com.artipie.auth") - .message("Okta authorize did not redirect") + .message(String.format("Okta authorize did not redirect: authorizeUrl=%s, issuer=%s", this.authorizeUrl, this.issuer)) .eventCategory("authentication") .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("http.status", resp.statusCode()) - .field("okta.authorize_url", this.authorizeUrl) - .field("okta.issuer", this.issuer) + .field("http.response.status_code", resp.statusCode()) .log(); return null; } @@ -381,7 +375,7 @@ private String exchangeSessionForCode(final String sessionToken, final String us .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("http.status", resp.statusCode()) + .field("http.response.status_code", resp.statusCode()) .log(); return null; } @@ -391,7 +385,6 @@ private String exchangeSessionForCode(final String sessionToken, final String us .eventCategory("authentication") .eventAction("login") .field("user.name", username) - .field("okta.redirect_location", location) .log(); final URI loc = URI.create(location); final String queryStr = loc.getQuery(); @@ -402,7 +395,6 @@ private String exchangeSessionForCode(final String sessionToken, final String us .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("okta.redirect_location", location) .log(); return null; } @@ -430,25 +422,21 @@ private String exchangeSessionForCode(final String sessionToken, final String us } if (error != null) { EcsLogger.error("com.artipie.auth") - .message("Okta authorize returned error") + .message(String.format("Okta authorize returned error: %s - %s", error, errorDesc != null ? errorDesc : "")) .eventCategory("authentication") .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("okta.error", error) - .field("okta.error_description", errorDesc != null ? errorDesc : "") .log(); return null; } if (code == null || !state.equals(returnedState)) { EcsLogger.error("com.artipie.auth") - .message("Okta authorize missing code or state mismatch") + .message(String.format("Okta authorize missing code or state mismatch: codePresent=%s, stateMatch=%s", code != null, state.equals(returnedState))) .eventCategory("authentication") .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("code_present", code != null) - .field("state_match", state.equals(returnedState)) .log(); return null; } @@ -479,7 +467,7 @@ private TokenResponse exchangeCodeForTokens(final String code, final String user .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("http.status", resp.statusCode()) + .field("http.response.status_code", resp.statusCode()) .log(); return null; } @@ -522,13 +510,11 @@ private OktaAuthResult parseIdToken( final String iss = json.getString("iss", ""); if (!this.issuer.equals(iss)) { EcsLogger.error("com.artipie.auth") - .message("id_token issuer mismatch") + .message(String.format("id_token issuer mismatch: expected=%s, actual=%s", this.issuer, iss)) .eventCategory("authentication") .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("okta.expected_issuer", this.issuer) - .field("okta.actual_issuer", iss) .log(); return null; } @@ -541,13 +527,11 @@ private OktaAuthResult parseIdToken( } if (!this.clientId.equals(aud)) { EcsLogger.error("com.artipie.auth") - .message("id_token audience mismatch") + .message(String.format("id_token audience mismatch: expected=%s, actual=%s", this.clientId, aud)) .eventCategory("authentication") .eventAction("login") .eventOutcome("failure") .field("user.name", username) - .field("okta.expected_aud", this.clientId) - .field("okta.actual_aud", aud) .log(); return null; } @@ -574,14 +558,12 @@ private OktaAuthResult parseIdToken( } } EcsLogger.info("com.artipie.auth") - .message("Okta authentication successful") + .message(String.format("Okta authentication successful: groups=[%s], groupsClaim=%s", String.join(",", groups), this.groupsClaim)) .eventCategory("authentication") .eventAction("login") .eventOutcome("success") .field("user.name", uname) .field("user.email", email != null ? email : "") - .field("okta.groups", String.join(",", groups)) - .field("okta.groups_claim", this.groupsClaim) .log(); return new OktaAuthResult(uname, email, groups); } catch (final IllegalArgumentException err) { @@ -623,24 +605,22 @@ private JsonObject fetchUserInfo(final String accessToken, final String username ); if (resp.statusCode() / 100 != 2) { EcsLogger.warn("com.artipie.auth") - .message("Okta userinfo endpoint failed") + .message(String.format("Okta userinfo endpoint failed: url=%s", this.userinfoUrl)) .eventCategory("authentication") .eventAction("userinfo") .eventOutcome("failure") .field("user.name", username) - .field("http.status", resp.statusCode()) - .field("okta.userinfo_url", this.userinfoUrl) + .field("http.response.status_code", resp.statusCode()) .log(); return null; } final JsonObject userinfo = json(resp.body()); EcsLogger.info("com.artipie.auth") - .message("Okta userinfo response") + .message(String.format("Okta userinfo response: keys=[%s]", String.join(",", userinfo.keySet()))) .eventCategory("authentication") .eventAction("userinfo") .eventOutcome("success") .field("user.name", username) - .field("okta.userinfo_keys", String.join(",", userinfo.keySet())) .log(); return userinfo; } catch (final IOException | InterruptedException err) { diff --git a/artipie-main/src/main/java/com/artipie/cluster/DbNodeRegistry.java b/artipie-main/src/main/java/com/artipie/cluster/DbNodeRegistry.java new file mode 100644 index 000000000..119790322 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/cluster/DbNodeRegistry.java @@ -0,0 +1,280 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cluster; + +import com.artipie.http.log.EcsLogger; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import javax.sql.DataSource; + +/** + * PostgreSQL-backed node registry for HA clustering. + *

+ * Nodes register on startup, send periodic heartbeats, and are + * automatically considered dead after missing heartbeats. + *

+ *

+ * Schema: artipie_nodes(node_id VARCHAR PRIMARY KEY, hostname VARCHAR, + * port INT, started_at TIMESTAMP, last_heartbeat TIMESTAMP, status VARCHAR) + *

+ * + * @since 1.20.13 + */ +public final class DbNodeRegistry { + + /** + * Logger name for this class. + */ + private static final String LOGGER = "com.artipie.cluster.DbNodeRegistry"; + + /** + * Node status: active. + */ + private static final String STATUS_ACTIVE = "active"; + + /** + * Node status: stopped. + */ + private static final String STATUS_STOPPED = "stopped"; + + /** + * Database source. + */ + private final DataSource source; + + /** + * Ctor. + * @param source Database data source + */ + public DbNodeRegistry(final DataSource source) { + this.source = source; + } + + /** + * Create the artipie_nodes table if it does not exist. + * Should be called once during application startup. + * @throws SQLException On database error + */ + public void createTable() throws SQLException { + try (Connection conn = this.source.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS artipie_nodes(", + " node_id VARCHAR(255) PRIMARY KEY,", + " hostname VARCHAR(255) NOT NULL,", + " port INT NOT NULL,", + " started_at TIMESTAMP NOT NULL,", + " last_heartbeat TIMESTAMP NOT NULL,", + " status VARCHAR(32) NOT NULL", + ");" + ) + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_nodes_status ON artipie_nodes(status)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_nodes_heartbeat ON artipie_nodes(last_heartbeat)" + ); + EcsLogger.info(DbNodeRegistry.LOGGER) + .message("artipie_nodes table initialized") + .eventCategory("database") + .eventAction("create_table") + .eventOutcome("success") + .log(); + } + } + + /** + * Register a node. If the node already exists, update its info (upsert). + * Sets status to 'active' and refreshes heartbeat. + * @param node Node info to register + * @throws SQLException On database error + */ + public void register(final NodeRegistry.NodeInfo node) throws SQLException { + final Timestamp now = Timestamp.from(Instant.now()); + final Timestamp started = Timestamp.from(node.startedAt()); + try (Connection conn = this.source.getConnection(); + PreparedStatement pstmt = conn.prepareStatement( + String.join( + "\n", + "INSERT INTO artipie_nodes(node_id, hostname, port, started_at, last_heartbeat, status)", + "VALUES(?, ?, ?, ?, ?, ?)", + "ON CONFLICT(node_id) DO UPDATE SET", + " hostname = EXCLUDED.hostname,", + " port = EXCLUDED.port,", + " started_at = EXCLUDED.started_at,", + " last_heartbeat = EXCLUDED.last_heartbeat,", + " status = EXCLUDED.status" + ) + )) { + pstmt.setString(1, node.nodeId()); + pstmt.setString(2, node.hostname()); + pstmt.setInt(3, 0); + pstmt.setTimestamp(4, started); + pstmt.setTimestamp(5, now); + pstmt.setString(6, DbNodeRegistry.STATUS_ACTIVE); + pstmt.executeUpdate(); + EcsLogger.info(DbNodeRegistry.LOGGER) + .message("Node registered: " + node.nodeId()) + .eventCategory("cluster") + .eventAction("node_register") + .eventOutcome("success") + .field("node.id", node.nodeId()) + .field("node.hostname", node.hostname()) + .log(); + } + } + + /** + * Send a heartbeat for the given node. + * Updates last_heartbeat to current time and ensures status is 'active'. + * @param nodeId Node identifier + * @throws SQLException On database error + */ + public void heartbeat(final String nodeId) throws SQLException { + final Timestamp now = Timestamp.from(Instant.now()); + try (Connection conn = this.source.getConnection(); + PreparedStatement pstmt = conn.prepareStatement( + "UPDATE artipie_nodes SET last_heartbeat = ?, status = ? WHERE node_id = ?" + )) { + pstmt.setTimestamp(1, now); + pstmt.setString(2, DbNodeRegistry.STATUS_ACTIVE); + pstmt.setString(3, nodeId); + final int updated = pstmt.executeUpdate(); + if (updated == 0) { + EcsLogger.warn(DbNodeRegistry.LOGGER) + .message("Heartbeat for unknown node: " + nodeId) + .eventCategory("cluster") + .eventAction("node_heartbeat") + .eventOutcome("failure") + .field("node.id", nodeId) + .log(); + } else { + EcsLogger.debug(DbNodeRegistry.LOGGER) + .message("Heartbeat received: " + nodeId) + .eventCategory("cluster") + .eventAction("node_heartbeat") + .eventOutcome("success") + .field("node.id", nodeId) + .log(); + } + } + } + + /** + * Deregister a node by setting its status to 'stopped'. + * The node row is retained for audit purposes; use {@link #evictStale} + * to physically remove old entries. + * @param nodeId Node identifier + * @throws SQLException On database error + */ + public void deregister(final String nodeId) throws SQLException { + try (Connection conn = this.source.getConnection(); + PreparedStatement pstmt = conn.prepareStatement( + "UPDATE artipie_nodes SET status = ? WHERE node_id = ?" + )) { + pstmt.setString(1, DbNodeRegistry.STATUS_STOPPED); + pstmt.setString(2, nodeId); + pstmt.executeUpdate(); + EcsLogger.info(DbNodeRegistry.LOGGER) + .message("Node deregistered: " + nodeId) + .eventCategory("cluster") + .eventAction("node_deregister") + .eventOutcome("success") + .field("node.id", nodeId) + .log(); + } + } + + /** + * Get all nodes whose last heartbeat is within the given timeout + * and whose status is 'active'. + * @param heartbeatTimeoutMs Maximum age of heartbeat in milliseconds + * @return List of live node info records + * @throws SQLException On database error + */ + public List liveNodes(final long heartbeatTimeoutMs) + throws SQLException { + final Timestamp cutoff = Timestamp.from( + Instant.now().minusMillis(heartbeatTimeoutMs) + ); + final List result = new ArrayList<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement pstmt = conn.prepareStatement( + String.join( + "\n", + "SELECT node_id, hostname, started_at, last_heartbeat", + "FROM artipie_nodes", + "WHERE status = ? AND last_heartbeat >= ?", + "ORDER BY started_at" + ) + )) { + pstmt.setString(1, DbNodeRegistry.STATUS_ACTIVE); + pstmt.setTimestamp(2, cutoff); + try (ResultSet rset = pstmt.executeQuery()) { + while (rset.next()) { + result.add( + new NodeRegistry.NodeInfo( + rset.getString("node_id"), + rset.getString("hostname"), + rset.getTimestamp("started_at").toInstant(), + rset.getTimestamp("last_heartbeat").toInstant() + ) + ); + } + } + } + EcsLogger.debug(DbNodeRegistry.LOGGER) + .message("Live nodes query returned " + result.size() + " nodes") + .eventCategory("cluster") + .eventAction("live_nodes_query") + .eventOutcome("success") + .field("cluster.live_count", result.size()) + .field("cluster.heartbeat_timeout_ms", heartbeatTimeoutMs) + .log(); + return result; + } + + /** + * Remove stale nodes whose last heartbeat is older than the given timeout. + * This physically deletes the rows from the database. + * @param heartbeatTimeoutMs Maximum age of heartbeat in milliseconds + * @return Number of evicted nodes + * @throws SQLException On database error + */ + public int evictStale(final long heartbeatTimeoutMs) throws SQLException { + final Timestamp cutoff = Timestamp.from( + Instant.now().minusMillis(heartbeatTimeoutMs) + ); + final int evicted; + try (Connection conn = this.source.getConnection(); + PreparedStatement pstmt = conn.prepareStatement( + "DELETE FROM artipie_nodes WHERE last_heartbeat < ?" + )) { + pstmt.setTimestamp(1, cutoff); + evicted = pstmt.executeUpdate(); + } + if (evicted > 0) { + EcsLogger.info(DbNodeRegistry.LOGGER) + .message("Evicted " + evicted + " stale nodes") + .eventCategory("cluster") + .eventAction("node_evict") + .eventOutcome("success") + .field("cluster.evicted_count", evicted) + .field("cluster.heartbeat_timeout_ms", heartbeatTimeoutMs) + .log(); + } + return evicted; + } +} diff --git a/artipie-main/src/main/java/com/artipie/cluster/NodeRegistry.java b/artipie-main/src/main/java/com/artipie/cluster/NodeRegistry.java new file mode 100644 index 000000000..90d07e401 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/cluster/NodeRegistry.java @@ -0,0 +1,114 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cluster; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Registry of cluster nodes for HA coordination. + * In-memory implementation for single-node mode. + * PostgreSQL-backed implementation for multi-node clusters. + * + * @since 1.20.13 + */ +public final class NodeRegistry { + + /** + * Default stale threshold. + */ + private static final Duration STALE_THRESHOLD = Duration.ofSeconds(30); + + /** + * This node's ID. + */ + private final String nodeId; + + /** + * This node's hostname. + */ + private final String hostname; + + /** + * Registered nodes (nodeId -> NodeInfo). + */ + private final Map nodes; + + /** + * Ctor with auto-generated node ID. + * @param hostname This node's hostname + */ + public NodeRegistry(final String hostname) { + this(UUID.randomUUID().toString(), hostname); + } + + /** + * Ctor. + * @param nodeId This node's unique ID + * @param hostname This node's hostname + */ + public NodeRegistry(final String nodeId, final String hostname) { + this.nodeId = nodeId; + this.hostname = hostname; + this.nodes = new ConcurrentHashMap<>(); + // Register self + this.nodes.put(nodeId, new NodeInfo(nodeId, hostname, Instant.now(), Instant.now())); + } + + /** + * Record a heartbeat for this node. + */ + public void heartbeat() { + this.nodes.compute(this.nodeId, (id, existing) -> { + if (existing != null) { + return new NodeInfo(id, this.hostname, existing.startedAt(), Instant.now()); + } + return new NodeInfo(id, this.hostname, Instant.now(), Instant.now()); + }); + } + + /** + * Get all active (non-stale) nodes. + * @return List of active node info + */ + public List activeNodes() { + final Instant cutoff = Instant.now().minus(STALE_THRESHOLD); + return this.nodes.values().stream() + .filter(n -> n.lastHeartbeat().isAfter(cutoff)) + .collect(Collectors.toList()); + } + + /** + * Get this node's ID. + * @return Node ID + */ + public String nodeId() { + return this.nodeId; + } + + /** + * Get total registered node count. + * @return Node count + */ + public int size() { + return this.nodes.size(); + } + + /** + * Node information record. + * @param nodeId Node ID + * @param hostname Node hostname + * @param startedAt When the node started + * @param lastHeartbeat Last heartbeat timestamp + */ + public record NodeInfo( + String nodeId, String hostname, Instant startedAt, Instant lastHeartbeat + ) { } +} diff --git a/artipie-main/src/main/java/com/artipie/cooldown/CooldownRepository.java b/artipie-main/src/main/java/com/artipie/cooldown/CooldownRepository.java index dc5314489..5dbb7c7d8 100644 --- a/artipie-main/src/main/java/com/artipie/cooldown/CooldownRepository.java +++ b/artipie-main/src/main/java/com/artipie/cooldown/CooldownRepository.java @@ -17,11 +17,11 @@ import java.util.Optional; import javax.sql.DataSource; -final class CooldownRepository { +public final class CooldownRepository { private final DataSource dataSource; - CooldownRepository(final DataSource dataSource) { + public CooldownRepository(final DataSource dataSource) { this.dataSource = Objects.requireNonNull(dataSource); } @@ -111,31 +111,76 @@ DbBlockRecord insertBlock( } } - void updateStatus( - final long blockId, - final BlockStatus status, - final Optional unblockedAt, - final Optional unblockedBy - ) { - final String sql = - "UPDATE artifact_cooldowns SET status = ?, unblocked_at = ?, unblocked_by = ? WHERE id = ?"; + /** + * Delete a cooldown block record by id. + * Callers must log the record details before calling this method. + * @param blockId Record id to delete + */ + void deleteBlock(final long blockId) { + final String sql = "DELETE FROM artifact_cooldowns WHERE id = ?"; try (Connection conn = this.dataSource.getConnection(); PreparedStatement stmt = conn.prepareStatement(sql)) { - stmt.setString(1, status.name()); - if (unblockedAt.isPresent()) { - stmt.setLong(2, unblockedAt.get().toEpochMilli()); - } else { - stmt.setNull(2, java.sql.Types.BIGINT); - } - if (unblockedBy.isPresent()) { - stmt.setString(3, unblockedBy.get()); - } else { - stmt.setNull(3, java.sql.Types.VARCHAR); - } - stmt.setLong(4, blockId); + stmt.setLong(1, blockId); stmt.executeUpdate(); } catch (final SQLException err) { - throw new IllegalStateException("Failed to update cooldown status", err); + throw new IllegalStateException("Failed to delete cooldown block", err); + } + } + + /** + * Delete all active blocks for a repository in a single statement. + * @param repoType Repository type + * @param repoName Repository name + * @return Number of deleted rows + */ + int deleteActiveBlocksForRepo(final String repoType, final String repoName) { + final String sql = + "DELETE FROM artifact_cooldowns WHERE repo_type = ? AND repo_name = ? AND status = ?"; + try (Connection conn = this.dataSource.getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql)) { + stmt.setString(1, repoType); + stmt.setString(2, repoName); + stmt.setString(3, BlockStatus.ACTIVE.name()); + return stmt.executeUpdate(); + } catch (final SQLException err) { + throw new IllegalStateException("Failed to delete active blocks for repo", err); + } + } + + /** + * Delete all active blocks globally. Used when cooldown is disabled. + * @param actor Username performing the action + * @return Number of deleted rows + */ + public int unblockAll(final String actor) { + final String sql = "DELETE FROM artifact_cooldowns WHERE status = ?"; + try (Connection conn = this.dataSource.getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql)) { + stmt.setString(1, BlockStatus.ACTIVE.name()); + return stmt.executeUpdate(); + } catch (final SQLException err) { + throw new IllegalStateException("Failed to unblock all cooldown blocks", err); + } + } + + /** + * Delete all active blocks for a specific repo type. Used when a repo type + * cooldown override is disabled. + * @param repoType Repository type (e.g. "maven-proxy") + * @param actor Username performing the action + * @return Number of deleted rows + */ + public int unblockByRepoType(final String repoType, final String actor) { + final String sql = + "DELETE FROM artifact_cooldowns WHERE repo_type = ? AND status = ?"; + try (Connection conn = this.dataSource.getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql)) { + stmt.setString(1, repoType); + stmt.setString(2, BlockStatus.ACTIVE.name()); + return stmt.executeUpdate(); + } catch (final SQLException err) { + throw new IllegalStateException( + "Failed to unblock blocks for repo type " + repoType, err); } } @@ -168,7 +213,7 @@ List findActiveForRepo(final String repoType, final String repoNa * @param repoName Repository name * @return Count of active blocks */ - long countActiveBlocks(final String repoType, final String repoName) { + public long countActiveBlocks(final String repoType, final String repoName) { final String sql = "SELECT COUNT(*) FROM artifact_cooldowns WHERE repo_type = ? AND repo_name = ? AND status = ?"; try (Connection conn = this.dataSource.getConnection(); @@ -322,6 +367,107 @@ int unmarkAllBlockedForRepo(final String repoType, final String repoName) { } } + /** + * Find all active blocks across all repos, paginated, with optional search. + * @param offset Row offset + * @param limit Max rows + * @param search Optional search term (filters artifact, repo_name, version) + * @return List of active block records + */ + public List findAllActivePaginated( + final int offset, final int limit, final String search + ) { + final boolean hasSearch = search != null && !search.isBlank(); + final String sql; + if (hasSearch) { + sql = "SELECT id, repo_type, repo_name, artifact, version, reason, status, blocked_by, " + + "blocked_at, blocked_until, unblocked_at, unblocked_by, installed_by " + + "FROM artifact_cooldowns WHERE status = ? " + + "AND (artifact ILIKE ? OR repo_name ILIKE ? OR version ILIKE ?) " + + "ORDER BY blocked_at DESC LIMIT ? OFFSET ?"; + } else { + sql = "SELECT id, repo_type, repo_name, artifact, version, reason, status, blocked_by, " + + "blocked_at, blocked_until, unblocked_at, unblocked_by, installed_by " + + "FROM artifact_cooldowns WHERE status = ? " + + "ORDER BY blocked_at DESC LIMIT ? OFFSET ?"; + } + try (Connection conn = this.dataSource.getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql)) { + int idx = 1; + stmt.setString(idx++, BlockStatus.ACTIVE.name()); + if (hasSearch) { + final String pattern = "%" + search.trim() + "%"; + stmt.setString(idx++, pattern); + stmt.setString(idx++, pattern); + stmt.setString(idx++, pattern); + } + stmt.setInt(idx++, limit); + stmt.setInt(idx, offset); + try (ResultSet rs = stmt.executeQuery()) { + final List result = new ArrayList<>(); + while (rs.next()) { + result.add(readRecord(rs)); + } + return result; + } + } catch (final SQLException err) { + throw new IllegalStateException("Failed to query active cooldowns", err); + } + } + + /** + * Find all active blocks (no search filter). + * @param offset Row offset + * @param limit Max rows + * @return List of active block records + */ + public List findAllActivePaginated(final int offset, final int limit) { + return this.findAllActivePaginated(offset, limit, null); + } + + /** + * Count total active blocks across all repos, with optional search. + * @param search Optional search term + * @return Total count + */ + public long countTotalActiveBlocks(final String search) { + final boolean hasSearch = search != null && !search.isBlank(); + final String sql; + if (hasSearch) { + sql = "SELECT COUNT(*) FROM artifact_cooldowns WHERE status = ? " + + "AND (artifact ILIKE ? OR repo_name ILIKE ? OR version ILIKE ?)"; + } else { + sql = "SELECT COUNT(*) FROM artifact_cooldowns WHERE status = ?"; + } + try (Connection conn = this.dataSource.getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql)) { + int idx = 1; + stmt.setString(idx++, BlockStatus.ACTIVE.name()); + if (hasSearch) { + final String pattern = "%" + search.trim() + "%"; + stmt.setString(idx++, pattern); + stmt.setString(idx++, pattern); + stmt.setString(idx, pattern); + } + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + return 0; + } + } catch (final SQLException err) { + throw new IllegalStateException("Failed to count active cooldowns", err); + } + } + + /** + * Count total active blocks (no search filter). + * @return Total count + */ + public long countTotalActiveBlocks() { + return this.countTotalActiveBlocks(null); + } + List cachedVersions( final String repoType, final String repoName, diff --git a/artipie-main/src/main/java/com/artipie/cooldown/CooldownSupport.java b/artipie-main/src/main/java/com/artipie/cooldown/CooldownSupport.java index 7e41bca88..7b4f92a6c 100644 --- a/artipie-main/src/main/java/com/artipie/cooldown/CooldownSupport.java +++ b/artipie-main/src/main/java/com/artipie/cooldown/CooldownSupport.java @@ -11,14 +11,21 @@ import com.artipie.cooldown.metadata.FilteredMetadataCache; import com.artipie.cooldown.metadata.FilteredMetadataCacheConfig; import com.artipie.cooldown.metadata.NoopCooldownMetadataService; +import com.artipie.db.dao.SettingsDao; import com.artipie.http.log.EcsLogger; import com.artipie.http.trace.TraceContextExecutor; import com.artipie.settings.Settings; +import java.time.Duration; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; +import javax.json.JsonObject; /** * Factory for cooldown services. @@ -76,6 +83,9 @@ public static CooldownService create(final Settings settings) { public static CooldownService create(final Settings settings, final Executor executor) { return settings.artifactsDatabase() .map(ds -> { + // Load DB-persisted cooldown config and apply over YAML defaults. + // This ensures overrides saved via the UI survive container restarts. + loadDbCooldownSettings(settings.cooldown(), ds); EcsLogger.info("com.artipie.cooldown") .message("Creating JdbcCooldownService (enabled: " + settings.cooldown().enabled() + ", min age: " + settings.cooldown().minimumAllowedAge().toString() + ")") .eventCategory("configuration") @@ -146,4 +156,85 @@ public static CooldownMetadataService createMetadataService( 50 // max versions to evaluate ); } + + /** + * Load cooldown settings from DB and apply to in-memory CooldownSettings. + * DB settings (saved via the UI) take precedence over YAML defaults. + * @param csettings In-memory cooldown settings to update + * @param ds Database data source + */ + @SuppressWarnings("PMD.CognitiveComplexity") + private static void loadDbCooldownSettings( + final CooldownSettings csettings, + final javax.sql.DataSource ds + ) { + try { + final SettingsDao dao = new SettingsDao(ds); + final Optional dbConfig = dao.get("cooldown"); + if (dbConfig.isEmpty()) { + return; + } + final JsonObject cfg = dbConfig.get(); + final boolean enabled = cfg.getBoolean("enabled", csettings.enabled()); + final Duration minAge = cfg.containsKey("minimum_allowed_age") + ? parseDuration(cfg.getString("minimum_allowed_age")) + : csettings.minimumAllowedAge(); + final Map overrides = new HashMap<>(); + if (cfg.containsKey("repo_types")) { + final JsonObject repoTypes = cfg.getJsonObject("repo_types"); + for (final String key : repoTypes.keySet()) { + final JsonObject rt = repoTypes.getJsonObject(key); + overrides.put( + key.toLowerCase(Locale.ROOT), + new CooldownSettings.RepoTypeConfig( + rt.getBoolean("enabled", true), + rt.containsKey("minimum_allowed_age") + ? parseDuration(rt.getString("minimum_allowed_age")) + : minAge + ) + ); + } + } + csettings.update(enabled, minAge, overrides); + EcsLogger.info("com.artipie.cooldown") + .message("Loaded cooldown settings from database (enabled: " + + enabled + ", overrides: " + overrides.size() + ")") + .eventCategory("configuration") + .eventAction("cooldown_db_load") + .log(); + } catch (final Exception ex) { + EcsLogger.warn("com.artipie.cooldown") + .message("Failed to load cooldown settings from DB, using YAML defaults: " + + ex.getMessage()) + .eventCategory("configuration") + .eventAction("cooldown_db_load") + .eventOutcome("failure") + .log(); + } + } + + /** + * Parse duration string (e.g. "7d", "24h", "30m") to Duration. + * @param value Duration string + * @return Duration + */ + private static Duration parseDuration(final String value) { + if (value == null || value.isEmpty()) { + return Duration.ofHours(CooldownSettings.DEFAULT_HOURS); + } + final String trimmed = value.trim().toLowerCase(Locale.ROOT); + final String num = trimmed.replaceAll("[^0-9]", ""); + if (num.isEmpty()) { + return Duration.ofHours(CooldownSettings.DEFAULT_HOURS); + } + final long amount = Long.parseLong(num); + if (trimmed.endsWith("d")) { + return Duration.ofDays(amount); + } else if (trimmed.endsWith("h")) { + return Duration.ofHours(amount); + } else if (trimmed.endsWith("m")) { + return Duration.ofMinutes(amount); + } + return Duration.ofHours(amount); + } } diff --git a/artipie-main/src/main/java/com/artipie/cooldown/DbBlockRecord.java b/artipie-main/src/main/java/com/artipie/cooldown/DbBlockRecord.java index 6b87ef295..6d88e8796 100644 --- a/artipie-main/src/main/java/com/artipie/cooldown/DbBlockRecord.java +++ b/artipie-main/src/main/java/com/artipie/cooldown/DbBlockRecord.java @@ -8,7 +8,7 @@ import java.time.Instant; import java.util.Optional; -final class DbBlockRecord { +public final class DbBlockRecord { private final long id; private final String repoType; @@ -54,55 +54,55 @@ final class DbBlockRecord { this.installedBy = installedBy; } - long id() { + public long id() { return this.id; } - String repoType() { + public String repoType() { return this.repoType; } - String repoName() { + public String repoName() { return this.repoName; } - String artifact() { + public String artifact() { return this.artifact; } - String version() { + public String version() { return this.version; } - CooldownReason reason() { + public CooldownReason reason() { return this.reason; } - BlockStatus status() { + public BlockStatus status() { return this.status; } - String blockedBy() { + public String blockedBy() { return this.blockedBy; } - Instant blockedAt() { + public Instant blockedAt() { return this.blockedAt; } - Instant blockedUntil() { + public Instant blockedUntil() { return this.blockedUntil; } - Optional unblockedAt() { + public Optional unblockedAt() { return this.unblockedAt; } - Optional unblockedBy() { + public Optional unblockedBy() { return this.unblockedBy; } - Optional installedBy() { + public Optional installedBy() { return this.installedBy; } } diff --git a/artipie-main/src/main/java/com/artipie/cooldown/JdbcCooldownService.java b/artipie-main/src/main/java/com/artipie/cooldown/JdbcCooldownService.java index e6bd698be..7868ea116 100644 --- a/artipie-main/src/main/java/com/artipie/cooldown/JdbcCooldownService.java +++ b/artipie-main/src/main/java/com/artipie/cooldown/JdbcCooldownService.java @@ -113,12 +113,11 @@ public void initializeMetrics() { metrics.setAllBlockedPackages(allBlocked); EcsLogger.info("com.artipie.cooldown") - .message("Initialized cooldown metrics from database") + .message(String.format( + "Initialized cooldown metrics from database: %d repositories, %d total blocks, %d all-blocked packages", + counts.size(), total, allBlocked)) .eventCategory("cooldown") .eventAction("metrics_init") - .field("repositories.count", counts.size()) - .field("blocks.total", total) - .field("all_blocked.packages", allBlocked) .log(); } catch (Exception e) { EcsLogger.error("com.artipie.cooldown") @@ -374,15 +373,13 @@ private CompletableFuture getBlockResult(final CooldownRequest r if (record.isPresent()) { final DbBlockRecord rec = record.get(); EcsLogger.info("com.artipie.cooldown") - .message("Block record found in database") + .message(String.format( + "Block record found in database: status=%s, reason=%s, blockedAt=%s, blockedUntil=%s", + rec.status().name(), rec.reason().name(), rec.blockedAt(), rec.blockedUntil())) .eventCategory("cooldown") .eventAction("block_lookup") .field("package.name", request.artifact()) .field("package.version", request.version()) - .field("block.status", rec.status().name()) - .field("block.reason", rec.reason().name()) - .field("block.blockedAt", rec.blockedAt().toString()) - .field("block.blockedUntil", rec.blockedUntil().toString()) .log(); if (rec.status() == BlockStatus.ACTIVE) { @@ -390,12 +387,13 @@ private CompletableFuture getBlockResult(final CooldownRequest r final Instant now = Instant.now(); if (rec.blockedUntil().isBefore(now)) { EcsLogger.info("com.artipie.cooldown") - .message("Block has EXPIRED - allowing artifact") + .message(String.format( + "Block has EXPIRED - allowing artifact (blockedUntil=%s)", + rec.blockedUntil())) .eventCategory("cooldown") .eventAction("block_expired") .field("package.name", request.artifact()) .field("package.version", request.version()) - .field("block.blockedUntil", rec.blockedUntil().toString()) .log(); // Expire the block this.expire(rec, now); @@ -528,16 +526,14 @@ private CompletableFuture shouldBlockNewArtifact( // Debug logging to diagnose blocking decisions EcsLogger.info("com.artipie.cooldown") - .message("Evaluating freshness") + .message(String.format( + "Evaluating freshness: cooldown=%s, release+cooldown=%s, requestTime=%s, isFresh=%s", + fresh, date.plus(fresh), now, date.plus(fresh).isAfter(now))) .eventCategory("cooldown") .eventAction("freshness_check") .field("package.name", request.artifact()) .field("package.version", request.version()) - .field("release.date", date.toString()) - .field("cooldown.period", fresh.toString()) - .field("release.plus.cooldown", date.plus(fresh).toString()) - .field("request.time", now.toString()) - .field("is.fresh", date.plus(fresh).isAfter(now)) + .field("package.release_date", date.toString()) .log(); if (date.plus(fresh).isAfter(now) @@ -628,7 +624,21 @@ private CompletableFuture createBlockInDatabase( } private void expire(final DbBlockRecord record, final Instant when) { - this.repository.updateStatus(record.id(), BlockStatus.EXPIRED, Optional.of(when), Optional.empty()); + EcsLogger.info("com.artipie.cooldown") + .message("Deleting expired cooldown block") + .eventCategory("cooldown") + .eventAction("block_expired_delete") + .field("package.name", record.artifact()) + .field("package.version", record.version()) + .field("repository.type", record.repoType()) + .field("repository.name", record.repoName()) + .field("cooldown.reason", record.reason().name()) + .field("cooldown.blocked_at", record.blockedAt().toString()) + .field("cooldown.blocked_until", record.blockedUntil().toString()) + .field("cooldown.blocked_by", record.blockedBy()) + .field("cooldown.expired_at", when.toString()) + .log(); + this.repository.deleteBlock(record.id()); // Decrement active blocks metric (O(1), no DB query) this.decrementActiveBlocksMetric(record.repoType(), record.repoName()); } @@ -654,12 +664,27 @@ private int unblockAllBlocking( final String actor ) { final Instant now = Instant.now(); + // Log each active block before bulk delete final List blocks = this.repository.findActiveForRepo(repoType, repoName); - final int count = (int) blocks.stream() - .filter(record -> record.status() == BlockStatus.ACTIVE) - .peek(record -> this.release(record, actor, now)) - .count(); - + for (final DbBlockRecord record : blocks) { + EcsLogger.info("com.artipie.cooldown") + .message("Deleting unblocked cooldown block (bulk unblock-all)") + .eventCategory("cooldown") + .eventAction("block_unblocked_delete") + .field("package.name", record.artifact()) + .field("package.version", record.version()) + .field("repository.type", repoType) + .field("repository.name", repoName) + .field("cooldown.reason", record.reason().name()) + .field("cooldown.blocked_at", record.blockedAt().toString()) + .field("cooldown.blocked_until", record.blockedUntil().toString()) + .field("cooldown.blocked_by", record.blockedBy()) + .field("cooldown.unblocked_by", actor) + .field("cooldown.unblocked_at", now.toString()) + .log(); + } + // Single bulk DELETE instead of N individual updates + final int count = this.repository.deleteActiveBlocksForRepo(repoType, repoName); // Clear inspector cache (works for all adapters: Docker, NPM, PyPI, etc.) com.artipie.cooldown.InspectorRegistry.instance() .clearAll(repoType, repoName); @@ -667,12 +692,22 @@ private int unblockAllBlocking( } private void release(final DbBlockRecord record, final String actor, final Instant when) { - this.repository.updateStatus( - record.id(), - BlockStatus.INACTIVE, - Optional.of(when), - Optional.of(actor) - ); + EcsLogger.info("com.artipie.cooldown") + .message("Deleting unblocked cooldown block") + .eventCategory("cooldown") + .eventAction("block_unblocked_delete") + .field("package.name", record.artifact()) + .field("package.version", record.version()) + .field("repository.type", record.repoType()) + .field("repository.name", record.repoName()) + .field("cooldown.reason", record.reason().name()) + .field("cooldown.blocked_at", record.blockedAt().toString()) + .field("cooldown.blocked_until", record.blockedUntil().toString()) + .field("cooldown.blocked_by", record.blockedBy()) + .field("cooldown.unblocked_by", actor) + .field("cooldown.unblocked_at", when.toString()) + .log(); + this.repository.deleteBlock(record.id()); } private CooldownBlock toCooldownBlock(final DbBlockRecord record) { @@ -757,12 +792,12 @@ private void unmarkAllBlockedForRepo(final String repoType, final String repoNam final long newTotal = this.repository.countAllBlockedPackages(); CooldownMetrics.getInstance().setAllBlockedPackages(newTotal); EcsLogger.debug("com.artipie.cooldown") - .message("Unmarked all-blocked packages for repo") + .message(String.format( + "Unmarked all-blocked packages for repo: %d packages unmarked", count)) .eventCategory("cooldown") .eventAction("all_blocked_unmark_all") .field("repository.type", repoType) .field("repository.name", repoName) - .field("packages.unmarked", count) .log(); } } catch (Exception e) { diff --git a/artipie-main/src/main/java/com/artipie/db/ArtifactDbFactory.java b/artipie-main/src/main/java/com/artipie/db/ArtifactDbFactory.java index 878c4cbc3..867903127 100644 --- a/artipie-main/src/main/java/com/artipie/db/ArtifactDbFactory.java +++ b/artipie-main/src/main/java/com/artipie/db/ArtifactDbFactory.java @@ -7,6 +7,7 @@ import com.amihaiemil.eoyaml.YamlMapping; import com.artipie.ArtipieException; import com.artipie.http.log.EcsLogger; +import com.artipie.http.misc.ConfigDefaults; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.sql.Connection; @@ -107,7 +108,8 @@ public final class ArtifactDbFactory { * * @since 1.19.2 */ - static final int DEFAULT_POOL_MAX_SIZE = 50; + static final int DEFAULT_POOL_MAX_SIZE = + ConfigDefaults.getInt("ARTIPIE_DB_POOL_MAX", 50); /** * Default connection pool minimum idle. @@ -115,7 +117,8 @@ public final class ArtifactDbFactory { * * @since 1.19.2 */ - static final int DEFAULT_POOL_MIN_IDLE = 10; + static final int DEFAULT_POOL_MIN_IDLE = + ConfigDefaults.getInt("ARTIPIE_DB_POOL_MIN", 10); /** * Default buffer time in seconds. @@ -200,16 +203,23 @@ public DataSource initialize() { hikariConfig.setPassword(password); hikariConfig.setMaximumPoolSize(poolMaxSize); hikariConfig.setMinimumIdle(poolMinIdle); - hikariConfig.setConnectionTimeout(30000); // 30 seconds - hikariConfig.setIdleTimeout(600000); // 10 minutes - hikariConfig.setMaxLifetime(1800000); // 30 minutes + hikariConfig.setConnectionTimeout( + ConfigDefaults.getLong("ARTIPIE_DB_CONNECTION_TIMEOUT_MS", 5000L) + ); + hikariConfig.setIdleTimeout( + ConfigDefaults.getLong("ARTIPIE_DB_IDLE_TIMEOUT_MS", 600_000L) + ); + hikariConfig.setMaxLifetime( + ConfigDefaults.getLong("ARTIPIE_DB_MAX_LIFETIME_MS", 1_800_000L) + ); hikariConfig.setPoolName("ArtipieDB-Pool"); - // Enable connection leak detection (120 seconds threshold) - // Logs a warning if a connection is not returned to the pool within 120 seconds - // Increased from 60s to reduce false positives during batch processing - // DbConsumer batch operations can take >60s under high load - hikariConfig.setLeakDetectionThreshold(120000); // 120 seconds + // Enable connection leak detection (300 seconds threshold) + // Logs a warning if a connection is not returned to the pool within 300 seconds + // Increased to reduce false positives during large batch processing (200 events/batch) + hikariConfig.setLeakDetectionThreshold( + ConfigDefaults.getLong("ARTIPIE_DB_LEAK_DETECTION_MS", 300000) + ); // Enable metrics and logging for connection pool monitoring hikariConfig.setRegisterMbeans(true); // Enable JMX metrics @@ -302,6 +312,10 @@ private static void createStructure(final DataSource source) { statement.executeUpdate( "ALTER TABLE artifacts ADD COLUMN IF NOT EXISTS release_date BIGINT" ); + // Migration: Add path_prefix column for path-based group index lookup + statement.executeUpdate( + "ALTER TABLE artifacts ADD COLUMN IF NOT EXISTS path_prefix VARCHAR" + ); // Performance indexes for artifacts table statement.executeUpdate( @@ -319,6 +333,110 @@ private static void createStructure(final DataSource source) { statement.executeUpdate( "CREATE INDEX IF NOT EXISTS idx_artifacts_release_date ON artifacts(release_date) WHERE release_date IS NOT NULL" ); + // Covering index for locate() — enables index-only scan + statement.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_locate ON artifacts (name, repo_name) INCLUDE (repo_type)" + ); + // Covering index for browse operations + statement.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_browse ON artifacts (repo_name, name, version) INCLUDE (size, created_date, owner)" + ); + // Index for path-prefix based locate() queries (group resolution) + statement.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_path_prefix ON artifacts (path_prefix, repo_name) WHERE path_prefix IS NOT NULL" + ); + // Migration: Add tsvector column for full-text search (B1) + // Uses 'simple' config to avoid language-specific stemming on artifact names + try { + statement.executeUpdate( + "ALTER TABLE artifacts ADD COLUMN IF NOT EXISTS search_tokens tsvector" + ); + } catch (final SQLException ex) { + EcsLogger.debug("com.artipie.db") + .message("Failed to add search_tokens column (may already exist)") + .error(ex) + .log(); + } + // GIN index for fast full-text search on search_tokens + try { + statement.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_artifacts_search ON artifacts USING gin(search_tokens)" + ); + } catch (final SQLException ex) { + EcsLogger.debug("com.artipie.db") + .message("Failed to create GIN index idx_artifacts_search (may already exist)") + .error(ex) + .log(); + } + // Trigger function to auto-populate search_tokens on INSERT/UPDATE. + // Uses translate() to replace dots, slashes, dashes, and underscores + // with spaces so each component becomes a separate searchable token. + // Without this, "auto1.base.test.txt" is one opaque token and + // searching for "test" won't match. + try { + statement.executeUpdate( + String.join( + "\n", + "CREATE OR REPLACE FUNCTION artifacts_search_update() RETURNS trigger AS $$", + "BEGIN", + " NEW.search_tokens := to_tsvector('simple',", + " translate(coalesce(NEW.name, ''), './-_', ' ') || ' ' ||", + " translate(coalesce(NEW.version, ''), './-_', ' ') || ' ' ||", + " coalesce(NEW.owner, '') || ' ' ||", + " translate(coalesce(NEW.repo_name, ''), './-_', ' ') || ' ' ||", + " translate(coalesce(NEW.repo_type, ''), './-_', ' '));", + " RETURN NEW;", + "END;", + "$$ LANGUAGE plpgsql;" + ) + ); + } catch (final SQLException ex) { + EcsLogger.debug("com.artipie.db") + .message("Failed to create artifacts_search_update function") + .error(ex) + .log(); + } + // Attach trigger to artifacts table (drop first for idempotent re-creation) + try { + statement.executeUpdate( + "DROP TRIGGER IF EXISTS trg_artifacts_search ON artifacts" + ); + statement.executeUpdate( + String.join( + "\n", + "CREATE TRIGGER trg_artifacts_search", + " BEFORE INSERT OR UPDATE ON artifacts", + " FOR EACH ROW EXECUTE FUNCTION artifacts_search_update();" + ) + ); + } catch (final SQLException ex) { + EcsLogger.debug("com.artipie.db") + .message("Failed to create trigger trg_artifacts_search") + .error(ex) + .log(); + } + // Backfill search_tokens for all rows using the same translate logic + // as the trigger — splits dots/slashes/dashes/underscores into + // separate tokens for partial matching. + try { + statement.executeUpdate( + String.join( + " ", + "UPDATE artifacts SET search_tokens = to_tsvector('simple',", + "translate(coalesce(name, ''), './-_', ' ') || ' ' ||", + "translate(coalesce(version, ''), './-_', ' ') || ' ' ||", + "coalesce(owner, '') || ' ' ||", + "translate(coalesce(repo_name, ''), './-_', ' ') || ' ' ||", + "translate(coalesce(repo_type, ''), './-_', ' '))", + "WHERE TRUE" + ) + ); + } catch (final SQLException ex) { + EcsLogger.debug("com.artipie.db") + .message("Failed to backfill search_tokens (may have no rows)") + .error(ex) + .log(); + } statement.executeUpdate( String.join( "\n", @@ -349,15 +467,21 @@ private static void createStructure(final DataSource source) { statement.executeUpdate( "ALTER TABLE artifact_cooldowns DROP CONSTRAINT IF EXISTS cooldown_parent_fk" ); - } catch (SQLException ignored) { - // Constraint may not exist + } catch (final SQLException ex) { + EcsLogger.debug("com.artipie.db") + .message("Failed to drop constraint cooldown_parent_fk (may not exist)") + .error(ex) + .log(); } try { statement.executeUpdate( "ALTER TABLE artifact_cooldowns DROP COLUMN IF EXISTS parent_block_id" ); - } catch (SQLException ignored) { - // Column may not exist + } catch (final SQLException ex) { + EcsLogger.debug("com.artipie.db") + .message("Failed to drop column parent_block_id (may not exist)") + .error(ex) + .log(); } // Migration: Drop artifact_cooldown_attempts table (no longer used) statement.executeUpdate( @@ -369,6 +493,18 @@ private static void createStructure(final DataSource source) { statement.executeUpdate( "CREATE INDEX IF NOT EXISTS idx_cooldowns_status ON artifact_cooldowns(status)" ); + // Composite index for paginated active blocks query (ORDER BY blocked_at DESC) + statement.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_cooldowns_status_blocked_at ON artifact_cooldowns(status, blocked_at DESC)" + ); + // Composite index for per-repo active block counts + statement.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_cooldowns_repo_status ON artifact_cooldowns(repo_type, repo_name, status)" + ); + // Index for server-side search within active blocks + statement.executeUpdate( + "CREATE INDEX IF NOT EXISTS idx_cooldowns_status_artifact ON artifact_cooldowns(status, artifact, repo_name)" + ); statement.executeUpdate( "UPDATE artifact_cooldowns SET status = 'INACTIVE' WHERE status = 'MANUAL'" ); diff --git a/artipie-main/src/main/java/com/artipie/db/DbConsumer.java b/artipie-main/src/main/java/com/artipie/db/DbConsumer.java index 7393fa0ad..9e63a1900 100644 --- a/artipie-main/src/main/java/com/artipie/db/DbConsumer.java +++ b/artipie-main/src/main/java/com/artipie/db/DbConsumer.java @@ -5,19 +5,22 @@ package com.artipie.db; import com.artipie.scheduling.ArtifactEvent; -import com.artipie.group.GroupNegativeCache; import com.artipie.http.log.EcsLogger; +import com.artipie.http.misc.ConfigDefaults; import io.reactivex.rxjava3.annotations.NonNull; import io.reactivex.rxjava3.core.Observer; import io.reactivex.rxjava3.disposables.Disposable; import io.reactivex.rxjava3.schedulers.Schedulers; import io.reactivex.rxjava3.subjects.PublishSubject; +import java.io.IOException; +import java.nio.file.Path; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import javax.sql.DataSource; @@ -30,12 +33,14 @@ public final class DbConsumer implements Consumer { /** * Default buffer time in seconds. */ - private static final int DEFAULT_BUFFER_TIME_SECONDS = 2; + private static final int DEFAULT_BUFFER_TIME_SECONDS = + ConfigDefaults.getInt("ARTIPIE_DB_BUFFER_SECONDS", 2); /** * Default buffer size (max events per batch). */ - private static final int DEFAULT_BUFFER_SIZE = 50; + private static final int DEFAULT_BUFFER_SIZE = + ConfigDefaults.getInt("ARTIPIE_DB_BATCH_SIZE", 200); /** * Publish subject @@ -91,8 +96,11 @@ private static String normalizeRepoName(final String name) { * @param record Artifact event that was persisted */ private static void logArtifactPublish(final ArtifactEvent record) { - final EcsLogger logger = EcsLogger.info("com.artipie.audit") - .message("Artifact publish recorded") + final String msg = record.releaseDate().isPresent() + ? String.format("Artifact publish recorded (release=%s)", record.releaseDate().get()) + : "Artifact publish recorded"; + EcsLogger.info("com.artipie.audit") + .message(msg) .eventCategory("artifact") .eventAction("artifact_publish") .eventOutcome("success") @@ -101,50 +109,8 @@ private static void logArtifactPublish(final ArtifactEvent record) { .field("package.name", record.artifactName()) .field("package.version", record.artifactVersion()) .field("package.size", record.size()) - .userName(record.owner()); - record.releaseDate().ifPresent(release -> logger.field("artifact.release", release)); - logger.log(); - } - - /** - * Invalidate group negative cache for a package. - * This ensures newly published packages are immediately visible via group repos. - * @param packageName Package name (e.g., "@retail/backoffice-interaction-notes") - */ - private static void invalidateGroupNegativeCache(final String packageName) { - try { - GroupNegativeCache.invalidatePackageGlobally(packageName) - .whenComplete((v, err) -> { - if (err != null) { - EcsLogger.warn("com.artipie.db") - .message("Failed to invalidate group negative cache") - .eventCategory("cache") - .eventAction("invalidate_negative_cache") - .eventOutcome("failure") - .field("package.name", packageName) - .error(err) - .log(); - } else { - EcsLogger.debug("com.artipie.db") - .message("Invalidated group negative cache for published package") - .eventCategory("cache") - .eventAction("invalidate_negative_cache") - .eventOutcome("success") - .field("package.name", packageName) - .log(); - } - }); - } catch (final Exception ex) { - // Don't fail the publish if cache invalidation fails - EcsLogger.warn("com.artipie.db") - .message("Exception during group negative cache invalidation") - .eventCategory("cache") - .eventAction("invalidate_negative_cache") - .eventOutcome("failure") - .field("package.name", packageName) - .error(ex) - .log(); - } + .userName(record.owner()) + .log(); } /** @@ -153,6 +119,12 @@ private static void invalidateGroupNegativeCache(final String packageName) { */ private final class DbObserver implements Observer> { + /** + * Tracks consecutive batch commit failures to prevent infinite re-queuing. + * Reset to 0 on successful commit; events are dropped after 3 consecutive failures. + */ + private final AtomicInteger consecutiveFailures = new AtomicInteger(0); + @Override public void onSubscribe(final @NonNull Disposable disposable) { EcsLogger.debug("com.artipie.db") @@ -182,11 +154,12 @@ public void onNext(final @NonNull List events) { try ( Connection conn = DbConsumer.this.source.getConnection(); PreparedStatement upsert = conn.prepareStatement( - "INSERT INTO artifacts (repo_type, repo_name, name, version, size, created_date, release_date, owner) " + - "VALUES (?,?,?,?,?,?,?,?) " + + "INSERT INTO artifacts (repo_type, repo_name, name, version, size, created_date, release_date, owner, path_prefix) " + + "VALUES (?,?,?,?,?,?,?,?,?) " + "ON CONFLICT (repo_name, name, version) " + "DO UPDATE SET repo_type = EXCLUDED.repo_type, size = EXCLUDED.size, " + - "created_date = EXCLUDED.created_date, release_date = EXCLUDED.release_date, owner = EXCLUDED.owner" + "created_date = EXCLUDED.created_date, release_date = EXCLUDED.release_date, " + + "owner = EXCLUDED.owner, path_prefix = COALESCE(EXCLUDED.path_prefix, artifacts.path_prefix)" ); PreparedStatement deletev = conn.prepareStatement( "DELETE FROM artifacts WHERE repo_name = ? AND name = ? AND version = ?;" @@ -209,13 +182,9 @@ public void onNext(final @NonNull List events) { upsert.setLong(6, record.createdDate()); upsert.setLong(7, release); upsert.setString(8, record.owner()); + upsert.setString(9, record.pathPrefix()); upsert.execute(); logArtifactPublish(record); - // Invalidate group negative cache for npm packages - // This ensures newly published packages are immediately visible via group repos - if ("npm".equals(record.repoType())) { - invalidateGroupNegativeCache(record.artifactName()); - } } else if (record.eventType() == ArtifactEvent.Type.DELETE_VERSION) { deletev.setString(1, normalizeRepoName(record.repoName())); deletev.setString(2, record.artifactName()); @@ -240,19 +209,71 @@ public void onNext(final @NonNull List events) { } } conn.commit(); + this.consecutiveFailures.set(0); } catch (final SQLException ex) { - EcsLogger.error("com.artipie.db") - .message("Failed to commit artifact events batch (" + sortedEvents.size() + " events)") - .eventCategory("database") - .eventAction("batch_commit") - .eventOutcome("failure") - .error(ex) - .log(); - sortedEvents.forEach(DbConsumer.this.subject::onNext); + final int failures = this.consecutiveFailures.incrementAndGet(); + if (failures <= 3) { + EcsLogger.error("com.artipie.db") + .message("Batch commit failed, re-queuing " + sortedEvents.size() + + " events (attempt " + failures + "/3)") + .eventCategory("database") + .eventAction("batch_commit") + .eventOutcome("failure") + .error(ex) + .log(); + final long backoffMs = Math.min( + 1000L * (1L << (failures - 1)), 8000L + ); + try { + Thread.sleep(backoffMs); + } catch (final InterruptedException ie) { + Thread.currentThread().interrupt(); + } + sortedEvents.forEach(DbConsumer.this.subject::onNext); + } else { + EcsLogger.error("com.artipie.db") + .message("Writing " + sortedEvents.size() + + " events to dead-letter after " + failures + + " consecutive batch failures") + .eventCategory("database") + .eventAction("batch_dead_letter") + .eventOutcome("failure") + .error(ex) + .log(); + try { + final DeadLetterWriter dlWriter = new DeadLetterWriter( + Path.of(System.getProperty( + "artipie.home", "/var/artipie" + )).resolve(".dead-letter") + ); + dlWriter.write(sortedEvents, ex, failures); + } catch (final IOException dlError) { + EcsLogger.error("com.artipie.db") + .message(String.format( + "Failed to write dead-letter file, dropping %d events", + sortedEvents.size())) + .eventCategory("database") + .eventAction("dead_letter_write") + .eventOutcome("failure") + .error(dlError) + .log(); + } + } error = true; } - if (!error) { - errors.forEach(DbConsumer.this.subject::onNext); + if (!error && !errors.isEmpty()) { + if (errors.size() <= 5) { + // Only re-queue a small number of individual errors + errors.forEach(DbConsumer.this.subject::onNext); + } else { + EcsLogger.error("com.artipie.db") + .message("Dropping " + errors.size() + + " individually failed events (too many errors in batch)") + .eventCategory("database") + .eventAction("event_drop") + .eventOutcome("failure") + .log(); + } } } diff --git a/artipie-main/src/main/java/com/artipie/db/DbManager.java b/artipie-main/src/main/java/com/artipie/db/DbManager.java new file mode 100644 index 000000000..27756da3d --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/DbManager.java @@ -0,0 +1,47 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db; + +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Runs Flyway database migrations. + * Called once from VertxMain.start() before verticle deployment. + * @since 1.0 + */ +public final class DbManager { + + private static final Logger LOG = LoggerFactory.getLogger(DbManager.class); + + private DbManager() { + } + + /** + * Run all pending Flyway migrations. + * baselineVersion("99") ensures Flyway skips versions 1-99. + * The existing artifact tables (artifacts, artifact_cooldowns, import_sessions) + * are still managed by ArtifactDbFactory.createStructure() with IF NOT EXISTS. + * Only V100+ migrations (settings tables) are managed by Flyway. + * This will be consolidated in a future phase. + * @param datasource HikariCP DataSource + */ + public static void migrate(final DataSource datasource) { + LOG.info("Running Flyway database migrations..."); + final Flyway flyway = Flyway.configure() + .dataSource(datasource) + .locations("classpath:db/migration") + .baselineOnMigrate(true) + .baselineVersion("99") + .load(); + final var result = flyway.migrate(); + LOG.info( + "Flyway migration complete: {} migrations applied", + result.migrationsExecuted + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/DeadLetterWriter.java b/artipie-main/src/main/java/com/artipie/db/DeadLetterWriter.java new file mode 100644 index 000000000..398a455c8 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/DeadLetterWriter.java @@ -0,0 +1,94 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db; + +import com.artipie.http.log.EcsLogger; +import com.artipie.scheduling.ArtifactEvent; + +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObjectBuilder; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.List; + +/** + * Writes failed database events to a dead-letter JSON file. + * Events that cannot be persisted after all retries are written here + * for manual review and replay. + * + * @since 1.20.13 + */ +public final class DeadLetterWriter { + + /** + * Base directory for dead-letter files. + */ + private final Path baseDir; + + /** + * Constructor. + * @param baseDir Base directory (e.g., /var/artipie/.dead-letter/) + */ + public DeadLetterWriter(final Path baseDir) { + this.baseDir = baseDir; + } + + /** + * Write failed events to a dead-letter file. + * + * @param events Events that failed to persist + * @param error The error that caused the failure + * @param retryCount Number of retries attempted + * @return Path of the dead-letter file written + * @throws IOException If writing fails + */ + public Path write(final List events, final Throwable error, + final int retryCount) throws IOException { + Files.createDirectories(this.baseDir); + final String timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now()); + final String filename = String.format("db-events-%s.json", + timestamp.replace(':', '-')); + final Path file = this.baseDir.resolve(filename); + final JsonObjectBuilder root = Json.createObjectBuilder() + .add("timestamp", timestamp) + .add("retryCount", retryCount) + .add("error", error.toString()) + .add("eventCount", events.size()); + final JsonArrayBuilder eventsArray = Json.createArrayBuilder(); + for (final ArtifactEvent event : events) { + eventsArray.add( + Json.createObjectBuilder() + .add("repoType", event.repoType()) + .add("repoName", event.repoName()) + .add("artifactName", event.artifactName()) + .add("version", event.artifactVersion()) + .add("owner", event.owner()) + .add("size", event.size()) + .add("created", event.createdDate()) + .add("eventType", event.eventType().name()) + ); + } + root.add("events", eventsArray); + try (Writer writer = Files.newBufferedWriter(file, + StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + Json.createWriter(writer).writeObject(root.build()); + } + EcsLogger.error("com.artipie.db") + .message(String.format("Wrote %d failed events to dead-letter file: %s", + events.size(), file)) + .eventCategory("database") + .eventAction("dead_letter_write") + .eventOutcome("success") + .field("file.path", file.toString()) + .log(); + return file; + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/AuditLogDao.java b/artipie-main/src/main/java/com/artipie/db/dao/AuditLogDao.java new file mode 100644 index 000000000..cbfb69e7a --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/AuditLogDao.java @@ -0,0 +1,47 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import javax.json.JsonObject; +import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class AuditLogDao { + + private static final Logger LOG = LoggerFactory.getLogger(AuditLogDao.class); + + private static final String INSERT = String.join( + " ", + "INSERT INTO audit_log (actor, action, resource_type, resource_name,", + "old_value, new_value) VALUES (?, ?, ?, ?, ?::jsonb, ?::jsonb)" + ); + + private final DataSource source; + + public AuditLogDao(final DataSource source) { + this.source = source; + } + + public void log( + final String actor, final String action, final String resourceType, + final String resourceName, final JsonObject oldValue, final JsonObject newValue + ) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(INSERT)) { + ps.setString(1, actor); + ps.setString(2, action); + ps.setString(3, resourceType); + ps.setString(4, resourceName); + ps.setString(5, oldValue != null ? oldValue.toString() : null); + ps.setString(6, newValue != null ? newValue.toString() : null); + ps.executeUpdate(); + } catch (final Exception ex) { + LOG.error("Failed to write audit log: {} {} {}", action, resourceType, resourceName, ex); + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/AuthProviderDao.java b/artipie-main/src/main/java/com/artipie/db/dao/AuthProviderDao.java new file mode 100644 index 000000000..0e685f3ee --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/AuthProviderDao.java @@ -0,0 +1,126 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.io.StringReader; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; + +public final class AuthProviderDao { + + private final DataSource source; + + public AuthProviderDao(final DataSource source) { + this.source = source; + } + + public List list() { + return query("SELECT id, type, priority, config, enabled FROM auth_providers ORDER BY priority"); + } + + public List listEnabled() { + return query("SELECT id, type, priority, config, enabled FROM auth_providers WHERE enabled = TRUE ORDER BY priority"); + } + + /** + * UPSERT by type. Uses ON CONFLICT (type) since type has UNIQUE constraint. + */ + public void put(final String type, final int priority, final JsonObject config) { + final String sql = String.join(" ", + "INSERT INTO auth_providers (type, priority, config) VALUES (?, ?, ?::jsonb)", + "ON CONFLICT (type) DO UPDATE SET priority = ?, config = ?::jsonb" + ); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + final String json = config.toString(); + ps.setString(1, type); + ps.setInt(2, priority); + ps.setString(3, json); + ps.setInt(4, priority); + ps.setString(5, json); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to put auth provider: " + type, ex); + } + } + + /** + * Update the config JSON for an existing auth provider by ID. + * @param id Provider ID + * @param config New config JSON + */ + public void updateConfig(final int id, final JsonObject config) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE auth_providers SET config = ?::jsonb WHERE id = ?" + )) { + ps.setString(1, config.toString()); + ps.setInt(2, id); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to update auth provider config: " + id, ex); + } + } + + public void delete(final int id) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "DELETE FROM auth_providers WHERE id = ?" + )) { + ps.setInt(1, id); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to delete auth provider: " + id, ex); + } + } + + public void enable(final int id) { + setEnabled(id, true); + } + + public void disable(final int id) { + setEnabled(id, false); + } + + private void setEnabled(final int id, final boolean enabled) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE auth_providers SET enabled = ? WHERE id = ?" + )) { + ps.setBoolean(1, enabled); + ps.setInt(2, id); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to toggle auth provider: " + id, ex); + } + } + + private List query(final String sql) { + final List result = new ArrayList<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + result.add(Json.createObjectBuilder() + .add("id", rs.getInt("id")) + .add("type", rs.getString("type")) + .add("priority", rs.getInt("priority")) + .add("config", Json.createReader( + new StringReader(rs.getString("config"))).readObject()) + .add("enabled", rs.getBoolean("enabled")) + .build()); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to query auth providers", ex); + } + return result; + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/RepositoryDao.java b/artipie-main/src/main/java/com/artipie/db/dao/RepositoryDao.java new file mode 100644 index 000000000..6e4a54e1e --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/RepositoryDao.java @@ -0,0 +1,163 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.io.StringReader; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import javax.json.Json; +import javax.json.JsonStructure; +import javax.sql.DataSource; +import com.artipie.api.RepositoryName; +import com.artipie.settings.repo.CrudRepoSettings; + +/** + * PostgreSQL-backed repository configuration storage. + * Drop-in replacement for ManageRepoSettings. + */ +public final class RepositoryDao implements CrudRepoSettings { + + private final DataSource source; + + public RepositoryDao(final DataSource source) { + this.source = source; + } + + @Override + public Collection listAll() { + final List result = new ArrayList<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT name FROM repositories ORDER BY name" + )) { + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + result.add(rs.getString("name")); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to list repositories", ex); + } + return result; + } + + @Override + public Collection list(final String uname) { + // For now, returns all repos. User-scoped filtering will be added + // when the API layer applies permission checks. + return this.listAll(); + } + + @Override + public boolean exists(final RepositoryName rname) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT 1 FROM repositories WHERE name = ?" + )) { + ps.setString(1, rname.toString()); + return ps.executeQuery().next(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to check repo: " + rname, ex); + } + } + + @Override + public JsonStructure value(final RepositoryName rname) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT config FROM repositories WHERE name = ?" + )) { + ps.setString(1, rname.toString()); + final ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return Json.createReader( + new StringReader(rs.getString("config")) + ).readObject(); + } + throw new IllegalStateException("Repository not found: " + rname); + } catch (final IllegalStateException ex) { + throw ex; + } catch (final Exception ex) { + throw new IllegalStateException("Failed to get repo: " + rname, ex); + } + } + + @Override + public void save(final RepositoryName rname, final JsonStructure value) { + this.save(rname, value, (String) null); + } + + @Override + public void save(final RepositoryName rname, final JsonStructure value, + final String actor) { + final String type = extractType(value); + final String sql = String.join(" ", + "INSERT INTO repositories (name, type, config, created_by, updated_by)", + "VALUES (?, ?, ?::jsonb, ?, ?)", + "ON CONFLICT (name) DO UPDATE SET type = ?, config = ?::jsonb,", + "updated_at = NOW(), updated_by = ?" + ); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + final String json = value.toString(); + ps.setString(1, rname.toString()); + ps.setString(2, type); + ps.setString(3, json); + ps.setString(4, actor); + ps.setString(5, actor); + ps.setString(6, type); + ps.setString(7, json); + ps.setString(8, actor); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to save repo: " + rname, ex); + } + } + + @Override + public void delete(final RepositoryName rname) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "DELETE FROM repositories WHERE name = ?" + )) { + ps.setString(1, rname.toString()); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to delete repo: " + rname, ex); + } + } + + @Override + public void move(final RepositoryName rname, final RepositoryName newrname) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE repositories SET name = ?, updated_at = NOW() WHERE name = ?" + )) { + ps.setString(1, newrname.toString()); + ps.setString(2, rname.toString()); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException( + "Failed to move repo: " + rname + " -> " + newrname, ex + ); + } + } + + @Override + public boolean hasSettingsDuplicates(final RepositoryName rname) { + return false; + } + + private static String extractType(final JsonStructure value) { + try { + return value.asJsonObject().getJsonObject("repo").getString("type"); + } catch (final Exception ex) { + return "unknown"; + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/RoleDao.java b/artipie-main/src/main/java/com/artipie/db/dao/RoleDao.java new file mode 100644 index 000000000..d7fdcbfcb --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/RoleDao.java @@ -0,0 +1,139 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.io.StringReader; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.sql.DataSource; +import com.artipie.settings.users.CrudRoles; + +/** + * PostgreSQL-backed role storage. + * Drop-in replacement for ManageRoles. + */ +public final class RoleDao implements CrudRoles { + + private final DataSource source; + + public RoleDao(final DataSource source) { + this.source = source; + } + + @Override + public JsonArray list() { + final JsonArrayBuilder arr = Json.createArrayBuilder(); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT name, permissions, enabled FROM roles ORDER BY name" + )) { + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + arr.add(roleFromRow(rs)); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to list roles", ex); + } + return arr.build(); + } + + @Override + public Optional get(final String rname) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT name, permissions, enabled FROM roles WHERE name = ?" + )) { + ps.setString(1, rname); + final ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return Optional.of(roleFromRow(rs)); + } + return Optional.empty(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to get role: " + rname, ex); + } + } + + @Override + public void addOrUpdate(final JsonObject info, final String rname) { + final String sql = String.join(" ", + "INSERT INTO roles (name, permissions) VALUES (?, ?::jsonb)", + "ON CONFLICT (name) DO UPDATE SET permissions = ?::jsonb,", + "updated_at = NOW()" + ); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + final String permsJson = info.toString(); + ps.setString(1, rname); + ps.setString(2, permsJson); + ps.setString(3, permsJson); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to add/update role: " + rname, ex); + } + } + + @Override + public void disable(final String rname) { + setEnabled(rname, false); + } + + @Override + public void enable(final String rname) { + setEnabled(rname, true); + } + + @Override + public void remove(final String rname) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "DELETE FROM roles WHERE name = ?" + )) { + ps.setString(1, rname); + final int rows = ps.executeUpdate(); + if (rows == 0) { + throw new IllegalStateException("Role not found: " + rname); + } + } catch (final IllegalStateException ex) { + throw ex; + } catch (final Exception ex) { + throw new IllegalStateException("Failed to remove role: " + rname, ex); + } + } + + private void setEnabled(final String rname, final boolean enabled) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE roles SET enabled = ?, updated_at = NOW() WHERE name = ?" + )) { + ps.setBoolean(1, enabled); + ps.setString(2, rname); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to toggle role: " + rname, ex); + } + } + + private static JsonObject roleFromRow(final ResultSet rs) throws Exception { + final JsonObject perms = Json.createReader( + new StringReader(rs.getString("permissions")) + ).readObject(); + final javax.json.JsonObjectBuilder bld = Json.createObjectBuilder() + .add("name", rs.getString("name")) + .add("enabled", rs.getBoolean("enabled")); + // Merge permission keys at top level (not nested under "permissions") + for (final String key : perms.keySet()) { + bld.add(key, perms.get(key)); + } + return bld.build(); + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/SettingsDao.java b/artipie-main/src/main/java/com/artipie/db/dao/SettingsDao.java new file mode 100644 index 000000000..c7df561ac --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/SettingsDao.java @@ -0,0 +1,94 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.io.StringReader; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; + +public final class SettingsDao { + + private final DataSource source; + + public SettingsDao(final DataSource source) { + this.source = source; + } + + public void put(final String key, final JsonObject value, final String actor) { + final String sql = String.join(" ", + "INSERT INTO settings (key, value, updated_by) VALUES (?, ?::jsonb, ?)", + "ON CONFLICT (key) DO UPDATE SET value = ?::jsonb,", + "updated_at = NOW(), updated_by = ?" + ); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + final String json = value.toString(); + ps.setString(1, key); + ps.setString(2, json); + ps.setString(3, actor); + ps.setString(4, json); + ps.setString(5, actor); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to put setting: " + key, ex); + } + } + + public Optional get(final String key) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT value FROM settings WHERE key = ?" + )) { + ps.setString(1, key); + final ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return Optional.of( + Json.createReader(new StringReader(rs.getString("value"))).readObject() + ); + } + return Optional.empty(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to get setting: " + key, ex); + } + } + + public Map listAll() { + final Map result = new LinkedHashMap<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT key, value FROM settings ORDER BY key" + )) { + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + result.put( + rs.getString("key"), + Json.createReader(new StringReader(rs.getString("value"))).readObject() + ); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to list settings", ex); + } + return result; + } + + public void delete(final String key) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "DELETE FROM settings WHERE key = ?" + )) { + ps.setString(1, key); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to delete setting: " + key, ex); + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/StorageAliasDao.java b/artipie-main/src/main/java/com/artipie/db/dao/StorageAliasDao.java new file mode 100644 index 000000000..78668a065 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/StorageAliasDao.java @@ -0,0 +1,128 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.io.StringReader; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; + +public final class StorageAliasDao { + + private final DataSource source; + + public StorageAliasDao(final DataSource source) { + this.source = source; + } + + public List listGlobal() { + return listByRepo(null); + } + + public List listForRepo(final String repoName) { + return listByRepo(repoName); + } + + public void put(final String name, final String repoName, final JsonObject config) { + final String sql; + if (repoName == null) { + sql = String.join(" ", + "INSERT INTO storage_aliases (name, repo_name, config)", + "VALUES (?, NULL, ?::jsonb)", + "ON CONFLICT (name) WHERE repo_name IS NULL", + "DO UPDATE SET config = ?::jsonb, updated_at = NOW()" + ); + } else { + sql = String.join(" ", + "INSERT INTO storage_aliases (name, repo_name, config)", + "VALUES (?, ?, ?::jsonb)", + "ON CONFLICT (name, repo_name)", + "DO UPDATE SET config = ?::jsonb, updated_at = NOW()" + ); + } + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + final String json = config.toString(); + ps.setString(1, name); + if (repoName == null) { + ps.setString(2, json); + ps.setString(3, json); + } else { + ps.setString(2, repoName); + ps.setString(3, json); + ps.setString(4, json); + } + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to put alias: " + name, ex); + } + } + + public void delete(final String name, final String repoName) { + final String sql = repoName == null + ? "DELETE FROM storage_aliases WHERE name = ? AND repo_name IS NULL" + : "DELETE FROM storage_aliases WHERE name = ? AND repo_name = ?"; + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, name); + if (repoName != null) { + ps.setString(2, repoName); + } + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to delete alias: " + name, ex); + } + } + + /** + * Find repos whose config JSONB references this alias name via + * the `repo.storage` field. + */ + public List findReposUsing(final String aliasName) { + final List result = new ArrayList<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT name FROM repositories WHERE config->'repo'->>'storage' = ?" + )) { + ps.setString(1, aliasName); + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + result.add(rs.getString("name")); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to find repos using: " + aliasName, ex); + } + return result; + } + + private List listByRepo(final String repoName) { + final List result = new ArrayList<>(); + final String sql = repoName == null + ? "SELECT name, config FROM storage_aliases WHERE repo_name IS NULL ORDER BY name" + : "SELECT name, config FROM storage_aliases WHERE repo_name = ? ORDER BY name"; + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + if (repoName != null) { + ps.setString(1, repoName); + } + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + result.add(Json.createObjectBuilder() + .add("name", rs.getString("name")) + .add("config", Json.createReader( + new StringReader(rs.getString("config"))).readObject()) + .build()); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to list aliases", ex); + } + return result; + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/UserDao.java b/artipie-main/src/main/java/com/artipie/db/dao/UserDao.java new file mode 100644 index 000000000..354853f98 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/UserDao.java @@ -0,0 +1,294 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import com.artipie.http.log.EcsLogger; +import java.io.StringReader; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.sql.DataSource; +import com.artipie.settings.users.CrudUsers; + +/** + * PostgreSQL-backed user storage. + * Drop-in replacement for ManageUsers. + */ +public final class UserDao implements CrudUsers { + + private final DataSource source; + + public UserDao(final DataSource source) { + this.source = source; + } + + @Override + public JsonArray list() { + final JsonArrayBuilder arr = Json.createArrayBuilder(); + final String sql = String.join(" ", + "SELECT u.username, u.email, u.enabled, u.auth_provider,", + "COALESCE(json_agg(r.name) FILTER (WHERE r.name IS NOT NULL), '[]') AS roles", + "FROM users u", + "LEFT JOIN user_roles ur ON u.id = ur.user_id", + "LEFT JOIN roles r ON ur.role_id = r.id", + "GROUP BY u.id ORDER BY u.username" + ); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + arr.add(userFromRow(rs)); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to list users", ex); + } + return arr.build(); + } + + @Override + public Optional get(final String uname) { + final String sql = String.join(" ", + "SELECT u.username, u.email, u.enabled, u.auth_provider,", + "COALESCE(json_agg(r.name) FILTER (WHERE r.name IS NOT NULL), '[]') AS roles", + "FROM users u", + "LEFT JOIN user_roles ur ON u.id = ur.user_id", + "LEFT JOIN roles r ON ur.role_id = r.id", + "WHERE u.username = ?", + "GROUP BY u.id" + ); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, uname); + final ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return Optional.of(userFromRow(rs)); + } + return Optional.empty(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to get user: " + uname, ex); + } + } + + @Override + public void addOrUpdate(final JsonObject info, final String uname) { + final String sql = String.join(" ", + "INSERT INTO users (username, password_hash, email, auth_provider)", + "VALUES (?, ?, ?, ?)", + "ON CONFLICT (username) DO UPDATE SET", + "password_hash = COALESCE(?, users.password_hash),", + "email = COALESCE(?, users.email),", + "auth_provider = COALESCE(?, users.auth_provider),", + "updated_at = NOW()" + ); + try (Connection conn = this.source.getConnection()) { + conn.setAutoCommit(false); + try { + final String pass; + if (info.containsKey("pass")) { + pass = info.getString("pass"); + } else if (info.containsKey("password")) { + pass = info.getString("password"); + } else { + pass = null; + } + final String email = info.containsKey("email") + ? info.getString("email") : null; + // Map password format types (plain, sha256) to "artipie" provider. + // Only actual provider names (keycloak, okta) are stored literally. + final String rawType = info.containsKey("type") + ? info.getString("type") : "artipie"; + final String provider = "plain".equals(rawType) || "sha256".equals(rawType) + ? "artipie" : rawType; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, uname); + ps.setString(2, pass); + ps.setString(3, email); + ps.setString(4, provider); + ps.setString(5, pass); + ps.setString(6, email); + ps.setString(7, provider); + ps.executeUpdate(); + } + // Update role assignments if roles are provided + if (info.containsKey("roles")) { + updateUserRoles(conn, uname, info.getJsonArray("roles")); + } + conn.commit(); + } catch (final Exception ex) { + conn.rollback(); + throw ex; + } finally { + conn.setAutoCommit(true); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to add/update user: " + uname, ex); + } + } + + @Override + public void disable(final String uname) { + setEnabled(uname, false); + } + + @Override + public void enable(final String uname) { + setEnabled(uname, true); + } + + @Override + public void remove(final String uname) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "DELETE FROM users WHERE username = ?" + )) { + ps.setString(1, uname); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to remove user: " + uname, ex); + } + } + + @Override + public void alterPassword(final String uname, final JsonObject info) { + final String newPass = info.getString("new_pass"); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE users SET password_hash = ?, updated_at = NOW() WHERE username = ?" + )) { + ps.setString(1, newPass); + ps.setString(2, uname); + final int rows = ps.executeUpdate(); + if (rows == 0) { + throw new IllegalStateException("User not found: " + uname); + } + } catch (final IllegalStateException ex) { + throw ex; + } catch (final Exception ex) { + throw new IllegalStateException("Failed to alter password: " + uname, ex); + } + } + + private void setEnabled(final String uname, final boolean enabled) { + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE users SET enabled = ?, updated_at = NOW() WHERE username = ?" + )) { + ps.setBoolean(1, enabled); + ps.setString(2, uname); + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to toggle user: " + uname, ex); + } + } + + private static void updateUserRoles(final Connection conn, final String uname, + final JsonArray roles) throws Exception { + // Get user ID + final int userId; + try (PreparedStatement ps = conn.prepareStatement( + "SELECT id FROM users WHERE username = ?")) { + ps.setString(1, uname); + final ResultSet rs = ps.executeQuery(); + if (!rs.next()) { + EcsLogger.warn("com.artipie.db") + .message("updateUserRoles: user not found in DB after insert") + .eventCategory("user") + .eventAction("role_assignment") + .eventOutcome("failure") + .field("user.name", uname) + .log(); + return; + } + userId = rs.getInt("id"); + } + // Delete existing role assignments + try (PreparedStatement ps = conn.prepareStatement( + "DELETE FROM user_roles WHERE user_id = ?")) { + ps.setInt(1, userId); + final int deleted = ps.executeUpdate(); + EcsLogger.info("com.artipie.db") + .message("updateUserRoles: cleared existing roles") + .eventCategory("user") + .eventAction("role_assignment") + .field("user.name", uname) + .field("user.id", userId) + .field("deleted.count", deleted) + .log(); + } + // Insert new role assignments + if (roles != null && !roles.isEmpty()) { + final java.util.List roleNames = new java.util.ArrayList<>(); + for (int idx = 0; idx < roles.size(); idx++) { + roleNames.add(roles.getString(idx)); + } + EcsLogger.info("com.artipie.db") + .message("updateUserRoles: assigning roles") + .eventCategory("user") + .eventAction("role_assignment") + .field("user.name", uname) + .field("user.id", userId) + .field("roles", String.join(",", roleNames)) + .field("roles.count", roleNames.size()) + .log(); + // Auto-create roles that don't exist yet (e.g. SSO default "reader") + try (PreparedStatement ps = conn.prepareStatement( + "INSERT INTO roles (name, permissions) VALUES (?, '{}'::jsonb) " + + "ON CONFLICT (name) DO NOTHING")) { + for (final String roleName : roleNames) { + ps.setString(1, roleName); + ps.executeUpdate(); + } + } + try (PreparedStatement ps = conn.prepareStatement( + "INSERT INTO user_roles (user_id, role_id) " + + "SELECT ?, id FROM roles WHERE name = ?")) { + for (final String roleName : roleNames) { + ps.setInt(1, userId); + ps.setString(2, roleName); + final int inserted = ps.executeUpdate(); + EcsLogger.info("com.artipie.db") + .message("updateUserRoles: role assignment result") + .eventCategory("user") + .eventAction("role_assignment") + .field("user.name", uname) + .field("role.name", roleName) + .field("rows.inserted", inserted) + .log(); + } + } + } else { + EcsLogger.warn("com.artipie.db") + .message("updateUserRoles: no roles to assign") + .eventCategory("user") + .eventAction("role_assignment") + .field("user.name", uname) + .field("roles.null", roles == null) + .log(); + } + } + + private static JsonObject userFromRow(final ResultSet rs) throws Exception { + final JsonObjectBuilder bld = Json.createObjectBuilder() + .add("name", rs.getString("username")) + .add("enabled", rs.getBoolean("enabled")) + .add("auth_provider", rs.getString("auth_provider")); + final String email = rs.getString("email"); + if (email != null) { + bld.add("email", email); + } + final String rolesJson = rs.getString("roles"); + if (rolesJson != null) { + bld.add("roles", + Json.createReader(new StringReader(rolesJson)).readArray()); + } + return bld.build(); + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/dao/UserTokenDao.java b/artipie-main/src/main/java/com/artipie/db/dao/UserTokenDao.java new file mode 100644 index 000000000..8da40dd14 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/dao/UserTokenDao.java @@ -0,0 +1,194 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.UUID; +import javax.sql.DataSource; + +/** + * DAO for user API tokens (user_tokens table). + * @since 1.21.0 + */ +public final class UserTokenDao { + + /** + * Database data source. + */ + private final DataSource source; + + /** + * Ctor. + * @param source Database data source + */ + public UserTokenDao(final DataSource source) { + this.source = source; + } + + /** + * Store a newly issued token. + * @param id Token UUID (same as jti claim) + * @param username Username + * @param label Human-readable label + * @param tokenValue Raw JWT string (hashed before storage) + * @param expiresAt Expiry timestamp, null for permanent + */ + public void store(final UUID id, final String username, final String label, + final String tokenValue, final Instant expiresAt) { + final String sql = String.join(" ", + "INSERT INTO user_tokens (id, username, label, token_hash, expires_at)", + "VALUES (?, ?, ?, ?, ?)" + ); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, id); + ps.setString(2, username); + ps.setString(3, label); + ps.setString(4, sha256(tokenValue)); + if (expiresAt != null) { + ps.setTimestamp(5, Timestamp.from(expiresAt)); + } else { + ps.setNull(5, java.sql.Types.TIMESTAMP); + } + ps.executeUpdate(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to store token", ex); + } + } + + /** + * List active (non-revoked) tokens for a user. + * @param username Username + * @return List of token info records + */ + public List listByUser(final String username) { + final String sql = String.join(" ", + "SELECT id, label, expires_at, created_at", + "FROM user_tokens", + "WHERE username = ? AND revoked = FALSE", + "ORDER BY created_at DESC" + ); + final List result = new ArrayList<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, username); + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + final Timestamp exp = rs.getTimestamp("expires_at"); + result.add(new TokenInfo( + rs.getObject("id", UUID.class), + rs.getString("label"), + exp != null ? exp.toInstant() : null, + rs.getTimestamp("created_at").toInstant() + )); + } + } catch (final Exception ex) { + throw new IllegalStateException("Failed to list tokens", ex); + } + return result; + } + + /** + * Revoke a token by ID for a given user. + * @param id Token UUID + * @param username Username (ownership check) + * @return True if token was revoked + */ + public boolean revoke(final UUID id, final String username) { + final String sql = + "UPDATE user_tokens SET revoked = TRUE WHERE id = ? AND username = ?"; + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, id); + ps.setString(2, username); + return ps.executeUpdate() > 0; + } catch (final Exception ex) { + throw new IllegalStateException("Failed to revoke token", ex); + } + } + + /** + * Check if a token ID is valid (exists and not revoked). + * @param id Token UUID (jti) + * @return True if valid + */ + public boolean isValid(final UUID id) { + final String sql = + "SELECT 1 FROM user_tokens WHERE id = ? AND revoked = FALSE"; + try (Connection conn = this.source.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, id); + return ps.executeQuery().next(); + } catch (final Exception ex) { + throw new IllegalStateException("Failed to check token validity", ex); + } + } + + /** + * SHA-256 hash of a token value. + * @param value Token string + * @return Hex-encoded hash + */ + private static String sha256(final String value) { + try { + final byte[] hash = MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(hash); + } catch (final NoSuchAlgorithmException ex) { + throw new IllegalStateException(ex); + } + } + + /** + * Token metadata record. + */ + public static final class TokenInfo { + + private final UUID id; + private final String label; + private final Instant expiresAt; + private final Instant createdAt; + + /** + * Ctor. + * @param id Token ID + * @param label Human-readable label + * @param expiresAt Expiry (null = permanent) + * @param createdAt Creation timestamp + */ + public TokenInfo(final UUID id, final String label, + final Instant expiresAt, final Instant createdAt) { + this.id = id; + this.label = label; + this.expiresAt = expiresAt; + this.createdAt = createdAt; + } + + public UUID id() { + return this.id; + } + + public String label() { + return this.label; + } + + public Instant expiresAt() { + return this.expiresAt; + } + + public Instant createdAt() { + return this.createdAt; + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/db/migration/YamlToDbMigrator.java b/artipie-main/src/main/java/com/artipie/db/migration/YamlToDbMigrator.java new file mode 100644 index 000000000..5c581f268 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/db/migration/YamlToDbMigrator.java @@ -0,0 +1,450 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.migration; + +import com.amihaiemil.eoyaml.Node; +import com.amihaiemil.eoyaml.Yaml; +import com.amihaiemil.eoyaml.YamlMapping; +import com.amihaiemil.eoyaml.YamlNode; +import com.amihaiemil.eoyaml.YamlSequence; +import com.artipie.api.RepositoryName; +import com.artipie.db.dao.AuthProviderDao; +import com.artipie.db.dao.RoleDao; +import com.artipie.db.dao.RepositoryDao; +import com.artipie.db.dao.SettingsDao; +import com.artipie.db.dao.StorageAliasDao; +import com.artipie.db.dao.UserDao; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.sql.DataSource; +import org.mindrot.jbcrypt.BCrypt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * One-time migration from YAML config files to PostgreSQL. + * Checks for {@code migration_completed} flag in settings table. + * If absent, reads YAML files and populates DB tables. + * If present, skips entirely. + * @since 1.0 + */ +public final class YamlToDbMigrator { + + /** + * Logger. + */ + private static final Logger LOG = LoggerFactory.getLogger(YamlToDbMigrator.class); + + /** + * Settings key for migration flag. + */ + private static final String MIGRATION_KEY = "migration_completed"; + + /** + * Current migration version. Bump this when new migration logic + * is added. The migration re-runs (idempotently) when the stored + * version is lower than this value. + */ + private static final int MIGRATION_VERSION = 3; + + /** + * DataSource for DB access. + */ + private final DataSource source; + + /** + * Path to security directory (contains roles/, users/). + */ + private final Path securityDir; + + /** + * Path to the repos directory (contains *.yaml repo configs). + */ + private final Path reposDir; + + /** + * Path to the artipie.yml config file. + */ + private final Path artipieYml; + + /** + * Ctor. + * @param source DataSource for DB + * @param securityDir Path to the security directory (contains roles/, users/) + * @param reposDir Path to the repos directory + */ + public YamlToDbMigrator(final DataSource source, final Path securityDir, + final Path reposDir) { + this(source, securityDir, reposDir, null); + } + + /** + * Full ctor. + * @param source DataSource for DB + * @param securityDir Path to the security directory (contains roles/, users/) + * @param reposDir Path to the repos directory + * @param artipieYml Path to artipie.yml (null to skip settings migration) + */ + public YamlToDbMigrator(final DataSource source, final Path securityDir, + final Path reposDir, final Path artipieYml) { + this.source = source; + this.securityDir = securityDir; + this.reposDir = reposDir; + this.artipieYml = artipieYml; + } + + /** + * Run migration. Uses a single versioned flag: if the stored version + * is lower than {@link #MIGRATION_VERSION}, the full migration re-runs + * idempotently (all DAOs use upsert). + * @return True if migration was executed, false if skipped + */ + public boolean migrate() { + final SettingsDao settings = new SettingsDao(this.source); + final int storedVersion = settings.get(YamlToDbMigrator.MIGRATION_KEY) + .map(obj -> obj.getInt("version", 0)) + .orElse(0); + if (storedVersion >= YamlToDbMigrator.MIGRATION_VERSION) { + LOG.info("YAML-to-DB migration v{} already completed, skipping", + storedVersion); + return false; + } + LOG.info("Running YAML-to-DB migration (v{} -> v{})...", + storedVersion, YamlToDbMigrator.MIGRATION_VERSION); + this.migrateRepos(); + final Path rolesDir = this.securityDir.resolve("roles"); + if (Files.isDirectory(rolesDir)) { + this.migrateRoles(rolesDir); + } + final Path usersDir = this.securityDir.resolve("users"); + if (Files.isDirectory(usersDir)) { + this.migrateUsers(usersDir); + } + this.migrateArtipieYml(); + settings.put( + YamlToDbMigrator.MIGRATION_KEY, + Json.createObjectBuilder() + .add("completed", true) + .add("version", YamlToDbMigrator.MIGRATION_VERSION) + .add("timestamp", System.currentTimeMillis()) + .build(), + "system" + ); + LOG.info("YAML-to-DB migration v{} completed", YamlToDbMigrator.MIGRATION_VERSION); + return true; + } + + /** + * Migrate repository YAML configs from reposDir. + */ + private void migrateRepos() { + if (!Files.isDirectory(this.reposDir)) { + LOG.info("No repos directory at {}, skipping", this.reposDir); + return; + } + final RepositoryDao dao = new RepositoryDao(this.source); + try (DirectoryStream stream = + Files.newDirectoryStream(this.reposDir, "*.{yaml,yml}")) { + for (final Path file : stream) { + try { + final String name = file.getFileName().toString() + .replaceAll("\\.(yaml|yml)$", ""); + if (name.startsWith("_")) { + continue; + } + final YamlMapping yaml = Yaml.createYamlInput( + Files.readString(file) + ).readYamlMapping(); + dao.save( + new RepositoryName.Simple(name), + yamlToJson(yaml), + "migration" + ); + LOG.info("Migrated repository: {}", name); + } catch (final Exception ex) { + LOG.error("Failed to migrate repo file: {}", file, ex); + } + } + } catch (final IOException ex) { + LOG.error("Failed to read repos directory: {}", this.reposDir, ex); + } + } + + /** + * Migrate user YAML files. + * @param usersDir Path to security/users directory + */ + private void migrateUsers(final Path usersDir) { + final UserDao dao = new UserDao(this.source); + try (DirectoryStream stream = + Files.newDirectoryStream(usersDir, "*.{yaml,yml}")) { + for (final Path file : stream) { + try { + final String name = file.getFileName().toString() + .replaceAll("\\.(yaml|yml)$", ""); + final YamlMapping yaml = Yaml.createYamlInput( + Files.readString(file) + ).readYamlMapping(); + final JsonObjectBuilder builder = Json.createObjectBuilder(); + builder.add("name", name); + // Hash password with bcrypt if type is "plain" + final String pass = yaml.string("pass"); + final String credType = yaml.string("type"); + if (pass != null) { + if ("plain".equals(credType)) { + builder.add("pass", BCrypt.hashpw(pass, BCrypt.gensalt())); + } else { + builder.add("pass", pass); + } + } + // Preserve the original auth type from YAML. + // "plain" and "sha256" are password formats → map to "artipie". + // Actual provider names (okta, keycloak) are preserved. + if (credType != null + && !"plain".equals(credType) && !"sha256".equals(credType)) { + builder.add("type", credType); + } else { + builder.add("type", "artipie"); + } + if (yaml.string("email") != null) { + builder.add("email", yaml.string("email")); + } + final String enabled = yaml.string("enabled"); + builder.add( + "enabled", + enabled == null || Boolean.parseBoolean(enabled) + ); + // Migrate role assignments + final YamlSequence rolesSeq = yaml.yamlSequence("roles"); + if (rolesSeq != null) { + final JsonArrayBuilder rolesArr = Json.createArrayBuilder(); + for (final YamlNode node : rolesSeq) { + rolesArr.add(node.asScalar().value()); + } + builder.add("roles", rolesArr); + } + dao.addOrUpdate(builder.build(), name); + LOG.info("Migrated user: {}", name); + } catch (final Exception ex) { + LOG.error("Failed to migrate user file: {}", file, ex); + } + } + } catch (final IOException ex) { + LOG.error("Failed to read users directory: {}", usersDir, ex); + } + } + + /** + * Migrate role YAML files. + * @param rolesDir Path to security/roles directory + */ + private void migrateRoles(final Path rolesDir) { + final RoleDao dao = new RoleDao(this.source); + try (DirectoryStream stream = + Files.newDirectoryStream(rolesDir, "*.{yaml,yml}")) { + for (final Path file : stream) { + try { + final String name = file.getFileName().toString() + .replaceAll("\\.(yaml|yml)$", ""); + final YamlMapping yaml = Yaml.createYamlInput( + Files.readString(file) + ).readYamlMapping(); + dao.addOrUpdate(yamlToJson(yaml), name); + LOG.info("Migrated role: {}", name); + } catch (final Exception ex) { + LOG.error("Failed to migrate role file: {}", file, ex); + } + } + } catch (final IOException ex) { + LOG.error("Failed to read roles directory: {}", rolesDir, ex); + } + } + + /** + * Migrate artipie.yml meta section to settings + auth_providers tables. + * Imports ALL configuration sections: simple keys, jwt, cooldown, + * http_client, http_server, metrics, caches, global_prefixes, + * storage aliases, and auth providers. + */ + @SuppressWarnings({"PMD.CognitiveComplexity", "PMD.NPathComplexity"}) + private void migrateArtipieYml() { + if (this.artipieYml == null || !Files.isRegularFile(this.artipieYml)) { + LOG.info("No artipie.yml path provided or file not found, skipping settings migration"); + return; + } + try { + final YamlMapping yaml = Yaml.createYamlInput( + Files.readString(this.artipieYml) + ).readYamlMapping(); + final YamlMapping meta = yaml.yamlMapping("meta"); + if (meta == null) { + return; + } + final SettingsDao settings = new SettingsDao(this.source); + // Migrate simple key-value settings + for (final String key : new String[]{"layout", "port", "base_path"}) { + final String val = meta.string(key); + if (val != null) { + settings.put( + key, + Json.createObjectBuilder() + .add("value", resolveEnvVars(val)).build(), + "migration" + ); + } + } + // Migrate all nested settings sections as JSONB + for (final String section : new String[]{ + "jwt", "cooldown", "http_client", "http_server", "metrics", "caches" + }) { + final YamlMapping nested = meta.yamlMapping(section); + if (nested != null) { + settings.put(section, yamlToJson(nested), "migration"); + LOG.info("Migrated settings section: {}", section); + } + } + // Migrate global_prefixes as a JSON array + final YamlSequence prefixes = meta.yamlSequence("global_prefixes"); + if (prefixes != null) { + final JsonArrayBuilder arr = Json.createArrayBuilder(); + for (final YamlNode node : prefixes) { + arr.add(resolveEnvVars(node.asScalar().value())); + } + settings.put( + "global_prefixes", + Json.createObjectBuilder().add("prefixes", arr).build(), + "migration" + ); + LOG.info("Migrated global_prefixes"); + } + // Migrate global storage aliases from meta.storage + final YamlMapping storage = meta.yamlMapping("storage"); + if (storage != null) { + final StorageAliasDao aliasDao = new StorageAliasDao(this.source); + aliasDao.put("default", null, yamlToJson(storage)); + LOG.info("Migrated default storage alias"); + } + // Migrate named storage aliases from meta.storages + final YamlMapping storages = meta.yamlMapping("storages"); + if (storages != null) { + final StorageAliasDao aliasDao = new StorageAliasDao(this.source); + for (final YamlNode key : storages.keys()) { + final String aliasName = key.asScalar().value(); + final YamlMapping aliasConfig = storages.yamlMapping(key); + if (aliasConfig != null) { + aliasDao.put(aliasName, null, yamlToJson(aliasConfig)); + LOG.info("Migrated storage alias: {}", aliasName); + } + } + } + // Migrate auth providers (credentials list) + final YamlSequence creds = meta.yamlSequence("credentials"); + if (creds != null) { + final AuthProviderDao authDao = new AuthProviderDao(this.source); + int priority = 1; + for (final YamlNode node : creds) { + final YamlMapping provider = node.asMapping(); + final String type = provider.string("type"); + if (type != null) { + authDao.put(type, priority, yamlToJson(provider)); + priority++; + } + } + LOG.info("Migrated {} auth providers", priority - 1); + } + LOG.info("Migrated artipie.yml settings (all sections)"); + } catch (final Exception ex) { + LOG.error("Failed to migrate artipie.yml: {}", this.artipieYml, ex); + } + } + + /** + * Convert YAML mapping to JsonObject, including nested sequences and mappings. + * @param yaml YAML mapping to convert + * @return JsonObject representation + */ + static JsonObject yamlToJson(final YamlMapping yaml) { + final JsonObjectBuilder builder = Json.createObjectBuilder(); + for (final YamlNode key : yaml.keys()) { + final String keyStr = key.asScalar().value(); + final String scalar = yaml.string(keyStr); + if (scalar != null) { + final String resolved = resolveEnvVars(scalar); + // Try to preserve booleans and numbers + if ("true".equals(resolved) || "false".equals(resolved)) { + builder.add(keyStr, Boolean.parseBoolean(resolved)); + } else { + try { + builder.add(keyStr, Long.parseLong(resolved)); + } catch (final NumberFormatException nfe) { + builder.add(keyStr, resolved); + } + } + } else { + // Try sequence before mapping: eo-yaml may interpret + // a sequence of single-key mappings (- key: val) as a + // mapping, losing entries. Sequence check first is safer. + final YamlSequence seq = yaml.yamlSequence(key); + if (seq != null) { + builder.add(keyStr, yamlSeqToJson(seq)); + } else { + final YamlMapping nested = yaml.yamlMapping(key); + if (nested != null) { + builder.add(keyStr, yamlToJson(nested)); + } + } + } + } + return builder.build(); + } + + /** + * Convert YAML sequence to JsonArray, handling nested mappings, scalars, + * and nested sequences. + * @param seq YAML sequence to convert + * @return JsonArray representation + */ + private static javax.json.JsonArray yamlSeqToJson(final YamlSequence seq) { + final JsonArrayBuilder arr = Json.createArrayBuilder(); + for (final YamlNode node : seq) { + if (node.type() == Node.SCALAR) { + arr.add(resolveEnvVars(node.asScalar().value())); + } else if (node.type() == Node.MAPPING) { + arr.add(yamlToJson(node.asMapping())); + } else if (node.type() == Node.SEQUENCE) { + arr.add(yamlSeqToJson(node.asSequence())); + } + } + return arr.build(); + } + + /** + * Resolve ${VAR_NAME} placeholders with actual environment variable values. + * @param value String that may contain env var placeholders + * @return Resolved string + */ + private static String resolveEnvVars(final String value) { + if (value == null || !value.contains("${")) { + return value; + } + String result = value; + final java.util.regex.Matcher matcher = + java.util.regex.Pattern.compile("\\$\\{([^}]+)}").matcher(value); + while (matcher.find()) { + final String envName = matcher.group(1); + final String envVal = System.getenv(envName); + if (envVal != null) { + result = result.replace("${" + envName + "}", envVal); + } + } + return result; + } +} diff --git a/artipie-main/src/main/java/com/artipie/diagnostics/BlockedThreadDiagnostics.java b/artipie-main/src/main/java/com/artipie/diagnostics/BlockedThreadDiagnostics.java index e88ac5347..bad3cb4b6 100644 --- a/artipie-main/src/main/java/com/artipie/diagnostics/BlockedThreadDiagnostics.java +++ b/artipie-main/src/main/java/com/artipie/diagnostics/BlockedThreadDiagnostics.java @@ -103,12 +103,11 @@ public static synchronized BlockedThreadDiagnostics initialize() { instance = new BlockedThreadDiagnostics(); instance.start(); EcsLogger.info("com.artipie.diagnostics") - .message("Blocked thread diagnostics initialized") + .message(String.format( + "Blocked thread diagnostics initialized: GC check interval 1s, thread check interval 5s, GC pause threshold %dms", + GC_PAUSE_THRESHOLD_MS)) .eventCategory("system") .eventAction("diagnostics_init") - .field("gc.check.interval.sec", 1) - .field("thread.check.interval.sec", 5) - .field("gc.pause.threshold.ms", GC_PAUSE_THRESHOLD_MS) .log(); } return instance; @@ -158,13 +157,11 @@ private void checkGcActivity() { if (gcTimeDelta > GC_PAUSE_THRESHOLD_MS && gcCountDelta > 0) { final long avgPauseMs = gcTimeDelta / gcCountDelta; EcsLogger.warn("com.artipie.diagnostics") - .message("Long GC pause detected - may cause blocked thread warnings") + .message(String.format( + "Long GC pause detected - may cause blocked thread warnings: time delta %dms, %d collections, avg pause %dms, total GC time %dms", + gcTimeDelta, gcCountDelta, avgPauseMs, totalGcTime)) .eventCategory("system") .eventAction("gc_pause") - .field("gc.time.delta.ms", gcTimeDelta) - .field("gc.count.delta", gcCountDelta) - .field("gc.avg.pause.ms", avgPauseMs) - .field("gc.total.time.ms", totalGcTime) .log(); // Also log thread states during long GC @@ -207,12 +204,11 @@ private void logEventLoopThreadStates() { // Only log if there are blocked event loop threads if (blockedCount > 0) { EcsLogger.warn("com.artipie.diagnostics") - .message("Event loop threads in BLOCKED state") + .message(String.format( + "Event loop threads in BLOCKED state: %d blocked, %d waiting, %d runnable", + blockedCount, waitingCount, runnableCount)) .eventCategory("system") .eventAction("thread_state") - .field("eventloop.blocked", blockedCount) - .field("eventloop.waiting", waitingCount) - .field("eventloop.runnable", runnableCount) .log(); this.logAllBlockedThreads(); } @@ -242,13 +238,13 @@ private void logAllBlockedThreads() { } EcsLogger.error("com.artipie.diagnostics") - .message("Blocked event loop thread details") + .message(String.format( + "Blocked event loop thread details: lock=%s, lock owner=%s", + info.getLockName(), info.getLockOwnerName())) .eventCategory("system") .eventAction("blocked_thread") - .field("thread.name", info.getThreadName()) - .field("lock.name", info.getLockName()) - .field("lock.owner", info.getLockOwnerName()) - .field("stack.trace", sb.toString()) + .field("process.thread.name", info.getThreadName()) + .field("error.stack_trace", sb.toString()) .log(); } } @@ -279,7 +275,7 @@ public static VertxOptions getOptimizedVertxOptions(final int cpuCores) { } /** - * Shutdown diagnostics. + * Shutdown diagnostics instance. */ public void shutdown() { this.scheduler.shutdown(); @@ -292,4 +288,15 @@ public void shutdown() { Thread.currentThread().interrupt(); } } + + /** + * Shutdown the singleton diagnostics instance (if initialized). + * Safe to call even if never initialized. + */ + public static synchronized void shutdownInstance() { + if (instance != null) { + instance.shutdown(); + instance = null; + } + } } diff --git a/artipie-main/src/main/java/com/artipie/group/ArtifactNameParser.java b/artipie-main/src/main/java/com/artipie/group/ArtifactNameParser.java new file mode 100644 index 000000000..be40be428 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/group/ArtifactNameParser.java @@ -0,0 +1,342 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Extracts the artifact name from a raw URL path based on the repository type. + * Each adapter stores artifacts with a specific {@code name} format in the DB. + * This parser reverses the URL path back to that format so GroupSlice can do + * an indexed lookup via {@code WHERE name = ?} instead of expensive fan-out. + * + * @since 1.21.0 + */ +public final class ArtifactNameParser { + + /** + * Docker v2 path pattern: /v2/{name}/(manifests|blobs|tags)/... + */ + private static final Pattern DOCKER_PATH = + Pattern.compile("/v2/(.+?)/(manifests|blobs|tags)/.*"); + + /** + * Maven file extensions (artifact files, checksums, signatures, metadata). + */ + private static final Pattern MAVEN_FILE_EXT = Pattern.compile( + ".*\\.(jar|pom|xml|war|aar|ear|module|sha1|sha256|sha512|md5|asc|sig)$" + ); + + private ArtifactNameParser() { + } + + /** + * Parse artifact name from URL path based on repository type. + * + * @param repoType Repository type (e.g., "maven-group", "npm-proxy", "docker-group") + * @param urlPath Raw URL path from HTTP request (may have leading slash) + * @return Parsed artifact name matching the DB {@code name} column, or empty if unparseable + */ + public static Optional parse(final String repoType, final String urlPath) { + if (repoType == null || urlPath == null || urlPath.isEmpty()) { + return Optional.empty(); + } + final String base = normalizeType(repoType); + return switch (base) { + case "maven", "gradle" -> parseMaven(urlPath); + case "npm" -> parseNpm(urlPath); + case "docker" -> parseDocker(urlPath); + case "pypi" -> parsePypi(urlPath); + case "go" -> parseGo(urlPath); + case "gem" -> parseGem(urlPath); + case "php" -> parseComposer(urlPath); + default -> Optional.empty(); + }; + } + + /** + * Strip group/proxy/local suffix: "maven-group" -> "maven", "npm-proxy" -> "npm". + */ + static String normalizeType(final String repoType) { + return repoType.replaceAll("-(group|proxy|local|remote)$", ""); + } + + /** + * Maven URL path to artifact name. + *

+ * Maven URLs follow: {groupId-path}/{artifactId}/{version}/{filename} + * DB name format: groupId.artifactId (slashes replaced with dots) + *

+ * Examples: + *

    + *
  • {@code com/google/guava/guava/31.1/guava-31.1.jar} -> {@code com.google.guava.guava}
    • + *
  • {@code com/google/guava/guava/maven-metadata.xml} -> {@code com.google.guava.guava}
    • + *
  • {@code org/apache/maven/plugins/maven-compiler-plugin/3.11.0/maven-compiler-plugin-3.11.0.pom} + * -> {@code org.apache.maven.plugins.maven-compiler-plugin}
    • + *
+ */ + static Optional parseMaven(final String urlPath) { + final String clean = stripLeadingSlash(urlPath); + final String[] segments = clean.split("/"); + if (segments.length < 2) { + return Optional.empty(); + } + int end = segments.length; + // Strip filename if last segment looks like a file + if (MAVEN_FILE_EXT.matcher(segments[end - 1]).matches()) { + end--; + } + if (end < 1) { + return Optional.empty(); + } + // Strip version directory if it starts with a digit + if (end > 1 && !segments[end - 1].isEmpty() + && Character.isDigit(segments[end - 1].charAt(0))) { + end--; + } + if (end < 1) { + return Optional.empty(); + } + // Join remaining segments with dots + final StringBuilder name = new StringBuilder(); + for (int i = 0; i < end; i++) { + if (i > 0) { + name.append('.'); + } + name.append(segments[i]); + } + final String result = name.toString(); + return result.isEmpty() ? Optional.empty() : Optional.of(result); + } + + /** + * npm URL path to package name. + *

+ * npm URLs follow: + *

    + *
  • {@code /lodash} -> {@code lodash} (metadata)
    • + *
  • {@code /lodash/-/lodash-4.17.21.tgz} -> {@code lodash} (tarball)
    • + *
  • {@code /@babel/core} -> {@code @babel/core} (scoped metadata)
    • + *
  • {@code /@babel/core/-/@babel/core-7.23.0.tgz} -> {@code @babel/core} (scoped tarball)
    • + *
+ */ + static Optional parseNpm(final String urlPath) { + final String clean = stripLeadingSlash(urlPath); + if (clean.isEmpty()) { + return Optional.empty(); + } + // Tarball URLs contain /-/ separator + final int sep = clean.indexOf("/-/"); + if (sep > 0) { + return Optional.of(clean.substring(0, sep)); + } + // Metadata URLs: the path IS the package name + // Scoped: @scope/package (2 segments) + // Unscoped: package (1 segment) + if (clean.startsWith("@")) { + // Scoped package: take first two segments + final String[] parts = clean.split("/", 3); + if (parts.length >= 2) { + return Optional.of(parts[0] + "/" + parts[1]); + } + return Optional.empty(); + } + // Unscoped: take first segment only + final String[] parts = clean.split("/", 2); + return Optional.of(parts[0]); + } + + /** + * Docker URL path to image name. + *

+ * Docker URLs follow: /v2/{name}/(manifests|blobs|tags)/... + * DB name format: the image name as-is (e.g., "library/nginx") + */ + static Optional parseDocker(final String urlPath) { + final Matcher matcher = DOCKER_PATH.matcher(urlPath); + if (matcher.matches()) { + return Optional.of(matcher.group(1)); + } + return Optional.empty(); + } + + /** + * PyPI URL path to package name. + *

+ * PyPI URLs follow: + *

    + *
  • {@code /simple/numpy/} -> {@code numpy}
    • + *
  • {@code /simple/my-package/} -> {@code my-package}
    • + *
  • {@code /packages/numpy-1.24.0.whl} -> {@code numpy}
    • + *
  • {@code /packages/my_package-1.0.0.tar.gz} -> {@code my-package} (normalized)
    • + *
+ * PyPI normalizes names: underscores, dots, and hyphens collapse to hyphens, + * then lowercased. + */ + static Optional parsePypi(final String urlPath) { + final String clean = stripLeadingSlash(urlPath); + // /simple/{name}/ pattern + if (clean.startsWith("simple/")) { + final String rest = clean.substring("simple/".length()); + final String name = rest.endsWith("/") + ? rest.substring(0, rest.length() - 1) : rest.split("/")[0]; + return name.isEmpty() ? Optional.empty() + : Optional.of(normalizePypiName(name)); + } + // /packages/{filename} pattern — extract name from filename + if (clean.startsWith("packages/")) { + final String filename = clean.substring("packages/".length()); + // Remove nested paths if any + final String base = filename.contains("/") + ? filename.substring(filename.lastIndexOf('/') + 1) : filename; + return extractPypiNameFromFilename(base); + } + return Optional.empty(); + } + + /** + * Go module URL path to module name. + *

+ * Go URLs follow: /{module}/@v/{version}.{ext} + * or /{module}/@latest + */ + static Optional parseGo(final String urlPath) { + final String clean = stripLeadingSlash(urlPath); + final int atv = clean.indexOf("/@v/"); + if (atv > 0) { + return Optional.of(clean.substring(0, atv)); + } + final int atl = clean.indexOf("/@latest"); + if (atl > 0) { + return Optional.of(clean.substring(0, atl)); + } + return Optional.empty(); + } + + /** + * RubyGems URL path to gem name. + *

+ * Gem URLs follow: + *

    + *
  • {@code /gems/rails-7.1.2.gem} -> {@code rails}
    • + *
  • {@code /api/v1/dependencies?gems=rails} -> {@code rails}
    • + *
  • {@code /api/v1/gems/rails.json} -> {@code rails}
    • + *
  • {@code /quick/Marshal.4.8/rails-7.1.2.gemspec.rz} -> {@code rails}
    • + *
+ */ + static Optional parseGem(final String urlPath) { + final String clean = stripLeadingSlash(urlPath); + // /gems/{name}-{version}.gem + if (clean.startsWith("gems/")) { + final String filename = clean.substring("gems/".length()); + return extractGemName(filename); + } + // /api/v1/dependencies?gems={name} + if (clean.contains("dependencies")) { + final int qmark = clean.indexOf("gems="); + if (qmark >= 0) { + final String names = clean.substring(qmark + "gems=".length()); + final String first = names.split(",")[0].trim(); + return first.isEmpty() ? Optional.empty() : Optional.of(first); + } + } + // /api/v1/gems/{name}.json + if (clean.startsWith("api/v1/gems/")) { + final String rest = clean.substring("api/v1/gems/".length()); + if (rest.endsWith(".json")) { + return Optional.of(rest.substring(0, rest.length() - ".json".length())); + } + } + // /quick/Marshal.4.8/{name}-{version}.gemspec.rz + if (clean.startsWith("quick/")) { + final int lastSlash = clean.lastIndexOf('/'); + if (lastSlash >= 0) { + return extractGemName(clean.substring(lastSlash + 1)); + } + } + return Optional.empty(); + } + + /** + * Composer/PHP URL path to package name. + *

+ * Composer URLs follow: + *

    + *
  • {@code /p2/vendor/package.json} -> {@code vendor/package}
    • + *
  • {@code /p2/vendor/package$hash.json} -> {@code vendor/package}
    • + *
  • {@code /p/vendor/package.json} -> {@code vendor/package}
    • + *
+ */ + static Optional parseComposer(final String urlPath) { + final String clean = stripLeadingSlash(urlPath); + // /p2/vendor/package.json or /p/vendor/package.json + final Matcher matcher = Pattern.compile( + "p2?/([^/]+)/([^/$]+)(?:\\$[a-f0-9]+)?\\.json$" + ).matcher(clean); + if (matcher.find()) { + return Optional.of(matcher.group(1) + "/" + matcher.group(2)); + } + return Optional.empty(); + } + + /** + * Extract gem name from a filename like "rails-7.1.2.gem" or "rails-7.1.2.gemspec.rz". + * Name is everything before the last hyphen-followed-by-digit. + */ + private static Optional extractGemName(final String filename) { + // Remove extensions + String base = filename; + if (base.endsWith(".gem")) { + base = base.substring(0, base.length() - ".gem".length()); + } else if (base.endsWith(".gemspec.rz")) { + base = base.substring(0, base.length() - ".gemspec.rz".length()); + } else { + return Optional.empty(); + } + // Name is everything before the LAST "-{digit}" pattern + final Matcher m = Pattern.compile("^(.+)-\\d").matcher(base); + if (m.find()) { + return Optional.of(m.group(1)); + } + return Optional.of(base); + } + + /** + * Normalize a PyPI project name: replace [-_.] runs with single hyphen, lowercase. + */ + private static String normalizePypiName(final String name) { + return name.replaceAll("[-_.]+", "-").toLowerCase(); + } + + /** + * Extract PyPI package name from a distribution filename. + * Wheel: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl + * Sdist: {name}-{version}.tar.gz or {name}-{version}.zip + */ + private static Optional extractPypiNameFromFilename(final String filename) { + // Remove extension + String base = filename; + if (base.endsWith(".tar.gz")) { + base = base.substring(0, base.length() - ".tar.gz".length()); + } else if (base.endsWith(".whl") || base.endsWith(".zip") || base.endsWith(".egg")) { + base = base.substring(0, base.lastIndexOf('.')); + } else { + return Optional.empty(); + } + // Name is everything before the first hyphen followed by a digit + // e.g., "numpy-1.24.0" -> "numpy", "my_package-2.0.0rc1" -> "my_package" + final Matcher m = Pattern.compile("^(.+?)-\\d").matcher(base); + if (m.find()) { + return Optional.of(normalizePypiName(m.group(1))); + } + return Optional.empty(); + } + + private static String stripLeadingSlash(final String path) { + return path.startsWith("/") ? path.substring(1) : path; + } +} diff --git a/artipie-main/src/main/java/com/artipie/group/GroupMetadataCache.java b/artipie-main/src/main/java/com/artipie/group/GroupMetadataCache.java index c5f9268a8..50479580e 100644 --- a/artipie-main/src/main/java/com/artipie/group/GroupMetadataCache.java +++ b/artipie-main/src/main/java/com/artipie/group/GroupMetadataCache.java @@ -15,6 +15,8 @@ import java.time.Instant; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; /** @@ -67,6 +69,13 @@ public final class GroupMetadataCache { */ private final String groupName; + /** + * Last-known-good metadata (never expires). + * Populated on every successful put(), survives L1/L2 invalidation/expiry. + * Used as stale fallback when upstream is unreachable. + */ + private final ConcurrentMap lastKnownGood; + /** * Create group metadata cache with defaults. * @param groupName Group repository name @@ -108,6 +117,7 @@ public GroupMetadataCache( .expireAfterWrite(l1Ttl.toMillis(), TimeUnit.MILLISECONDS) .recordStats() .build(); + this.lastKnownGood = new ConcurrentHashMap<>(); } /** @@ -155,12 +165,31 @@ public CompletableFuture> get(final String path) { }); } + /** + * Get stale (last-known-good) metadata. This data never expires and is + * populated on every successful {@link #put}. Use as fallback when all + * group members are unreachable and the primary cache has expired. + * @param path Metadata path + * @return Optional containing last-known-good bytes, or empty if never cached + */ + public CompletableFuture> getStale(final String path) { + final byte[] data = this.lastKnownGood.get(path); + if (data != null) { + recordCacheHit("lkg"); + return CompletableFuture.completedFuture(Optional.of(data)); + } + recordCacheMiss("lkg"); + return CompletableFuture.completedFuture(Optional.empty()); + } + /** * Put metadata in cache (both L1 and L2). * @param path Metadata path * @param data Metadata bytes */ public void put(final String path, final byte[] data) { + // Always update last-known-good (never expires) + this.lastKnownGood.put(path, data); // Put in L1 final CachedMetadata entry = new CachedMetadata(data, Instant.now()); this.l1Cache.put(path, entry); diff --git a/artipie-main/src/main/java/com/artipie/group/GroupNegativeCache.java b/artipie-main/src/main/java/com/artipie/group/GroupNegativeCache.java deleted file mode 100644 index a2c5e430d..000000000 --- a/artipie-main/src/main/java/com/artipie/group/GroupNegativeCache.java +++ /dev/null @@ -1,652 +0,0 @@ -/* - * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com - * https://github.com/artipie/artipie/blob/master/LICENSE.txt - */ -package com.artipie.group; - -import com.artipie.asto.Key; -import com.artipie.cache.GlobalCacheConfig; -import com.artipie.cache.NegativeCacheConfig; -import com.artipie.cache.ValkeyConnection; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import io.lettuce.core.api.async.RedisAsyncCommands; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Negative cache for group repositories. - * Caches 404 responses per member to avoid repeated queries for missing artifacts. - * - *

Key format: {@code negative:group:{group_name}:{member_name}:{path}}

- * - *

Two-tier architecture:

- *
    - *
  • L1 (Caffeine): Fast in-memory cache, short TTL
    • - *
  • L2 (Valkey/Redis): Distributed cache, full TTL
    • - *
- * - *

Configuration is read from unified {@link NegativeCacheConfig}.

- * - * @since 1.0 - */ -public final class GroupNegativeCache { - - /** - * Global registry of all GroupNegativeCache instances by group name. - * Allows L1 cache invalidation without process restart. - */ - private static final ConcurrentHashMap< - String, - GroupNegativeCache - > INSTANCES = new ConcurrentHashMap<>(); - - /** - * Sentinel value for cache entries. - */ - private static final Boolean CACHED = Boolean.TRUE; - - /** - * Dedicated executor for Redis operations to prevent event loop blocking. - * Lettuce's SharedLock can block on lock acquisition, so we offload - * all Redis operations to this executor pool. - * - * Pool size: 4 threads is sufficient for async Redis operations - * since actual I/O is still non-blocking in Lettuce. - */ - private static final ExecutorService REDIS_EXECUTOR = - Executors.newFixedThreadPool( - 4, - new ThreadFactory() { - private final AtomicInteger counter = new AtomicInteger(0); - - @Override - public Thread newThread(final Runnable r) { - final Thread thread = new Thread(r); - thread.setName( - "artipie-redis-cache-" + counter.getAndIncrement() - ); - thread.setDaemon(true); - return thread; - } - } - ); - - /** - * L1 cache (in-memory). - */ - private final Cache l1Cache; - - /** - * L2 cache (Valkey/Redis), may be null. - */ - private final RedisAsyncCommands l2; - - /** - * Whether two-tier caching is enabled. - */ - private final boolean twoTier; - - /** - * Cache TTL for L2. - */ - private final Duration l2Ttl; - - /** - * Timeout for L2 operations in milliseconds. - */ - private final long l2TimeoutMs; - - /** - * Group repository name. - */ - private final String groupName; - - /** - * Whether negative caching is enabled. - */ - private final boolean enabled; - - /** - * Create group negative cache using unified NegativeCacheConfig. - * @param groupName Group repository name - */ - public GroupNegativeCache(final String groupName) { - this(groupName, NegativeCacheConfig.getInstance()); - } - - /** - * Create group negative cache with explicit config. - * @param groupName Group repository name - * @param config Negative cache configuration - */ - public GroupNegativeCache( - final String groupName, - final NegativeCacheConfig config - ) { - this.groupName = groupName; - this.enabled = true; - - // Check global valkey connection - final ValkeyConnection actualValkey = - GlobalCacheConfig.valkeyConnection().orElse(null); - this.twoTier = config.isValkeyEnabled() && (actualValkey != null); - this.l2 = this.twoTier ? actualValkey.async() : null; - this.l2Ttl = config.l2Ttl(); - this.l2TimeoutMs = config.l2Timeout().toMillis(); - - // L1 cache configuration from unified config - final Duration l1Ttl = this.twoTier ? config.l1Ttl() : config.ttl(); - final int l1Size = this.twoTier ? config.l1MaxSize() : config.maxSize(); - - this.l1Cache = Caffeine.newBuilder() - .maximumSize(l1Size) - .expireAfterWrite(l1Ttl.toMillis(), TimeUnit.MILLISECONDS) - .recordStats() - .build(); - - // Register this instance for global invalidation - INSTANCES.put(groupName, this); - } - - /** - * Build cache key. - * Format: negative:group:{group_name}:{member_name}:{path} - */ - private String buildKey(final String memberName, final Key path) { - return ( - "negative:group:" + - this.groupName + - ":" + - memberName + - ":" + - path.string() - ); - } - - /** - * Check if member returned 404 for this path (L1 only for fast path). - * @param memberName Member repository name - * @param path Request path - * @return True if cached as not found in L1 - */ - public boolean isNotFoundL1(final String memberName, final Key path) { - if (!this.enabled) { - return false; - } - final String key = buildKey(memberName, path); - final boolean found = this.l1Cache.getIfPresent(key) != null; - - // Record metrics - if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { - if (found) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheHit( - "group_negative", - "l1" - ); - } else { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheMiss( - "group_negative", - "l1" - ); - } - } - return found; - } - - /** - * Check if member returned 404 - checks L1 first, then L2 if L1 miss. - * @param memberName Member repository name - * @param path Request path - * @return Future with true if cached as not found in either L1 or L2 - */ - public CompletableFuture isNotFoundAsync( - final String memberName, - final Key path - ) { - if (!this.enabled) { - return CompletableFuture.completedFuture(false); - } - - final String key = buildKey(memberName, path); - - // Check L1 first (synchronous, in-memory - safe on event loop) - final boolean foundL1 = this.l1Cache.getIfPresent(key) != null; - if (foundL1) { - if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheHit( - "group_negative", - "l1" - ); - } - return CompletableFuture.completedFuture(true); - } - - if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheMiss( - "group_negative", - "l1" - ); - } - - // L1 MISS - check L2 if available - if (!this.twoTier) { - return CompletableFuture.completedFuture(false); - } - - // CRITICAL FIX: Execute Redis operations on dedicated executor - // to prevent Lettuce's SharedLock from blocking Vert.x event loop. - // The lock acquisition in DefaultEndpoint.write() can block indefinitely - // if Redis is slow or connection pool is contended. - return CompletableFuture.supplyAsync( - () -> { - try { - final byte[] bytes = this.l2.get(key) - .toCompletableFuture() - .get(this.l2TimeoutMs, TimeUnit.MILLISECONDS); - - if (bytes != null) { - // L2 HIT - promote to L1 - this.l1Cache.put(key, CACHED); - if ( - com.artipie.metrics.MicrometerMetrics.isInitialized() - ) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheHit( - "group_negative", - "l2" - ); - } - return true; - } - if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheMiss( - "group_negative", - "l2" - ); - } - return false; - } catch (Exception e) { - // Timeout or Redis error - treat as cache miss, don't block - if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheError( - "group_negative", - "l2", - e.getClass().getSimpleName() - ); - } - return false; - } - }, - REDIS_EXECUTOR - ); - } - - /** - * Check L2 cache asynchronously (call after L1 miss if needed). - * @param memberName Member repository name - * @param path Request path - * @return Future with true if cached as not found in L2 - */ - public CompletableFuture isNotFoundL2Async( - final String memberName, - final Key path - ) { - if (!this.enabled || !this.twoTier) { - return CompletableFuture.completedFuture(false); - } - - final String key = buildKey(memberName, path); - - // Execute on dedicated executor to prevent event loop blocking - return CompletableFuture.supplyAsync( - () -> { - try { - final byte[] bytes = this.l2.get(key) - .toCompletableFuture() - .get(this.l2TimeoutMs, TimeUnit.MILLISECONDS); - - if (bytes != null) { - // L2 HIT - promote to L1 - this.l1Cache.put(key, CACHED); - if ( - com.artipie.metrics.MicrometerMetrics.isInitialized() - ) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheHit( - "group_negative", - "l2" - ); - } - return true; - } - if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheMiss( - "group_negative", - "l2" - ); - } - return false; - } catch (Exception e) { - if (com.artipie.metrics.MicrometerMetrics.isInitialized()) { - com.artipie.metrics.MicrometerMetrics.getInstance().recordCacheError( - "group_negative", - "l2", - e.getClass().getSimpleName() - ); - } - return false; - } - }, - REDIS_EXECUTOR - ); - } - - /** - * Cache member as returning 404 for this path. - * @param memberName Member repository name - * @param path Request path - */ - public void cacheNotFound(final String memberName, final Key path) { - if (!this.enabled) { - return; - } - - final String key = buildKey(memberName, path); - - // Cache in L1 - this.l1Cache.put(key, CACHED); - - // Cache in L2 (if enabled) - if (this.twoTier) { - final byte[] value = new byte[] { 1 }; // Sentinel value - this.l2.setex(key, this.l2Ttl.getSeconds(), value); - } - } - - /** - * Invalidate cached 404 for a member/path (e.g., when artifact is deployed). - * @param memberName Member repository name - * @param path Request path - */ - public void invalidate(final String memberName, final Key path) { - final String key = buildKey(memberName, path); - this.l1Cache.invalidate(key); - if (this.twoTier) { - this.l2.del(key); - } - } - - /** - * Invalidate all cached 404s for a member. - * @param memberName Member repository name - */ - public void invalidateMember(final String memberName) { - final String prefix = - "negative:group:" + this.groupName + ":" + memberName + ":"; - - // L1: Remove matching entries - this.l1Cache.asMap() - .keySet() - .removeIf(k -> k.startsWith(prefix)); - - // L2: Scan and delete - if (this.twoTier) { - this.l2.keys(prefix + "*").thenAccept(keys -> { - if (keys != null && !keys.isEmpty()) { - this.l2.del(keys.toArray(new String[0])); - } - }); - } - } - - /** - * Clear entire cache for this group. - */ - public void clear() { - final String prefix = "negative:group:" + this.groupName + ":"; - - this.l1Cache.asMap() - .keySet() - .removeIf(k -> k.startsWith(prefix)); - - if (this.twoTier) { - this.l2.keys(prefix + "*").thenAccept(keys -> { - if (keys != null && !keys.isEmpty()) { - this.l2.del(keys.toArray(new String[0])); - } - }); - } - } - - /** - * Get L1 cache size. - * @return Estimated number of entries - */ - public long size() { - return this.l1Cache.estimatedSize(); - } - - /** - * Check if two-tier caching is enabled. - * @return True if L2 (Valkey) is configured - */ - public boolean isTwoTier() { - return this.twoTier; - } - - // ========== Static methods for global cache invalidation ========== - - /** - * Invalidate negative cache entries for a package across ALL group instances. - * This invalidates both L1 (in-memory) and L2 (Valkey) caches. - * - *

Use this when a package is published to ensure group repos can find it.

- * - * @param packagePath Package path (e.g., "@retail/backoffice-interaction-notes") - * @return CompletableFuture that completes when L2 invalidation is done - */ - public static CompletableFuture invalidatePackageGlobally( - final String packagePath - ) { - final List> futures = new ArrayList<>(); - - for (final GroupNegativeCache instance : INSTANCES.values()) { - futures.add(instance.invalidatePackageInAllMembers(packagePath)); - } - futures.add(invalidateGlobalL2Only(packagePath)); - return CompletableFuture.allOf( - futures.toArray(new CompletableFuture[0]) - ); - } - - /** - * Invalidate negative cache entries for a package in a specific group. - * This invalidates both L1 (in-memory) and L2 (Valkey) caches. - * - * @param groupName Group repository name - * @param packagePath Package path (e.g., "@retail/backoffice-interaction-notes") - * @return CompletableFuture that completes when invalidation is done - */ - public static CompletableFuture invalidatePackageInGroup( - final String groupName, - final String packagePath - ) { - final GroupNegativeCache instance = INSTANCES.get(groupName); - if (instance != null) { - return instance.invalidatePackageInAllMembers(packagePath); - } - // Group not found - try L2 directly if available - return invalidateL2Only(groupName, packagePath); - } - - /** - * Clear all negative cache entries for a specific group. - * This clears both L1 (in-memory) and L2 (Valkey) caches. - * - * @param groupName Group repository name - * @return CompletableFuture that completes when clearing is done - */ - public static CompletableFuture clearGroup(final String groupName) { - final GroupNegativeCache instance = INSTANCES.get(groupName); - if (instance != null) { - instance.clear(); - return CompletableFuture.completedFuture(null); - } - // Group not found - try L2 directly if available - return clearL2Only(groupName); - } - - /** - * Get list of all registered group names. - * @return List of group names with active negative caches - */ - public static List registeredGroups() { - return new ArrayList<>(INSTANCES.keySet()); - } - - /** - * Get a specific group's cache instance (for diagnostics). - * @param groupName Group repository name - * @return Optional cache instance - */ - public static java.util.Optional getInstance( - final String groupName - ) { - return java.util.Optional.ofNullable(INSTANCES.get(groupName)); - } - - /** - * Invalidate package entries in all members of this group. - * @param packagePath Package path - * @return CompletableFuture that completes when done - */ - private CompletableFuture invalidatePackageInAllMembers( - final String packagePath - ) { - final String prefix = "negative:group:" + this.groupName + ":"; - final String suffix = ":" + packagePath; - - // Invalidate L1: remove all entries for this package across all members - this.l1Cache.asMap() - .keySet() - .removeIf(k -> k.startsWith(prefix) && k.endsWith(suffix)); - - // Invalidate L2: scan and delete matching keys - if (this.twoTier) { - final String pattern = prefix + "*" + suffix; - return this.l2.keys(pattern) - .thenCompose(keys -> { - if (keys != null && !keys.isEmpty()) { - return this.l2.del( - keys.toArray(new String[0]) - ).thenApply(count -> null); - } - return CompletableFuture.completedFuture(null); - }) - .toCompletableFuture() - .thenApply(v -> null); - } - - return CompletableFuture.completedFuture(null); - } - - /** - * Invalidate L2 cache only (when no L1 instance exists). - * @param groupName Group name - * @param packagePath Package path - * @return CompletableFuture that completes when done - */ - private static CompletableFuture invalidateL2Only( - final String groupName, - final String packagePath - ) { - final ValkeyConnection valkey = - GlobalCacheConfig.valkeyConnection().orElse(null); - if (valkey == null) { - return CompletableFuture.completedFuture(null); - } - - final String pattern = - "negative:group:" + groupName + ":*:" + packagePath; - return valkey - .async() - .keys(pattern) - .thenCompose(keys -> { - if (keys != null && !keys.isEmpty()) { - return valkey - .async() - .del(keys.toArray(new String[0])) - .thenApply(count -> null); - } - return CompletableFuture.completedFuture(null); - }) - .toCompletableFuture() - .thenApply(v -> null); - } - - /** - * Clear L2 cache only for a group (when no L1 instance exists). - * @param groupName Group name - * @return CompletableFuture that completes when done - */ - private static CompletableFuture clearL2Only(final String groupName) { - final ValkeyConnection valkey = - GlobalCacheConfig.valkeyConnection().orElse(null); - if (valkey == null) { - return CompletableFuture.completedFuture(null); - } - - final String pattern = "negative:group:" + groupName + ":*"; - return valkey - .async() - .keys(pattern) - .thenCompose(keys -> { - if (keys != null && !keys.isEmpty()) { - return valkey - .async() - .del(keys.toArray(new String[0])) - .thenApply(count -> null); - } - return CompletableFuture.completedFuture(null); - }) - .toCompletableFuture() - .thenApply(v -> null); - } - - /** - * Invalidate L2 cache globally for a package, even if no in-memory instances exist. - * @param packagePath Package path - * @return CompletableFuture that completes when done - */ - private static CompletableFuture invalidateGlobalL2Only( - final String packagePath - ) { - final ValkeyConnection valkey = - GlobalCacheConfig.valkeyConnection().orElse(null); - if (valkey == null) { - return CompletableFuture.completedFuture(null); - } - final String pattern = "negative:group:*:*:" + packagePath; - return valkey - .async() - .keys(pattern) - .thenCompose(keys -> { - if (keys != null && !keys.isEmpty()) { - return valkey - .async() - .del(keys.toArray(new String[0])) - .thenApply(count -> null); - } - return CompletableFuture.completedFuture(null); - }) - .toCompletableFuture() - .thenApply(v -> null); - } -} diff --git a/artipie-main/src/main/java/com/artipie/group/GroupSlice.java b/artipie-main/src/main/java/com/artipie/group/GroupSlice.java index e444d661d..14a6453f0 100644 --- a/artipie-main/src/main/java/com/artipie/group/GroupSlice.java +++ b/artipie-main/src/main/java/com/artipie/group/GroupSlice.java @@ -15,16 +15,22 @@ import com.artipie.http.log.EcsLogEvent; import com.artipie.http.log.EcsLogger; import com.artipie.http.slice.KeyFromPath; +import com.artipie.http.misc.ConfigDefaults; +import com.artipie.index.ArtifactIndex; -import java.time.Duration; import java.util.ArrayList; +import java.util.concurrent.Semaphore; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.slf4j.MDC; @@ -48,9 +54,24 @@ public final class GroupSlice implements Slice { /** - * Default timeout for member requests in seconds. + * Resolved drain permits from env/system property/default. */ - private static final long DEFAULT_TIMEOUT_SECONDS = 120; + private static final int DRAIN_LIMIT = + ConfigDefaults.getInt("ARTIPIE_GROUP_DRAIN_PERMITS", 20); + + /** + * Semaphore limiting concurrent response body drains to prevent memory pressure. + * At 30MB per npm metadata response, 20 concurrent drains = 600MB max. + */ + private static final Semaphore DRAIN_PERMITS = new Semaphore(DRAIN_LIMIT); + + static { + EcsLogger.info("com.artipie.group") + .message("GroupSlice drain permits configured: " + DRAIN_LIMIT) + .eventCategory("configuration") + .eventAction("group_init") + .log(); + } /** * Group repository name. @@ -63,14 +84,27 @@ public final class GroupSlice implements Slice { private final List members; /** - * Timeout for member requests. + * Routing rules for directing paths to specific members. + */ + private final List routingRules; + + /** + * Optional artifact index for O(1) group lookups. + */ + private final Optional artifactIndex; + + /** + * Repository type for adapter-aware name parsing (e.g., "maven-group", "npm-group"). + * Used by {@link ArtifactNameParser} to extract artifact name from URL path. */ - private final Duration timeout; + private final String repoType; /** - * Negative cache for member 404s. + * Names of members that are proxy repositories. + * Proxy members must always be queried on index miss because their + * content is only indexed after being cached. */ - private final GroupNegativeCache negativeCache; + private final Set proxyMembers; /** * Request context for enhanced logging (client IP, username, trace ID, package). @@ -131,7 +165,8 @@ public GroupSlice( final List members, final int port ) { - this(resolver, group, members, port, 0, DEFAULT_TIMEOUT_SECONDS); + this(resolver, group, members, port, 0, 0, + Collections.emptyList(), Optional.empty(), Collections.emptySet()); } /** @@ -150,11 +185,64 @@ public GroupSlice( final int port, final int depth ) { - this(resolver, group, members, port, depth, DEFAULT_TIMEOUT_SECONDS); + this(resolver, group, members, port, depth, 0, + Collections.emptyList(), Optional.empty(), Collections.emptySet(), ""); } /** * Constructor with depth and timeout. + */ + public GroupSlice( + final SliceResolver resolver, + final String group, + final List members, + final int port, + final int depth, + final long timeoutSeconds + ) { + this(resolver, group, members, port, depth, timeoutSeconds, + Collections.emptyList(), Optional.empty(), Collections.emptySet(), ""); + } + + /** + * Constructor with depth, timeout, routing rules, and artifact index (backward compatible). + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + public GroupSlice( + final SliceResolver resolver, + final String group, + final List members, + final int port, + final int depth, + final long timeoutSeconds, + final List routingRules, + final Optional artifactIndex + ) { + this(resolver, group, members, port, depth, timeoutSeconds, + routingRules, artifactIndex, Collections.emptySet(), ""); + } + + /** + * Backward-compatible constructor without repoType. + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + public GroupSlice( + final SliceResolver resolver, + final String group, + final List members, + final int port, + final int depth, + final long timeoutSeconds, + final List routingRules, + final Optional artifactIndex, + final Set proxyMembers + ) { + this(resolver, group, members, port, depth, timeoutSeconds, + routingRules, artifactIndex, proxyMembers, ""); + } + + /** + * Full constructor with proxy member awareness and repo type. * * @param resolver Slice resolver/cache * @param group Group repository name @@ -162,32 +250,44 @@ public GroupSlice( * @param port Server port * @param depth Nesting depth (ignored) * @param timeoutSeconds Timeout for member requests + * @param routingRules Routing rules for path-based member selection + * @param artifactIndex Optional artifact index for O(1) lookups + * @param proxyMembers Names of members that are proxy repositories + * @param repoType Repository type for name parsing (e.g., "maven-group") */ + @SuppressWarnings("PMD.ExcessiveParameterList") public GroupSlice( final SliceResolver resolver, final String group, final List members, final int port, final int depth, - final long timeoutSeconds + final long timeoutSeconds, + final List routingRules, + final Optional artifactIndex, + final Set proxyMembers, + final String repoType ) { this.group = Objects.requireNonNull(group, "group"); - this.timeout = Duration.ofSeconds(timeoutSeconds); - this.negativeCache = new GroupNegativeCache(group); + this.repoType = repoType != null ? repoType : ""; + this.routingRules = routingRules != null ? routingRules : Collections.emptyList(); + this.artifactIndex = artifactIndex != null ? artifactIndex : Optional.empty(); + this.proxyMembers = proxyMembers != null ? proxyMembers : Collections.emptySet(); // Deduplicate members (simple flattening for now) final List flatMembers = new ArrayList<>(new LinkedHashSet<>(members)); - // Create MemberSlice wrappers with circuit breakers + // Create MemberSlice wrappers with circuit breakers and proxy flags this.members = flatMembers.stream() .map(name -> new MemberSlice( name, - resolver.slice(new Key.From(name), port, 0) + resolver.slice(new Key.From(name), port, 0), + this.proxyMembers.contains(name) )) .toList(); EcsLogger.debug("com.artipie.group") - .message("GroupSlice initialized with members (" + this.members.size() + " unique, " + members.size() + " total)") + .message("GroupSlice initialized with members (" + this.members.size() + " unique, " + members.size() + " total, " + this.proxyMembers.size() + " proxies)") .eventCategory("repository") .eventAction("group_init") .field("repository.name", group) @@ -225,6 +325,66 @@ public CompletableFuture response( recordRequestStart(); final long requestStartTime = System.currentTimeMillis(); + // Index-first: try O(1) lookup before parallel fan-out + if (this.artifactIndex.isPresent()) { + final ArtifactIndex idx = this.artifactIndex.get(); + // Try adapter-aware name parsing first (indexed, fast) + final Optional parsedName = + ArtifactNameParser.parse(this.repoType, path); + final CompletableFuture> locateFuture; + if (parsedName.isPresent()) { + locateFuture = idx.locateByName(parsedName.get()); + } else { + // Fallback to path_prefix matching for unknown adapter types + final String locatePath = path.startsWith("/") + ? path.substring(1) : path; + locateFuture = idx.locate(locatePath); + } + return locateFuture + .thenCompose(repos -> { + if (!repos.isEmpty()) { + // Filter to members of this group + final Set indexHits = Set.copyOf(repos); + final List targeted = this.members.stream() + .filter(m -> indexHits.contains(m.name())) + .toList(); + if (!targeted.isEmpty()) { + EcsLogger.debug("com.artipie.group") + .message("Index hit via " + + (parsedName.isPresent() ? "name" : "path_prefix") + + ": targeting " + targeted.size() + " member(s)") + .eventCategory("repository") + .eventAction("group_index_hit") + .field("repository.name", this.group) + .field("url.path", path) + .log(); + return queryTargetedMembers(targeted, line, headers, body, ctx); + } + } + // Index miss: fall back to querying all members + EcsLogger.debug("com.artipie.group") + .message("Index miss: falling back to all members" + + (parsedName.isPresent() + ? " (parsed name: " + parsedName.get() + ")" + : " (name parse failed)")) + .eventCategory("repository") + .eventAction("group_index_miss") + .field("repository.name", this.group) + .field("url.path", path) + .log(); + return queryAllMembersInParallel(line, headers, body, ctx); + }) + .whenComplete((resp, err) -> { + final long duration = System.currentTimeMillis() - requestStartTime; + if (err != null) { + recordGroupRequest("error", duration); + } else if (resp.status().success()) { + recordGroupRequest("success", duration); + } else { + recordGroupRequest("not_found", duration); + } + }); + } return queryAllMembersInParallel(line, headers, body, ctx) .whenComplete((resp, err) -> { final long duration = System.currentTimeMillis() - requestStartTime; @@ -255,27 +415,134 @@ private CompletableFuture queryAllMembersInParallel( final Key pathKey = new KeyFromPath(line.uri().getPath()); return body.asBytesFuture().thenCompose(requestBytes -> { + // Apply routing rules to filter members for this path + final List eligibleMembers = this.filterByRoutingRules( + line.uri().getPath() + ); final CompletableFuture result = new CompletableFuture<>(); final AtomicBoolean completed = new AtomicBoolean(false); - final AtomicInteger pending = new AtomicInteger(this.members.size()); - - // Start ALL members in parallel - for (MemberSlice member : this.members) { - queryMember(member, line, headers, requestBytes, ctx) - .orTimeout(this.timeout.getSeconds(), java.util.concurrent.TimeUnit.SECONDS) - .whenComplete((resp, err) -> { - if (err != null) { - handleMemberFailure(member, err, completed, pending, result, ctx); - } else { - handleMemberResponse(member, resp, completed, pending, result, startTime, pathKey, ctx); - } - }); + final AtomicInteger pending = new AtomicInteger(eligibleMembers.size()); + final AtomicBoolean anyServerError = new AtomicBoolean(false); + // Track all member futures for best-effort cancellation on first success + final List> memberFutures = + new ArrayList<>(eligibleMembers.size()); + + if (eligibleMembers.isEmpty()) { + result.complete(ResponseBuilder.notFound().build()); + return result; + } + + // Start eligible members in parallel + for (MemberSlice member : eligibleMembers) { + final CompletableFuture memberFuture = + queryMember(member, line, headers, requestBytes, ctx); + memberFutures.add(memberFuture); + memberFuture.whenComplete((resp, err) -> { + if (err != null) { + handleMemberFailure(member, err, completed, pending, anyServerError, result, ctx); + } else { + handleMemberResponse(member, resp, completed, pending, anyServerError, result, startTime, pathKey, ctx); + } + }); + } + + // When first success completes the result, cancel remaining member requests + result.whenComplete((resp, err) -> { + for (CompletableFuture future : memberFutures) { + if (!future.isDone()) { + future.cancel(true); + } + } + }); + + return result; + }); + } + + /** + * Query only targeted members (from index hits) in parallel. + */ + private CompletableFuture queryTargetedMembers( + final List targeted, + final RequestLine line, + final Headers headers, + final Content body, + final RequestContext ctx + ) { + final long startTime = System.currentTimeMillis(); + final Key pathKey = new KeyFromPath(line.uri().getPath()); + + return body.asBytesFuture().thenCompose(requestBytes -> { + final CompletableFuture result = new CompletableFuture<>(); + final AtomicBoolean completed = new AtomicBoolean(false); + final AtomicInteger pending = new AtomicInteger(targeted.size()); + final AtomicBoolean anyServerError = new AtomicBoolean(false); + final List> memberFutures = + new ArrayList<>(targeted.size()); + + for (MemberSlice member : targeted) { + final CompletableFuture memberFuture = + queryMemberDirect(member, line, headers, requestBytes, ctx); + memberFutures.add(memberFuture); + memberFuture.whenComplete((resp, err) -> { + if (err != null) { + handleMemberFailure(member, err, completed, pending, anyServerError, result, ctx); + } else { + handleMemberResponse(member, resp, completed, pending, anyServerError, result, startTime, pathKey, ctx); + } + }); } + result.whenComplete((resp, err) -> { + for (CompletableFuture future : memberFutures) { + if (!future.isDone()) { + future.cancel(true); + } + } + }); + return result; }); } + /** + * Query a single member directly (no negative cache check). + * Used for index-targeted queries where we already know the member has the artifact. + */ + private CompletableFuture queryMemberDirect( + final MemberSlice member, + final RequestLine line, + final Headers headers, + final byte[] requestBytes, + final RequestContext ctx + ) { + if (member.isCircuitOpen()) { + ctx.addTo(EcsLogger.warn("com.artipie.group") + .message("Member circuit OPEN, skipping") + .eventCategory("repository") + .eventAction("group_query") + .eventOutcome("failure") + .field("repository.name", this.group) + .field("member.name", member.name())) + .log(); + return CompletableFuture.completedFuture( + ResponseBuilder.unavailable().build() + ); + } + + final Content memberBody = requestBytes.length > 0 + ? new Content.From(requestBytes) + : Content.EMPTY; + + final RequestLine rewritten = member.rewritePath(line); + + return member.slice().response( + rewritten, + dropFullPathHeader(headers), + memberBody + ); + } + /** * Query a single member. * @@ -293,65 +560,41 @@ private CompletableFuture queryMember( final byte[] requestBytes, final RequestContext ctx ) { - final Key pathKey = new KeyFromPath(line.uri().getPath()); - - // Check negative cache FIRST (L1 then L2 if miss) - return this.negativeCache.isNotFoundAsync(member.name(), pathKey) - .thenCompose(isNotFound -> { - if (isNotFound) { - ctx.addTo(EcsLogger.debug("com.artipie.group") - .message("Member negative cache HIT") - .eventCategory("repository") - .eventAction("group_query") - .eventOutcome("cache_hit") - .field("repository.name", this.group) - .field("member.name", member.name()) - .field("url.path", pathKey.string())) - .log(); - return CompletableFuture.completedFuture( - ResponseBuilder.notFound().build() - ); - } - - if (member.isCircuitOpen()) { - ctx.addTo(EcsLogger.warn("com.artipie.group") - .message("Member circuit OPEN, skipping") - .eventCategory("repository") - .eventAction("group_query") - .eventOutcome("failure") - .field("repository.name", this.group) - .field("member.name", member.name())) - .log(); - return CompletableFuture.completedFuture( - ResponseBuilder.unavailable().build() - ); - } + if (member.isCircuitOpen()) { + ctx.addTo(EcsLogger.warn("com.artipie.group") + .message("Member circuit OPEN, skipping") + .eventCategory("repository") + .eventAction("group_query") + .eventOutcome("failure") + .field("repository.name", this.group) + .field("member.name", member.name())) + .log(); + return CompletableFuture.completedFuture( + ResponseBuilder.unavailable().build() + ); + } - // Create new Content instance from buffered bytes for each member - // This allows parallel requests with POST body (e.g., npm audit) - final Content memberBody = requestBytes.length > 0 - ? new Content.From(requestBytes) - : Content.EMPTY; + // Create new Content instance from buffered bytes for each member + final Content memberBody = requestBytes.length > 0 + ? new Content.From(requestBytes) + : Content.EMPTY; - final RequestLine rewritten = member.rewritePath(line); + final RequestLine rewritten = member.rewritePath(line); - // Log the path rewriting for troubleshooting - EcsLogger.info("com.artipie.group") - .message("Forwarding request to member") - .eventCategory("repository") - .eventAction("group_forward") - .field("repository.name", this.group) - .field("member.name", member.name()) - .field("original.path", line.uri().getPath()) - .field("rewritten.path", rewritten.uri().getPath()) - .log(); + // Log the path rewriting for troubleshooting + EcsLogger.info("com.artipie.group") + .message(String.format("Forwarding request to member: rewrote path %s to %s", line.uri().getPath(), rewritten.uri().getPath())) + .eventCategory("repository") + .eventAction("group_forward") + .field("repository.name", this.group) + .field("member.name", member.name()) + .log(); - return member.slice().response( - rewritten, - dropFullPathHeader(headers), - memberBody - ); - }); + return member.slice().response( + rewritten, + dropFullPathHeader(headers), + memberBody + ); } /** @@ -362,6 +605,7 @@ private void handleMemberResponse( final Response resp, final AtomicBoolean completed, final AtomicInteger pending, + final AtomicBoolean anyServerError, final CompletableFuture result, final long startTime, final Key pathKey, @@ -419,10 +663,9 @@ private void handleMemberResponse( drainBody(member.name(), resp.body()); } } else if (status == RsStatus.NOT_FOUND) { - // 404: Cache in negative cache and try next member - this.negativeCache.cacheNotFound(member.name(), pathKey); + // 404: try next member ctx.addTo(EcsLogger.info("com.artipie.group") - .message("Member returned 404, cached in negative cache") + .message("Member returned 404") .eventCategory("repository") .eventAction("group_query") .eventOutcome("not_found") @@ -432,19 +675,9 @@ private void handleMemberResponse( .log(); recordGroupMemberRequest(member.name(), "not_found"); drainBody(member.name(), resp.body()); - if (pending.decrementAndGet() == 0 && !completed.get()) { - ctx.addTo(EcsLogger.warn("com.artipie.group") - .message("All members exhausted, returning 404") - .eventCategory("repository") - .eventAction("group_query") - .eventOutcome("failure") - .field("repository.name", this.group)) - .log(); - recordNotFound(); - result.complete(ResponseBuilder.notFound().build()); - } + completeIfAllExhausted(pending, completed, anyServerError, result, ctx); } else { - // Other errors (500, etc.): try next member (don't cache) + // Server errors (500, 503, etc.): record failure, try next member ctx.addTo(EcsLogger.warn("com.artipie.group") .message("Member returned error status (" + (pending.get() - 1) + " pending)") .eventCategory("repository") @@ -454,19 +687,11 @@ private void handleMemberResponse( .field("member.name", member.name()) .field("http.response.status_code", status.code()) .log(); + member.recordFailure(); + anyServerError.set(true); recordGroupMemberRequest(member.name(), "error"); drainBody(member.name(), resp.body()); - if (pending.decrementAndGet() == 0 && !completed.get()) { - ctx.addTo(EcsLogger.warn("com.artipie.group") - .message("All members exhausted, returning 404") - .eventCategory("repository") - .eventAction("group_query") - .eventOutcome("failure") - .field("repository.name", this.group)) - .log(); - recordNotFound(); - result.complete(ResponseBuilder.notFound().build()); - } + completeIfAllExhausted(pending, completed, anyServerError, result, ctx); } } @@ -478,6 +703,7 @@ private void handleMemberFailure( final Throwable err, final AtomicBoolean completed, final AtomicInteger pending, + final AtomicBoolean anyServerError, final CompletableFuture result, final RequestContext ctx ) { @@ -491,10 +717,44 @@ private void handleMemberFailure( .field("error.message", err.getMessage())) .log(); member.recordFailure(); + anyServerError.set(true); + completeIfAllExhausted(pending, completed, anyServerError, result, ctx); + } + /** + * Complete the result future if all members have been exhausted. + * Returns 502 Bad Gateway if any member had a server error, + * otherwise returns 404 Not Found. + */ + private void completeIfAllExhausted( + final AtomicInteger pending, + final AtomicBoolean completed, + final AtomicBoolean anyServerError, + final CompletableFuture result, + final RequestContext ctx + ) { if (pending.decrementAndGet() == 0 && !completed.get()) { - recordNotFound(); - result.complete(ResponseBuilder.notFound().build()); + if (anyServerError.get()) { + ctx.addTo(EcsLogger.warn("com.artipie.group") + .message("All members exhausted with upstream errors, returning 502") + .eventCategory("repository") + .eventAction("group_query") + .eventOutcome("failure") + .field("repository.name", this.group)) + .log(); + result.complete(ResponseBuilder.badGateway() + .textBody("All upstream members failed").build()); + } else { + ctx.addTo(EcsLogger.warn("com.artipie.group") + .message("All members exhausted, returning 404") + .eventCategory("repository") + .eventAction("group_query") + .eventOutcome("failure") + .field("repository.name", this.group)) + .log(); + recordNotFound(); + result.complete(ResponseBuilder.notFound().build()); + } } } @@ -503,8 +763,19 @@ private void handleMemberFailure( * Uses streaming discard to avoid OOM on large responses (e.g., npm typescript ~30MB). */ private void drainBody(final String memberName, final Content body) { - // Use streaming subscriber that discards bytes without accumulating - // This prevents OOM when draining large npm package metadata + if (!DRAIN_PERMITS.tryAcquire()) { + // Too many concurrent drains — skip to prevent memory pressure + // The response will eventually be GC'd and the connection cleaned up + EcsLogger.debug("com.artipie.group") + .message("Skipping body drain (too many concurrent drains)") + .eventCategory("repository") + .eventAction("body_drain") + .eventOutcome("skipped") + .field("repository.name", GroupSlice.this.group) + .field("member.name", memberName) + .log(); + return; + } body.subscribe(new org.reactivestreams.Subscriber<>() { private org.reactivestreams.Subscription subscription; @@ -521,6 +792,7 @@ public void onNext(final java.nio.ByteBuffer item) { @Override public void onError(final Throwable err) { + DRAIN_PERMITS.release(); EcsLogger.warn("com.artipie.group") .message("Failed to drain response body") .eventCategory("repository") @@ -534,7 +806,7 @@ public void onError(final Throwable err) { @Override public void onComplete() { - // Body fully drained + DRAIN_PERMITS.release(); } }); } @@ -596,4 +868,33 @@ private void recordGroupMemberLatency(final String memberName, final String resu .recordGroupMemberLatency(this.group, memberName, result, latencyMs); } } + + /** + * Filter members by routing rules for the given path. + * If no routing rules are configured, all members are returned. + * Members with matching routing rules are included. Members with + * no routing rules also participate (default: include). + * + * @param path Request path + * @return Filtered list of members to query + */ + private List filterByRoutingRules(final String path) { + if (this.routingRules.isEmpty()) { + return this.members; + } + // Collect members that have explicit routing rules + final Set ruledMembers = this.routingRules.stream() + .map(RoutingRule::member) + .collect(Collectors.toSet()); + // Collect members whose rules match this path + final Set matchedMembers = this.routingRules.stream() + .filter(rule -> rule.matches(path)) + .map(RoutingRule::member) + .collect(Collectors.toSet()); + // Include: members with matching rules + members with no rules (default include) + return this.members.stream() + .filter(m -> matchedMembers.contains(m.name()) + || !ruledMembers.contains(m.name())) + .toList(); + } } diff --git a/artipie-main/src/main/java/com/artipie/group/MavenGroupSlice.java b/artipie-main/src/main/java/com/artipie/group/MavenGroupSlice.java index c47ee15b3..e60075d5d 100644 --- a/artipie-main/src/main/java/com/artipie/group/MavenGroupSlice.java +++ b/artipie-main/src/main/java/com/artipie/group/MavenGroupSlice.java @@ -104,6 +104,34 @@ public MavenGroupSlice( this.metadataCache = new GroupMetadataCache(group); } + /** + * Constructor with injectable cache (for testing). + * @param delegate Delegate group slice + * @param group Group repository name + * @param members Member repository names + * @param resolver Slice resolver + * @param port Server port + * @param depth Nesting depth + * @param cache Group metadata cache to use + */ + public MavenGroupSlice( + final Slice delegate, + final String group, + final List members, + final SliceResolver resolver, + final int port, + final int depth, + final GroupMetadataCache cache + ) { + this.delegate = delegate; + this.group = group; + this.members = members; + this.resolver = resolver; + this.port = port; + this.depth = depth; + this.metadataCache = cache; + } + @Override public CompletableFuture response( final RequestLine line, @@ -283,18 +311,35 @@ private CompletableFuture mergeMetadata( final long fetchDuration = System.currentTimeMillis() - fetchStartTime; if (metadataList.isEmpty()) { - EcsLogger.warn("com.artipie.maven") - .message("No metadata found in any member") - .eventCategory("repository") - .eventAction("metadata_merge") - .eventOutcome("failure") - .field("repository.name", this.group) - .field("url.path", path) - .field("fetch.duration.ms", fetchDuration) - .log(); - return CompletableFuture.completedFuture( - ResponseBuilder.notFound().build() - ); + // All members failed — try last-known-good stale fallback + return MavenGroupSlice.this.metadataCache.getStale(cacheKey) + .thenApply(stale -> { + if (stale.isPresent()) { + EcsLogger.warn("com.artipie.maven") + .message("Returning stale metadata (all members failed)") + .eventCategory("repository") + .eventAction("metadata_merge") + .eventOutcome("stale_fallback") + .field("repository.name", MavenGroupSlice.this.group) + .field("url.path", path) + .field("event.duration", fetchDuration * 1_000_000L) + .log(); + return ResponseBuilder.ok() + .header("Content-Type", "application/xml") + .body(stale.get()) + .build(); + } + EcsLogger.warn("com.artipie.maven") + .message("No metadata found in any member and no stale fallback") + .eventCategory("repository") + .eventAction("metadata_merge") + .eventOutcome("failure") + .field("repository.name", MavenGroupSlice.this.group) + .field("url.path", path) + .field("event.duration", fetchDuration * 1_000_000L) + .log(); + return ResponseBuilder.notFound().build(); + }); } // Track merge duration separately (actual XML processing time) @@ -316,28 +361,26 @@ private CompletableFuture mergeMetadata( // Log slow fetches (>500ms) - expected for proxy repos if (fetchDuration > 500) { EcsLogger.info("com.artipie.maven") - .message("Slow member fetch (" + metadataList.size() + " members)") + .message(String.format("Slow member fetch (%d members), merge took %dms", metadataList.size(), mergeDuration)) .eventCategory("repository") .eventAction("metadata_fetch") .eventOutcome("success") .field("repository.name", this.group) .field("url.path", path) - .field("fetch.duration.ms", fetchDuration) - .field("merge.duration.ms", mergeDuration) + .field("event.duration", fetchDuration * 1_000_000L) .log(); } // Log slow merges (>50ms) - indicates actual performance issue if (mergeDuration > 50) { EcsLogger.warn("com.artipie.maven") - .message("Slow metadata merge (" + metadataList.size() + " members)") + .message(String.format("Slow metadata merge (%d members), fetch took %dms", metadataList.size(), fetchDuration)) .eventCategory("repository") .eventAction("metadata_merge") .eventOutcome("success") .field("repository.name", this.group) .field("url.path", path) - .field("fetch.duration.ms", fetchDuration) - .field("merge.duration.ms", mergeDuration) + .field("event.duration", mergeDuration * 1_000_000L) .log(); } diff --git a/artipie-main/src/main/java/com/artipie/group/MemberSlice.java b/artipie-main/src/main/java/com/artipie/group/MemberSlice.java index dee225ace..3198bdee3 100644 --- a/artipie-main/src/main/java/com/artipie/group/MemberSlice.java +++ b/artipie-main/src/main/java/com/artipie/group/MemberSlice.java @@ -7,78 +7,102 @@ import com.artipie.http.Slice; import com.artipie.http.rq.RequestLine; import com.artipie.http.log.EcsLogger; +import com.artipie.http.timeout.AutoBlockRegistry; +import com.artipie.http.timeout.AutoBlockSettings; import java.net.URI; -import java.time.Duration; -import java.time.Instant; +import java.util.Locale; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; /** - * Member repository slice with circuit breaker for failure isolation. - * - *

Circuit breaker states: + * Member repository slice with circuit breaker delegating to {@link AutoBlockRegistry}. + * + *

Circuit breaker states (managed by the registry): *

    - *
  • CLOSED: Normal operation, requests pass through
    • - *
  • OPEN: Fast-fail mode, requests rejected immediately (after N failures)
    • - *
  • HALF_OPEN: Testing recovery, allow one request through
    • + *
  • ONLINE: Normal operation, requests pass through
    • + *
  • BLOCKED: Fast-fail mode, requests rejected immediately (after N failures)
    • + *
  • PROBING: Testing recovery, allow requests through
    • *
- * - *

Circuit breaker thresholds: - *

    - *
  • Open after 5 consecutive failures
    • - *
  • Stay open for 30 seconds
    • - *
  • Reset counter on first success
    • - *
- * + * * @since 1.18.23 */ public final class MemberSlice { /** - * Number of consecutive failures before opening circuit. + * Member repository name. */ - private static final int FAILURE_THRESHOLD = 5; + private final String name; /** - * How long to keep circuit open before attempting recovery. + * Underlying slice for this member. */ - private static final Duration RESET_TIMEOUT = Duration.ofSeconds(30); + private final Slice delegate; /** - * Member repository name. + * Auto-block registry for circuit breaker state. */ - private final String name; + private final AutoBlockRegistry registry; /** - * Underlying slice for this member. + * Whether this member is a proxy repository (fetches from upstream). + * Proxy members must always be queried on index miss because their + * content is not pre-indexed — it only gets indexed after being cached. */ - private final Slice delegate; + private final boolean proxy; /** - * Consecutive failure count. + * Backward-compatible constructor (non-proxy). + * Creates a local {@link AutoBlockRegistry} with default settings. + * + * @param name Member repository name + * @param delegate Underlying slice */ - private final AtomicInteger failureCount = new AtomicInteger(0); + public MemberSlice(final String name, final Slice delegate) { + this(name, delegate, new AutoBlockRegistry(AutoBlockSettings.defaults()), false); + } /** - * When circuit was opened (null if closed). + * Constructor with proxy flag. + * + * @param name Member repository name + * @param delegate Underlying slice + * @param proxy Whether this member is a proxy repository */ - private volatile Instant openedAt = null; + public MemberSlice(final String name, final Slice delegate, final boolean proxy) { + this(name, delegate, new AutoBlockRegistry(AutoBlockSettings.defaults()), proxy); + } /** - * Constructor. - * + * Constructor with shared registry (non-proxy). + * * @param name Member repository name * @param delegate Underlying slice + * @param registry Shared auto-block registry */ - public MemberSlice(final String name, final Slice delegate) { + public MemberSlice(final String name, final Slice delegate, + final AutoBlockRegistry registry) { + this(name, delegate, registry, false); + } + + /** + * Full constructor. + * + * @param name Member repository name + * @param delegate Underlying slice + * @param registry Shared auto-block registry + * @param proxy Whether this member is a proxy repository + */ + public MemberSlice(final String name, final Slice delegate, + final AutoBlockRegistry registry, final boolean proxy) { this.name = Objects.requireNonNull(name, "name"); - this.delegate = Objects.requireNonNull(delegate, "delegate"); + this.delegate = delegate; + this.registry = Objects.requireNonNull(registry, "registry"); + this.proxy = proxy; } /** * Get member repository name. - * + * * @return Member name */ public String name() { @@ -87,7 +111,7 @@ public String name() { /** * Get underlying slice. - * + * * @return Delegate slice */ public Slice slice() { @@ -95,78 +119,46 @@ public Slice slice() { } /** - * Check if circuit breaker is in OPEN state. - * + * Whether this member is a proxy repository. + * Proxy members fetch content from upstream registries on-demand. + * Their content is only indexed after being cached, so they must + * always be queried on an index miss. + * + * @return True if this member is a proxy + */ + public boolean isProxy() { + return this.proxy; + } + + /** + * Check if circuit breaker is in BLOCKED state. + * * @return True if circuit is open (fast-failing) */ public boolean isCircuitOpen() { - if (this.openedAt == null) { - return false; - } - - // Check if timeout has expired (transition to HALF_OPEN) - final Duration elapsed = Duration.between(this.openedAt, Instant.now()); - if (elapsed.compareTo(RESET_TIMEOUT) >= 0) { - EcsLogger.info("com.artipie.group") - .message("Circuit breaker entering HALF_OPEN state for member: " + this.name) - .eventCategory("repository") - .eventAction("circuit_breaker_half_open") - .eventOutcome("success") - .duration(elapsed.toMillis()) - .log(); - this.openedAt = null; - return false; - } - - return true; + return this.registry.isBlocked(this.name); } /** * Record successful response from this member. - * Resets circuit breaker state. + * Resets circuit breaker state via registry. */ public void recordSuccess() { - final int previousFailures = this.failureCount.getAndSet(0); - if (previousFailures > 0) { - EcsLogger.info("com.artipie.group") - .message("Member '" + this.name + "' recovered - circuit breaker CLOSED (previous failures: " + previousFailures + ")") - .eventCategory("repository") - .eventAction("circuit_breaker_close") - .eventOutcome("success") - .log(); - } - this.openedAt = null; + this.registry.recordSuccess(this.name); } /** * Record failed response from this member. - * May open circuit breaker if threshold exceeded. + * May block the remote via registry if threshold exceeded. */ public void recordFailure() { - final int failures = this.failureCount.incrementAndGet(); - - if (failures >= FAILURE_THRESHOLD && this.openedAt == null) { - this.openedAt = Instant.now(); - EcsLogger.warn("com.artipie.group") - .message("Circuit breaker OPENED for member '" + this.name + "' after " + failures + " consecutive failures") - .eventCategory("repository") - .eventAction("circuit_breaker_open") - .eventOutcome("failure") - .log(); - } else if (failures < FAILURE_THRESHOLD) { - EcsLogger.debug("com.artipie.group") - .message("Member '" + this.name + "' failure count incremented (" + failures + "/" + FAILURE_THRESHOLD + ")") - .eventCategory("repository") - .eventAction("circuit_breaker_failure") - .eventOutcome("failure") - .log(); - } + this.registry.recordFailure(this.name); } /** * Rewrite request path to include member repository name. * - *

Transforms: /path → /member/path + *

Transforms: /path -> /member/path * * @param original Original request line * @return Rewritten request line with member prefix @@ -195,48 +187,30 @@ public RequestLine rewritePath(final RequestLine original) { ); EcsLogger.info("com.artipie.group") - .message("MemberSlice rewritePath") + .message(String.format("MemberSlice rewritePath: %s to %s", raw, result.uri().getPath())) .eventCategory("repository") .eventAction("path_rewrite") .field("member.name", this.name) - .field("original.path", raw) - .field("rewritten.path", result.uri().getPath()) .log(); return result; } - /** - * Get current failure count for monitoring. - * - * @return Number of consecutive failures - */ - public int failureCount() { - return this.failureCount.get(); - } - /** * Get circuit breaker state for monitoring. - * - * @return "OPEN", "HALF_OPEN", or "CLOSED" + * + * @return "ONLINE", "BLOCKED", or "PROBING" */ public String circuitState() { - if (this.openedAt == null) { - return "CLOSED"; - } - final Duration elapsed = Duration.between(this.openedAt, Instant.now()); - if (elapsed.compareTo(RESET_TIMEOUT) >= 0) { - return "HALF_OPEN"; - } - return "OPEN"; + return this.registry.status(this.name).toUpperCase(Locale.ROOT); } @Override public String toString() { return String.format( - "MemberSlice{name=%s, failures=%d, circuit=%s}", + "MemberSlice{name=%s, proxy=%s, circuit=%s}", this.name, - this.failureCount.get(), + this.proxy, circuitState() ); } diff --git a/artipie-main/src/main/java/com/artipie/group/RoutingRule.java b/artipie-main/src/main/java/com/artipie/group/RoutingRule.java new file mode 100644 index 000000000..46ad975b0 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/group/RoutingRule.java @@ -0,0 +1,93 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Routing rule for group repositories. + * Routes specific request paths to designated members, + * preventing unnecessary upstream queries to non-matching members. + * + * @since 1.20.13 + */ +public sealed interface RoutingRule permits RoutingRule.PathPrefix, RoutingRule.PathPattern { + + /** + * The member this rule applies to. + * @return Member repository name + */ + String member(); + + /** + * Check if a request path matches this rule. + * @param path Request path (e.g., "/com/example/foo/1.0/foo-1.0.jar") + * @return True if this rule matches the path + */ + boolean matches(String path); + + /** + * Prefix-based routing rule. + * Matches any path that starts with the specified prefix. + * + * @param member Member repository name + * @param prefix Path prefix to match (e.g., "com/mycompany/") + */ + record PathPrefix(String member, String prefix) implements RoutingRule { + + /** + * Ctor. + * @param member Member repository name + * @param prefix Path prefix to match + */ + public PathPrefix { + Objects.requireNonNull(member, "member"); + Objects.requireNonNull(prefix, "prefix"); + } + + @Override + public boolean matches(final String path) { + final String normalized = path.startsWith("/") ? path.substring(1) : path; + return normalized.startsWith(this.prefix); + } + } + + /** + * Regex pattern-based routing rule. + * Matches any path that matches the specified regex pattern. + * + * @param member Member repository name + * @param regex Regex pattern string (e.g., "org/apache/.*") + */ + record PathPattern(String member, String regex) implements RoutingRule { + + /** + * Compiled pattern for efficient matching. + */ + private static final java.util.concurrent.ConcurrentHashMap PATTERNS = + new java.util.concurrent.ConcurrentHashMap<>(); + + /** + * Ctor. + * @param member Member repository name + * @param regex Regex pattern string + */ + public PathPattern { + Objects.requireNonNull(member, "member"); + Objects.requireNonNull(regex, "regex"); + // Pre-compile to catch invalid regex early + PATTERNS.computeIfAbsent(regex, Pattern::compile); + } + + @Override + public boolean matches(final String path) { + final String normalized = path.startsWith("/") ? path.substring(1) : path; + return PATTERNS.computeIfAbsent(this.regex, Pattern::compile) + .matcher(normalized) + .matches(); + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/group/WritableGroupSlice.java b/artipie-main/src/main/java/com/artipie/group/WritableGroupSlice.java new file mode 100644 index 000000000..22f7b7c5e --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/group/WritableGroupSlice.java @@ -0,0 +1,61 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import com.artipie.asto.Content; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.Slice; +import com.artipie.http.rq.RequestLine; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; + +/** + * Write-through group slice. + * Routes write operations (PUT, POST, DELETE) to a designated write target member, + * while read operations (GET, HEAD) use normal group resolution. + * + * @since 1.20.13 + */ +public final class WritableGroupSlice implements Slice { + + /** + * Delegate for read operations (group resolution). + */ + private final Slice readDelegate; + + /** + * Write target slice for PUT/POST/DELETE. + */ + private final Slice writeTarget; + + /** + * Ctor. + * @param readDelegate Group slice for reads + * @param writeTarget Target slice for writes + */ + public WritableGroupSlice(final Slice readDelegate, final Slice writeTarget) { + this.readDelegate = Objects.requireNonNull(readDelegate, "readDelegate"); + this.writeTarget = Objects.requireNonNull(writeTarget, "writeTarget"); + } + + @Override + public CompletableFuture response( + final RequestLine line, final Headers headers, final Content body + ) { + final String method = line.method().value(); + if ("GET".equals(method) || "HEAD".equals(method)) { + return this.readDelegate.response(line, headers, body); + } + if ("PUT".equals(method) || "POST".equals(method) || "DELETE".equals(method)) { + return this.writeTarget.response(line, headers, body); + } + return CompletableFuture.completedFuture( + ResponseBuilder.methodNotAllowed().build() + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/http/ApiRoutingSlice.java b/artipie-main/src/main/java/com/artipie/http/ApiRoutingSlice.java index 1771552f2..a4b644312 100644 --- a/artipie-main/src/main/java/com/artipie/http/ApiRoutingSlice.java +++ b/artipie-main/src/main/java/com/artipie/http/ApiRoutingSlice.java @@ -6,12 +6,14 @@ import com.artipie.asto.Content; import com.artipie.http.rq.RequestLine; +import com.artipie.settings.repo.Repositories; import org.apache.http.client.utils.URIBuilder; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -19,19 +21,20 @@ * Slice decorator which redirects API requests to repository format paths. * Supports multiple access patterns for different repository types. *

- * Supported patterns for most repositories: - * - /{repo_name} - * - /{prefix}/{repo_name} - * - /api/{repo_name} - * - /{prefix}/api/{repo_name} - * - /api/{repo_type}/{repo_name} - * - /{prefix}/api/{repo_type}/{repo_name} + * Supported patterns for all repositories: + *

    + *
  • /{repo_name}
    • + *
  • /{prefix}/{repo_name}
    • + *
  • /api/{repo_name}
    • + *
  • /{prefix}/api/{repo_name}
    • + *
  • /api/{repo_type}/{repo_name}
    • + *
  • /{prefix}/api/{repo_type}/{repo_name}
    • + *
*

- * For gradle, rpm, and maven (limited support): - * - /{repo_name} - * - /{prefix}/{repo_name} - * - /api/{repo_name} - * - /{prefix}/api/{repo_name} + * When the first segment after /api/ matches a known repo type (e.g., "npm"), + * the second segment is checked against the repository registry. If it is a + * known repo name, the repo_type interpretation is used. Otherwise, the first + * segment is treated as the repo name (repo_name interpretation). */ public final class ApiRoutingSlice implements Slice { @@ -75,11 +78,39 @@ public final class ApiRoutingSlice implements Slice { private final Slice origin; /** - * Decorates slice with API routing. + * Predicate to check if a repository name exists. + */ + private final Predicate repoExists; + + /** + * Constructor with repository registry for disambiguation. + * @param origin Origin slice + * @param repos Repository registry + */ + public ApiRoutingSlice(final Slice origin, final Repositories repos) { + this.origin = origin; + this.repoExists = name -> repos.config(name).isPresent(); + } + + /** + * Constructor without repository registry (backward compatible). + * Falls back to assuming segments[1] is always a repo name + * when first segment matches a repo type. * @param origin Origin slice */ public ApiRoutingSlice(final Slice origin) { this.origin = origin; + this.repoExists = name -> true; + } + + /** + * Constructor with custom predicate (for testing). + * @param origin Origin slice + * @param repoExists Predicate to check if a repo name exists + */ + ApiRoutingSlice(final Slice origin, final Predicate repoExists) { + this.origin = origin; + this.repoExists = repoExists; } @Override @@ -88,60 +119,61 @@ public CompletableFuture response( ) { final String path = line.uri().getPath(); final Matcher matcher = PTN_API.matcher(path); - + if (matcher.matches()) { final String prefix = matcher.group(1); // e.g., "/test_prefix" or null final String apiPath = matcher.group(2); // Everything after /api/ - + // Split the path into segments final String[] segments = apiPath.split("/", 3); if (segments.length < 1) { return this.origin.response(line, headers, body); } - - // Check if first segment is a repo_type + + // Check if first segment is a repo_type. + // Ambiguity: /api/npm/X — is "npm" the repo_type or repo_name? + // Resolved by checking if X is a known repository name. If yes, + // use repo_type interpretation. If not, treat first segment as + // the repo_name. final String firstSegment = segments[0]; - if (REPO_TYPE_MAPPING.containsKey(firstSegment) && segments.length >= 2) { + if (REPO_TYPE_MAPPING.containsKey(firstSegment) + && segments.length >= 2 + && this.repoExists.test(segments[1])) { // Pattern: /api/{repo_type}/{repo_name}[/rest] final String repoName = segments[1]; final String rest = segments.length > 2 ? "/" + segments[2] : ""; - final String newPath = (prefix != null ? prefix : "") + "/" + repoName + rest; - - // Preserve original path in header for metadata-url generation - final Headers newHeaders = headers.copy(); - newHeaders.add("X-Original-Path", path); - - return this.origin.response( - new RequestLine( - line.method().toString(), - new URIBuilder(line.uri()).setPath(newPath).toString(), - line.version() - ), - newHeaders, - body - ); + return this.rewrite(line, headers, body, path, prefix, repoName, rest); } else { // Pattern: /api/{repo_name}[/rest] final String repoName = firstSegment; - final String rest = segments.length > 1 ? "/" + apiPath.substring(repoName.length() + 1) : ""; - final String newPath = (prefix != null ? prefix : "") + "/" + repoName + rest; - - // Preserve original path in header for metadata-url generation - final Headers newHeaders = headers.copy(); - newHeaders.add("X-Original-Path", path); - - return this.origin.response( - new RequestLine( - line.method().toString(), - new URIBuilder(line.uri()).setPath(newPath).toString(), - line.version() - ), - newHeaders, - body - ); + final String rest = segments.length > 1 + ? "/" + apiPath.substring(repoName.length() + 1) : ""; + return this.rewrite(line, headers, body, path, prefix, repoName, rest); } } - + return this.origin.response(line, headers, body); } + + /** + * Rewrite the request path and forward to origin. + */ + private CompletableFuture rewrite( + final RequestLine line, final Headers headers, final Content body, + final String originalPath, final String prefix, + final String repoName, final String rest + ) { + final String newPath = (prefix != null ? prefix : "") + "/" + repoName + rest; + final Headers newHeaders = headers.copy(); + newHeaders.add("X-Original-Path", originalPath); + return this.origin.response( + new RequestLine( + line.method().toString(), + new URIBuilder(line.uri()).setPath(newPath).toString(), + line.version() + ), + newHeaders, + body + ); + } } diff --git a/artipie-main/src/main/java/com/artipie/http/HealthSlice.java b/artipie-main/src/main/java/com/artipie/http/HealthSlice.java index 7f1f10889..d726e022d 100644 --- a/artipie-main/src/main/java/com/artipie/http/HealthSlice.java +++ b/artipie-main/src/main/java/com/artipie/http/HealthSlice.java @@ -5,72 +5,27 @@ package com.artipie.http; import com.artipie.asto.Content; -import com.artipie.asto.Key; import com.artipie.http.rq.RequestLine; -import com.artipie.settings.Settings; -import javax.json.Json; -import java.nio.charset.StandardCharsets; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; /** - * Health check slice. - *

- * Returns JSON with verbose status checks, - * response status is {@code OK} if all status passed and {@code UNAVAILABLE} if any failed. + * Lightweight health check slice for NLB/load-balancer probes. + * Returns 200 OK immediately with no I/O, no probes, no blocking. + * Returns 200 OK with JSON body {@code {"status":"ok"}}. + * + * @since 1.20.13 */ -@SuppressWarnings("PMD.AvoidCatchingGenericException") public final class HealthSlice implements Slice { - /** - * Artipie settings. - */ - private final Settings settings; - - /** - * New health slice. - * @param settings Artipie settings - */ - public HealthSlice(final Settings settings) { - this.settings = settings; - } - @Override - public CompletableFuture response(RequestLine line, Headers headers, Content body) { - return this.storageStatus() - .thenApply( - ok -> { - if (ok) { - return ResponseBuilder.ok() - .jsonBody(Json.createArrayBuilder() - .add(Json.createObjectBuilder().add("storage", "ok")) - .build() - ) - .build(); - } - return ResponseBuilder.unavailable() - .jsonBody(Json.createArrayBuilder().add( - Json.createObjectBuilder().add("storage", "failure") - ).build()) - .build(); - } - ).toCompletableFuture(); - } - - /** - * Checks storage status by writing {@code OK} to storage. - * @return True if OK - */ - @SuppressWarnings("PMD.OnlyOneReturn") - private CompletionStage storageStatus() { - try { - return this.settings.configStorage().save( - new Key.From(".system", "test"), - new Content.From("OK".getBytes(StandardCharsets.US_ASCII)) - ).thenApply(none -> true).exceptionally(ignore -> false); - } catch (final Exception ignore) { - return CompletableFuture.completedFuture(false); - } + public CompletableFuture response( + final RequestLine line, final Headers headers, final Content body + ) { + return CompletableFuture.completedFuture( + ResponseBuilder.ok() + .jsonBody("{\"status\":\"ok\"}") + .build() + ); } } diff --git a/artipie-main/src/main/java/com/artipie/http/MainSlice.java b/artipie-main/src/main/java/com/artipie/http/MainSlice.java index 0d59e1d33..69bac4c57 100644 --- a/artipie-main/src/main/java/com/artipie/http/MainSlice.java +++ b/artipie-main/src/main/java/com/artipie/http/MainSlice.java @@ -64,47 +64,46 @@ private static Slice buildMainSlice(final Settings settings, final RepositorySli events, true ); - // Wrap entire routing in TimeoutSlice to prevent request leaks - // Without this, hung requests never timeout and accumulate indefinitely - return new TimeoutSlice( - new SliceRoute( - MainSlice.EMPTY_PATH, - new RtRulePath( - new RtRule.ByPath(Pattern.compile("/\\.health")), - new HealthSlice(settings) - ), - new RtRulePath( - new RtRule.All( - MethodRule.GET, - new RtRule.ByPath("/.version") - ), - new VersionSlice(new ArtipieProperties()) + // No wall-clock timeout here — idle-based timeout is handled by Vert.x + // (HttpServerOptions.setIdleTimeout). A global wall-clock timeout kills + // legitimate large transfers (multi-GB Docker blobs, Maven artifacts). + return new SliceRoute( + MainSlice.EMPTY_PATH, + new RtRulePath( + new RtRule.ByPath(Pattern.compile("/\\.health")), + new HealthSlice() + ), + new RtRulePath( + new RtRule.All( + MethodRule.GET, + new RtRule.ByPath("/.version") ), - new RtRulePath( - new RtRule.All( - new RtRule.ByPath("/\\.import/.*"), - new RtRule.Any(MethodRule.PUT, MethodRule.POST) - ), - new ImportSlice(imports) + new VersionSlice(new ArtipieProperties()) + ), + new RtRulePath( + new RtRule.All( + new RtRule.ByPath("/\\.import/.*"), + new RtRule.Any(MethodRule.PUT, MethodRule.POST) ), - new RtRulePath( - new RtRule.All( - new RtRule.ByPath("/\\.merge/.*"), - MethodRule.POST - ), - new MergeShardsSlice(slices) + new ImportSlice(imports) + ), + new RtRulePath( + new RtRule.All( + new RtRule.ByPath("/\\.merge/.*"), + MethodRule.POST ), - new RtRulePath( - RtRule.FALLBACK, - new DockerRoutingSlice( - settings, - new ApiRoutingSlice( - new SliceByPath(slices, settings.prefixes()) - ) + new MergeShardsSlice(slices) + ), + new RtRulePath( + RtRule.FALLBACK, + new DockerRoutingSlice( + settings, + new ApiRoutingSlice( + new SliceByPath(slices, settings.prefixes()), + slices.repositories() ) ) - ), - settings.httpClientSettings().proxyTimeout() // Use configured timeout (default 120s) + ) ); } } diff --git a/artipie-main/src/main/java/com/artipie/http/MergeShardsSlice.java b/artipie-main/src/main/java/com/artipie/http/MergeShardsSlice.java index ed7dd6141..674e5c2a4 100644 --- a/artipie-main/src/main/java/com/artipie/http/MergeShardsSlice.java +++ b/artipie-main/src/main/java/com/artipie/http/MergeShardsSlice.java @@ -13,6 +13,7 @@ import com.artipie.http.rq.RequestLine; import com.artipie.settings.repo.RepoConfig; import com.artipie.maven.metadata.MavenMetadata; +import com.artipie.maven.metadata.MavenTimestamp; import com.artipie.helm.metadata.IndexYaml; import com.artipie.helm.metadata.IndexYamlMapping; import com.artipie.http.log.EcsLogger; @@ -140,11 +141,14 @@ private CompletionStage mergePypiShards(final Storage storage) { }).toCompletableFuture()); } return CompletableFuture.allOf(lines.toArray(CompletableFuture[]::new)) - .thenApply(v -> { - final StringBuilder sb = new StringBuilder(); - lines.forEach(fut -> sb.append(fut.join())); - return String.format("\n\n \n%s\n\n", sb.toString()); - }) + .thenApply(v -> + String.format( + "\n\n \n%s\n\n", + lines.stream() + .map(CompletableFuture::join) + .collect(java.util.stream.Collectors.joining()) + ) + ) .thenCompose(html -> st.save(out, new Content.From(html.getBytes(StandardCharsets.UTF_8)))); })); } @@ -451,7 +455,7 @@ private CompletionStage mergeMavenShards(final Storage storage) { .ifPresent(rel -> d.add("release").set(rel).up()); d.add("versions"); versions.forEach(ver -> d.add("version").set(ver).up()); - d.up().add("lastUpdated").set(String.format("%tY% mergeHelmShards(final Storage storage, fina final String digest = obj.getString("sha256", null); final String name = obj.getString("name", chart); final String path = obj.getString("path", null); - + if (version != null && url != null) { // Use the path field from shard if available, it's more reliable if (path != null) { @@ -564,22 +568,22 @@ private CompletionStage mergeHelmShards(final Storage storage, fina } } } - + final Map entry = new HashMap<>(); // Required fields for Helm index.yaml entry.put("apiVersion", "v1"); entry.put("name", name); entry.put("version", version); entry.put("created", new DateTimeNow().asString()); - + // URL should be an array in Helm index.yaml entry.put("urls", java.util.List.of(url)); - + // Use proper field name for digest if (digest != null && !digest.isBlank()) { entry.put("digest", digest); } - + // Add appVersion, description, etc. if available in shard if (obj.containsKey("appVersion")) { entry.put("appVersion", obj.getString("appVersion")); @@ -593,22 +597,40 @@ private CompletionStage mergeHelmShards(final Storage storage, fina if (obj.containsKey("icon")) { entry.put("icon", obj.getString("icon")); } - + versions.add(entry); vers.incrementAndGet(); } } + }).exceptionally(err -> { + com.artipie.http.log.EcsLogger.warn("com.artipie.http") + .message("MergeShardsSlice: async operation failed") + .eventCategory("merge_shards") + .eventAction("async_error") + .eventOutcome("failure") + .error(err) + .log(); + return null; })); } // Wait for all shard reads for this chart, then add to mapping reads.add(CompletableFuture.allOf(chartReads.toArray(CompletableFuture[]::new)) .thenRun(() -> { - if (!versions.isEmpty()) { - synchronized (chartVersions) { + synchronized (chartVersions) { + if (!versions.isEmpty()) { chartVersions.put(chart, versions); } } + }).exceptionally(err -> { + com.artipie.http.log.EcsLogger.warn("com.artipie.http") + .message("MergeShardsSlice: async operation failed") + .eventCategory("merge_shards") + .eventAction("async_error") + .eventOutcome("failure") + .error(err) + .log(); + return null; })); }); diff --git a/artipie-main/src/main/java/com/artipie/http/slice/BrowsableSlice.java b/artipie-main/src/main/java/com/artipie/http/slice/BrowsableSlice.java index b486c040f..54f6226d0 100644 --- a/artipie-main/src/main/java/com/artipie/http/slice/BrowsableSlice.java +++ b/artipie-main/src/main/java/com/artipie/http/slice/BrowsableSlice.java @@ -272,6 +272,21 @@ private static Storage unwrapStorage(final Storage storage) { boolean unwrapped = false; try { + // Try DispatchedStorage unwrapping + if (className.equals("DispatchedStorage")) { + final java.lang.reflect.Field delegate = + current.getClass().getDeclaredField("delegate"); + delegate.setAccessible(true); + final Storage next = (Storage) delegate.get(current); + EcsLogger.debug("com.artipie.http") + .message("Unwrapped DispatchedStorage to: " + next.getClass().getSimpleName()) + .eventCategory("http") + .eventAction("storage_unwrap") + .log(); + current = next; + unwrapped = true; + } + // Try DiskCacheStorage unwrapping if (className.equals("DiskCacheStorage")) { final java.lang.reflect.Field backend = diff --git a/artipie-main/src/main/java/com/artipie/http/slice/FileSystemBrowseSlice.java b/artipie-main/src/main/java/com/artipie/http/slice/FileSystemBrowseSlice.java index 76e3ebeb7..be426a5c7 100644 --- a/artipie-main/src/main/java/com/artipie/http/slice/FileSystemBrowseSlice.java +++ b/artipie-main/src/main/java/com/artipie/http/slice/FileSystemBrowseSlice.java @@ -471,32 +471,60 @@ private static ByteBuffer toBuffer(final String text) { */ private static Path getBasePath(final Storage storage) { try { + // Unwrap decorators to find SubStorage / FileStorage + final Storage unwrapped = unwrapDecorators(storage); // Check if this is SubStorage - if (storage.getClass().getSimpleName().equals("SubStorage")) { + if (unwrapped.getClass().getSimpleName().equals("SubStorage")) { // Extract prefix from SubStorage - final Field prefixField = storage.getClass().getDeclaredField("prefix"); + final Field prefixField = unwrapped.getClass().getDeclaredField("prefix"); prefixField.setAccessible(true); - final Key prefix = (Key) prefixField.get(storage); - - // Extract origin (wrapped FileStorage) - final Field originField = storage.getClass().getDeclaredField("origin"); + final Key prefix = (Key) prefixField.get(unwrapped); + + // Extract origin (wrapped FileStorage, possibly via DispatchedStorage) + final Field originField = unwrapped.getClass().getDeclaredField("origin"); originField.setAccessible(true); - final Storage origin = (Storage) originField.get(storage); - + final Storage origin = unwrapDecorators((Storage) originField.get(unwrapped)); + // Get FileStorage base path final Path basePath = getFileStoragePath(origin); - + // Combine base path + prefix return basePath.resolve(prefix.string()); } else { // Direct FileStorage - return getFileStoragePath(storage); + return getFileStoragePath(unwrapped); } } catch (Exception e) { throw new RuntimeException("Failed to access storage base path", e); } } - + + /** + * Unwrap decorator storages (DispatchedStorage, etc.) to find the + * underlying SubStorage or FileStorage. + * + * @param storage Storage to unwrap + * @return Unwrapped storage + */ + private static Storage unwrapDecorators(final Storage storage) { + Storage current = storage; + for (int depth = 0; depth < 10; depth++) { + final String name = current.getClass().getSimpleName(); + if ("DispatchedStorage".equals(name)) { + try { + final Field delegate = current.getClass().getDeclaredField("delegate"); + delegate.setAccessible(true); + current = (Storage) delegate.get(current); + } catch (Exception e) { + break; + } + } else { + break; + } + } + return current; + } + /** * Extract the dir field from FileStorage. * diff --git a/artipie-main/src/main/java/com/artipie/http/slice/RepoMetricsSlice.java b/artipie-main/src/main/java/com/artipie/http/slice/RepoMetricsSlice.java index 8a2f8eae1..a297d297b 100644 --- a/artipie-main/src/main/java/com/artipie/http/slice/RepoMetricsSlice.java +++ b/artipie-main/src/main/java/com/artipie/http/slice/RepoMetricsSlice.java @@ -9,6 +9,7 @@ import com.artipie.http.Response; import com.artipie.http.RsStatus; import com.artipie.http.Slice; +import com.artipie.http.log.EcsLogger; import com.artipie.http.rq.RequestLine; import com.artipie.metrics.MicrometerMetrics; import io.reactivex.Flowable; @@ -114,8 +115,11 @@ public CompletableFuture response( respBytes ); } - } catch (NumberFormatException ignored) { - // Skip if Content-Length is invalid + } catch (final NumberFormatException ex) { + EcsLogger.debug("com.artipie.metrics") + .message("Invalid Content-Length header value") + .error(ex) + .log(); } }); } diff --git a/artipie-main/src/main/java/com/artipie/importer/MetadataRegenerator.java b/artipie-main/src/main/java/com/artipie/importer/MetadataRegenerator.java index b6a8bf8e8..0ccbd25b5 100644 --- a/artipie-main/src/main/java/com/artipie/importer/MetadataRegenerator.java +++ b/artipie-main/src/main/java/com/artipie/importer/MetadataRegenerator.java @@ -14,6 +14,7 @@ import com.artipie.helm.TgzArchive; import com.artipie.helm.metadata.IndexYaml; import com.artipie.importer.api.DigestType; +import com.artipie.maven.metadata.MavenTimestamp; import com.artipie.maven.metadata.Version; import com.artipie.npm.MetaUpdate; import com.artipie.http.log.EcsLogger; @@ -27,9 +28,7 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; + import java.util.Base64; import java.util.Collection; import java.util.EnumSet; @@ -54,12 +53,6 @@ @SuppressWarnings("PMD.AvoidDuplicateLiterals") public final class MetadataRegenerator { - /** - * Date format for Maven metadata lastUpdated field. - */ - private static final DateTimeFormatter MAVEN_TS = - DateTimeFormatter.ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC); - /** * Pattern to extract Composer dev suffix identifiers. */ @@ -310,13 +303,14 @@ private static TreeSet collectMavenVersions( try { new Version(firstSegment); versions.add(firstSegment); - } catch (final Exception ignored) { + } catch (final Exception ex) { // Not a valid version, skip it EcsLogger.debug("com.artipie.importer") .message("Skipping non-version directory") .eventCategory("repository") .eventAction("maven_metadata_regenerate") .field("file.directory", firstSegment) + .error(ex) .log(); } } @@ -360,7 +354,7 @@ private CompletionStage writeMavenMetadata( dirs.add("versions"); versions.forEach(version -> dirs.add("version").set(version).up()); dirs.up() // versions - .add("lastUpdated").set(MAVEN_TS.format(Instant.now())).up() + .add("lastUpdated").set(MavenTimestamp.now()).up() .up() // versioning .up(); // metadata final String metadata; diff --git a/artipie-main/src/main/java/com/artipie/index/DbArtifactIndex.java b/artipie-main/src/main/java/com/artipie/index/DbArtifactIndex.java new file mode 100644 index 000000000..90662e2b1 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/index/DbArtifactIndex.java @@ -0,0 +1,698 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.index; + +import com.artipie.http.log.EcsLogger; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * PostgreSQL-backed implementation of {@link ArtifactIndex}. + * Uses JDBC queries against the existing {@code artifacts} table. + *

+ * This implementation is always "warmed up" since the database is the + * authoritative source and is always consistent. No warmup scan is needed. + * + * @since 1.20.13 + */ +@SuppressWarnings({"PMD.TooManyMethods", "PMD.AvoidCatchingGenericException"}) +public final class DbArtifactIndex implements ArtifactIndex { + + /** + * UPSERT SQL — same pattern as DbConsumer. + */ + private static final String UPSERT_SQL = String.join( + " ", + "INSERT INTO artifacts", + "(repo_type, repo_name, name, version, size, created_date, release_date, owner, path_prefix)", + "VALUES (?,?,?,?,?,?,?,?,?)", + "ON CONFLICT (repo_name, name, version)", + "DO UPDATE SET repo_type = EXCLUDED.repo_type, size = EXCLUDED.size,", + "created_date = EXCLUDED.created_date, release_date = EXCLUDED.release_date,", + "owner = EXCLUDED.owner, path_prefix = COALESCE(EXCLUDED.path_prefix, artifacts.path_prefix)" + ); + + /** + * DELETE by repo and name. + */ + private static final String DELETE_SQL = + "DELETE FROM artifacts WHERE repo_name = ? AND name = ?"; + + /** + * Full-text search SQL using tsvector/GIN index with relevance ranking. + * Falls back to LIKE if tsvector returns no results. + */ + private static final String FTS_SEARCH_SQL = String.join( + " ", + "SELECT repo_type, repo_name, name, version, size, created_date, owner,", + "ts_rank(search_tokens, plainto_tsquery('simple', ?)) AS rank", + "FROM artifacts WHERE search_tokens @@ plainto_tsquery('simple', ?)", + "ORDER BY rank DESC, name, version LIMIT ? OFFSET ?" + ); + + /** + * Full-text count SQL using tsvector. + */ + private static final String FTS_COUNT_SQL = + "SELECT COUNT(*) FROM artifacts WHERE search_tokens @@ plainto_tsquery('simple', ?)"; + + /** + * Prefix-matching FTS SQL using to_tsquery with ':*' suffix. + * Matches words starting with query terms: "test" matches "test", "testing", etc. + */ + private static final String PREFIX_FTS_SEARCH_SQL = String.join( + " ", + "SELECT repo_type, repo_name, name, version, size, created_date, owner,", + "ts_rank(search_tokens, to_tsquery('simple', ?)) AS rank", + "FROM artifacts WHERE search_tokens @@ to_tsquery('simple', ?)", + "ORDER BY rank DESC, name, version LIMIT ? OFFSET ?" + ); + + /** + * Prefix-matching FTS count SQL. + */ + private static final String PREFIX_FTS_COUNT_SQL = + "SELECT COUNT(*) FROM artifacts WHERE search_tokens @@ to_tsquery('simple', ?)"; + + /** + * Fallback search SQL with LIKE (used when tsvector is unavailable or returns 0 results). + */ + private static final String LIKE_SEARCH_SQL = String.join( + " ", + "SELECT repo_type, repo_name, name, version, size, created_date, owner", + "FROM artifacts WHERE LOWER(name) LIKE LOWER(?)", + "ORDER BY name, version LIMIT ? OFFSET ?" + ); + + /** + * Fallback count SQL with LIKE. + */ + private static final String LIKE_COUNT_SQL = + "SELECT COUNT(*) FROM artifacts WHERE LOWER(name) LIKE LOWER(?)"; + + /** + * Locate SQL suffix — exact name match for locally published artifacts. + * The full query is built dynamically by {@link #buildLocateSql(int)} + * to include an IN clause with path prefix candidates. + */ + private static final String LOCATE_NAME_CLAUSE = " OR name = ?"; + + /** + * Locate SQL prefix — finds repos by matching decomposed path prefixes. + * Uses IN (?, ?, ...) for index-friendly equality lookups instead of + * reverse LIKE which forces a full table scan. + */ + private static final String LOCATE_PREFIX = + "SELECT DISTINCT repo_name FROM artifacts WHERE path_prefix IN ("; + + /** + * Locate by name SQL — fast indexed lookup on the {@code name} column. + * Uses idx_artifacts_locate (name, repo_name) for O(log n) performance. + */ + private static final String LOCATE_BY_NAME_SQL = + "SELECT DISTINCT repo_name FROM artifacts WHERE name = ?"; + + /** + * Total count SQL. + */ + private static final String TOTAL_COUNT_SQL = "SELECT COUNT(*) FROM artifacts"; + + /** + * Thread counter for executor threads. + */ + private static final AtomicInteger THREAD_COUNTER = new AtomicInteger(0); + + /** + * JDBC DataSource. + */ + private final DataSource source; + + /** + * Executor for async operations. + */ + private final ExecutorService executor; + + /** + * Whether the executor was created internally (and should be shut down on close). + */ + private final boolean ownedExecutor; + + /** + * Constructor with default executor. + * Creates a fixed thread pool sized to available processors. + * + * @param source JDBC DataSource + */ + public DbArtifactIndex(final DataSource source) { + this( + source, + Executors.newFixedThreadPool( + Math.max(2, Runtime.getRuntime().availableProcessors()), + r -> { + final Thread thread = new Thread( + r, + "db-artifact-index-" + THREAD_COUNTER.incrementAndGet() + ); + thread.setDaemon(true); + return thread; + } + ), + true + ); + } + + /** + * Constructor with explicit executor. + * + * @param source JDBC DataSource + * @param executor Executor for async operations + */ + public DbArtifactIndex(final DataSource source, final ExecutorService executor) { + this(source, executor, false); + } + + /** + * Internal constructor. + * + * @param source JDBC DataSource + * @param executor Executor for async operations + * @param ownedExecutor Whether the executor is owned by this instance + */ + private DbArtifactIndex( + final DataSource source, + final ExecutorService executor, + final boolean ownedExecutor + ) { + this.source = Objects.requireNonNull(source, "DataSource must not be null"); + this.executor = Objects.requireNonNull(executor, "ExecutorService must not be null"); + this.ownedExecutor = ownedExecutor; + } + + @Override + public CompletableFuture index(final ArtifactDocument doc) { + return CompletableFuture.runAsync(() -> { + try (Connection conn = this.source.getConnection(); + PreparedStatement stmt = conn.prepareStatement(UPSERT_SQL)) { + setUpsertParams(stmt, doc); + stmt.executeUpdate(); + } catch (final SQLException ex) { + EcsLogger.error("com.artipie.index") + .message("Failed to index artifact") + .eventCategory("index") + .eventAction("db_index") + .eventOutcome("failure") + .field("package.name", doc.artifactPath()) + .error(ex) + .log(); + throw new RuntimeException("Failed to index artifact: " + doc.artifactPath(), ex); + } + }, this.executor); + } + + @Override + public CompletableFuture remove(final String repoName, final String artifactPath) { + return CompletableFuture.runAsync(() -> { + try (Connection conn = this.source.getConnection(); + PreparedStatement stmt = conn.prepareStatement(DELETE_SQL)) { + stmt.setString(1, repoName); + stmt.setString(2, artifactPath); + stmt.executeUpdate(); + } catch (final SQLException ex) { + EcsLogger.error("com.artipie.index") + .message("Failed to remove artifact") + .eventCategory("index") + .eventAction("db_remove") + .eventOutcome("failure") + .field("repository.name", repoName) + .field("package.name", artifactPath) + .error(ex) + .log(); + throw new RuntimeException( + String.format("Failed to remove artifact %s from %s", artifactPath, repoName), + ex + ); + } + }, this.executor); + } + + @Override + public CompletableFuture search( + final String query, final int maxResults, final int offset + ) { + return CompletableFuture.supplyAsync(() -> { + // If query contains SQL wildcards, use LIKE directly + final boolean uselike = query.contains("%") || query.contains("_"); + if (uselike) { + return DbArtifactIndex.searchWithLike( + this.source, query, maxResults, offset + ); + } + // Use prefix-matching FTS: "test" → 'test:*' matches + // "test", "test.txt", "testing", etc. Uses GIN index + // for efficient search on large datasets (10M+ rows). + try { + final SearchResult ftsResult = DbArtifactIndex.searchWithPrefixFts( + this.source, query, maxResults, offset + ); + if (ftsResult.totalHits() == 0) { + // Fallback to exact-match FTS (handles phrases) + final SearchResult exact = DbArtifactIndex.searchWithFts( + this.source, query, maxResults, offset + ); + if (exact.totalHits() == 0) { + // Final fallback: LIKE search for special chars (@, /, -) + return DbArtifactIndex.searchWithLike( + this.source, "%" + query + "%", maxResults, offset + ); + } + return exact; + } + return ftsResult; + } catch (final SQLException ex) { + // Graceful degradation: if tsvector column doesn't exist or + // any FTS-related error occurs, fall back to LIKE + EcsLogger.warn("com.artipie.index") + .message("FTS search failed, falling back to LIKE: " + + ex.getMessage()) + .eventCategory("search") + .eventAction("db_fts_fallback") + .error(ex) + .log(); + return DbArtifactIndex.searchWithLike( + this.source, "%" + query + "%", maxResults, offset + ); + } + }, this.executor); + } + + /** + * Execute full-text search using tsvector/GIN index. + * + * @param source DataSource + * @param query Search query (plain text, not a pattern) + * @param maxResults Max results per page + * @param offset Pagination offset + * @return SearchResult with ranked results + * @throws SQLException On database error (caller should handle for fallback) + */ + private static SearchResult searchWithFts( + final DataSource source, final String query, + final int maxResults, final int offset + ) throws SQLException { + final long totalHits; + final List docs = new ArrayList<>(); + try (Connection conn = source.getConnection()) { + // Get total count using FTS + try (PreparedStatement countStmt = conn.prepareStatement(FTS_COUNT_SQL)) { + countStmt.setString(1, query); + try (ResultSet rs = countStmt.executeQuery()) { + rs.next(); + totalHits = rs.getLong(1); + } + } + // Get paginated results with relevance ranking + try (PreparedStatement stmt = conn.prepareStatement(FTS_SEARCH_SQL)) { + stmt.setString(1, query); + stmt.setString(2, query); + stmt.setInt(3, maxResults); + stmt.setInt(4, offset); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + docs.add(fromResultSet(rs)); + } + } + } + } + return new SearchResult(docs, totalHits, offset, null); + } + + /** + * Execute prefix-matching FTS: "test" becomes "test:*" tsquery, + * matching "test", "test.txt", "testing", etc. Uses GIN index. + * + * @param source DataSource + * @param query Raw user query + * @param maxResults Max results per page + * @param offset Pagination offset + * @return SearchResult with ranked results + * @throws SQLException On database error + */ + private static SearchResult searchWithPrefixFts( + final DataSource source, final String query, + final int maxResults, final int offset + ) throws SQLException { + final String tsquery = DbArtifactIndex.buildPrefixTsQuery(query); + if (tsquery.isEmpty()) { + return new SearchResult( + java.util.Collections.emptyList(), 0, offset, null + ); + } + final long totalHits; + final List docs = new ArrayList<>(); + try (Connection conn = source.getConnection()) { + try (PreparedStatement countStmt = + conn.prepareStatement(PREFIX_FTS_COUNT_SQL)) { + countStmt.setString(1, tsquery); + try (ResultSet rs = countStmt.executeQuery()) { + rs.next(); + totalHits = rs.getLong(1); + } + } + try (PreparedStatement stmt = + conn.prepareStatement(PREFIX_FTS_SEARCH_SQL)) { + stmt.setString(1, tsquery); + stmt.setString(2, tsquery); + stmt.setInt(3, maxResults); + stmt.setInt(4, offset); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + docs.add(fromResultSet(rs)); + } + } + } + } + return new SearchResult(docs, totalHits, offset, null); + } + + /** + * Build a prefix-matching tsquery from user input. + * Splits on whitespace and dots, appends ":*" to each term, + * joins with "&" for AND semantics. + * E.g. "test" → "test:*", "test txt" → "test:* & txt:*" + * + * @param query Raw user query + * @return tsquery string safe for to_tsquery('simple', ?) + */ + private static String buildPrefixTsQuery(final String query) { + final StringBuilder sb = new StringBuilder(); + for (final String word : query.trim().split("[\\s.@/\\-]+")) { + final String clean = word.replaceAll("[^a-zA-Z0-9_]", ""); + if (!clean.isEmpty()) { + if (sb.length() > 0) { + sb.append(" & "); + } + sb.append(clean).append(":*"); + } + } + return sb.toString(); + } + + /** + * Execute search using LIKE pattern matching (fallback). + * + * @param source DataSource + * @param pattern LIKE pattern (should include % wildcards) + * @param maxResults Max results per page + * @param offset Pagination offset + * @return SearchResult + */ + private static SearchResult searchWithLike( + final DataSource source, final String pattern, + final int maxResults, final int offset + ) { + final long totalHits; + final List docs = new ArrayList<>(); + try (Connection conn = source.getConnection()) { + // Get total count + try (PreparedStatement countStmt = conn.prepareStatement(LIKE_COUNT_SQL)) { + countStmt.setString(1, pattern); + try (ResultSet rs = countStmt.executeQuery()) { + rs.next(); + totalHits = rs.getLong(1); + } + } + // Get paginated results + try (PreparedStatement stmt = conn.prepareStatement(LIKE_SEARCH_SQL)) { + stmt.setString(1, pattern); + stmt.setInt(2, maxResults); + stmt.setInt(3, offset); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + docs.add(fromResultSet(rs)); + } + } + } + } catch (final SQLException ex) { + EcsLogger.error("com.artipie.index") + .message("LIKE search failed for pattern: " + pattern) + .eventCategory("search") + .eventAction("db_search_like") + .eventOutcome("failure") + .error(ex) + .log(); + return SearchResult.EMPTY; + } + return new SearchResult(docs, totalHits, offset, null); + } + + @Override + public CompletableFuture> locate(final String artifactPath) { + return CompletableFuture.supplyAsync(() -> { + final List prefixes = pathPrefixes(artifactPath); + final String sql = buildLocateSql(prefixes.size()); + final List repos = new ArrayList<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql)) { + int idx = 1; + for (final String prefix : prefixes) { + stmt.setString(idx++, prefix); + } + stmt.setString(idx, artifactPath); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + repos.add(rs.getString("repo_name")); + } + } + } catch (final SQLException ex) { + EcsLogger.error("com.artipie.index") + .message("Locate failed for path: " + artifactPath) + .eventCategory("search") + .eventAction("db_locate") + .eventOutcome("failure") + .error(ex) + .log(); + return List.of(); + } + return repos; + }, this.executor); + } + + @Override + public CompletableFuture> locateByName(final String artifactName) { + return CompletableFuture.supplyAsync(() -> { + final List repos = new ArrayList<>(); + try (Connection conn = this.source.getConnection(); + PreparedStatement stmt = conn.prepareStatement(LOCATE_BY_NAME_SQL)) { + stmt.setString(1, artifactName); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + repos.add(rs.getString("repo_name")); + } + } + } catch (final SQLException ex) { + EcsLogger.error("com.artipie.index") + .message("LocateByName failed for: " + artifactName) + .eventCategory("search") + .eventAction("db_locate_by_name") + .eventOutcome("failure") + .error(ex) + .log(); + return List.of(); + } + return repos; + }, this.executor); + } + + /** + * Decompose a path into all possible prefix candidates. + * E.g. "com/google/guava/31.1/guava.jar" produces: + * ["com", "com/google", "com/google/guava", "com/google/guava/31.1"] + * (the full path itself is excluded since path_prefix must be a proper prefix). + * + * @param path Artifact path + * @return List of prefix candidates (never empty — contains at least the full path) + */ + static List pathPrefixes(final String path) { + final String clean = path.startsWith("/") ? path.substring(1) : path; + final String[] segments = clean.split("/"); + final List prefixes = new ArrayList<>(segments.length); + final StringBuilder buf = new StringBuilder(clean.length()); + for (int i = 0; i < segments.length - 1; i++) { + if (i > 0) { + buf.append('/'); + } + buf.append(segments[i]); + prefixes.add(buf.toString()); + } + if (prefixes.isEmpty()) { + prefixes.add(clean); + } + return prefixes; + } + + /** + * Build locate SQL with the right number of IN placeholders. + * Result: SELECT DISTINCT repo_name FROM artifacts + * WHERE path_prefix IN (?,?,...) OR name = ? + * + * @param prefixCount Number of prefix candidates + * @return SQL string + */ + private static String buildLocateSql(final int prefixCount) { + final StringBuilder sql = new StringBuilder(LOCATE_PREFIX); + for (int i = 0; i < prefixCount; i++) { + if (i > 0) { + sql.append(','); + } + sql.append('?'); + } + sql.append(')'); + sql.append(LOCATE_NAME_CLAUSE); + return sql.toString(); + } + + @Override + public boolean isWarmedUp() { + return true; + } + + @Override + public void setWarmedUp() { + // No-op: DB-backed index is always consistent + } + + @Override + public CompletableFuture> getStats() { + return CompletableFuture.supplyAsync(() -> { + final Map stats = new HashMap<>(3); + try (Connection conn = this.source.getConnection(); + PreparedStatement stmt = conn.prepareStatement(TOTAL_COUNT_SQL); + ResultSet rs = stmt.executeQuery()) { + rs.next(); + stats.put("documents", rs.getLong(1)); + } catch (final SQLException ex) { + EcsLogger.error("com.artipie.index") + .message("Failed to get index stats") + .eventCategory("index") + .eventAction("db_stats") + .eventOutcome("failure") + .error(ex) + .log(); + stats.put("documents", -1L); + } + stats.put("warmedUp", true); + stats.put("type", "postgresql"); + stats.put("searchEngine", "tsvector/GIN"); + return stats; + }, this.executor); + } + + @Override + public CompletableFuture indexBatch(final Collection docs) { + if (docs.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + return CompletableFuture.runAsync(() -> { + try (Connection conn = this.source.getConnection(); + PreparedStatement stmt = conn.prepareStatement(UPSERT_SQL)) { + conn.setAutoCommit(false); + for (final ArtifactDocument doc : docs) { + setUpsertParams(stmt, doc); + stmt.addBatch(); + } + stmt.executeBatch(); + conn.commit(); + } catch (final SQLException ex) { + EcsLogger.error("com.artipie.index") + .message("Failed to batch index " + docs.size() + " artifacts") + .eventCategory("index") + .eventAction("db_index_batch") + .eventOutcome("failure") + .error(ex) + .log(); + throw new RuntimeException( + "Failed to batch index " + docs.size() + " artifacts", ex + ); + } + }, this.executor); + } + + @Override + public void close() { + if (this.ownedExecutor) { + this.executor.shutdown(); + try { + if (!this.executor.awaitTermination(5, TimeUnit.SECONDS)) { + this.executor.shutdownNow(); + } + } catch (final InterruptedException ex) { + this.executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + } + + /** + * Set UPSERT prepared statement parameters from an ArtifactDocument. + * + * @param stmt Prepared statement + * @param doc Artifact document + * @throws SQLException On SQL error + */ + private static void setUpsertParams( + final PreparedStatement stmt, final ArtifactDocument doc + ) throws SQLException { + stmt.setString(1, doc.repoType()); + stmt.setString(2, doc.repoName()); + stmt.setString(3, doc.artifactPath()); + stmt.setString(4, doc.version() != null ? doc.version() : ""); + stmt.setLong(5, doc.size()); + final long createdEpoch = doc.createdAt() != null + ? doc.createdAt().toEpochMilli() + : System.currentTimeMillis(); + stmt.setLong(6, createdEpoch); + stmt.setLong(7, createdEpoch); + stmt.setString(8, doc.owner() != null ? doc.owner() : ""); + stmt.setString(9, null); // path_prefix populated by DbConsumer from ArtifactEvent + } + + /** + * Convert a ResultSet row to an ArtifactDocument. + * + * @param rs ResultSet positioned at a row + * @return ArtifactDocument + * @throws SQLException On SQL error + */ + private static ArtifactDocument fromResultSet(final ResultSet rs) throws SQLException { + final String name = rs.getString("name"); + final long createdDate = rs.getLong("created_date"); + return new ArtifactDocument( + rs.getString("repo_type"), + rs.getString("repo_name"), + name, + name, + rs.getString("version"), + rs.getLong("size"), + Instant.ofEpochMilli(createdDate), + rs.getString("owner") + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/jetty/http3/Http3Server.java b/artipie-main/src/main/java/com/artipie/jetty/http3/Http3Server.java index 9cd167afc..b9998732f 100644 --- a/artipie-main/src/main/java/com/artipie/jetty/http3/Http3Server.java +++ b/artipie-main/src/main/java/com/artipie/jetty/http3/Http3Server.java @@ -85,6 +85,7 @@ public void start() { try { // Create PEM directory for QUIC native library (required by Quiche) final Path pemDir = Files.createTempDirectory("http3-pem"); + pemDir.toFile().deleteOnExit(); // Configure QUIC with Quiche native library final QuicheServerQuicConfiguration serverQuicConfig = diff --git a/artipie-main/src/main/java/com/artipie/metrics/AsyncMetricsVerticle.java b/artipie-main/src/main/java/com/artipie/metrics/AsyncMetricsVerticle.java index 4f3d18a81..4300bb037 100644 --- a/artipie-main/src/main/java/com/artipie/metrics/AsyncMetricsVerticle.java +++ b/artipie-main/src/main/java/com/artipie/metrics/AsyncMetricsVerticle.java @@ -155,13 +155,12 @@ public void start(final Promise startPromise) { this.server.listen(ar -> { if (ar.succeeded()) { EcsLogger.info("com.artipie.metrics.AsyncMetricsVerticle") - .message("Async metrics server started") + .message(String.format("Async metrics server started with cache TTL %dms", this.cacheTtlMs)) .eventCategory("configuration") .eventAction("metrics_server_start") .eventOutcome("success") .field("destination.port", this.port) .field("url.path", this.path) - .field("cache.ttl.ms", this.cacheTtlMs) .log(); startPromise.complete(); } else { @@ -263,7 +262,7 @@ private void handleMetricsRequest(final HttpServerRequest request) { promise.fail(e); } } - }, false, ar -> { // false = don't order results, allow concurrent execution + }, true, ar -> { // true = serialize scrapes to prevent cache overwrite race if (ar.succeeded()) { respondWithMetrics(request, ar.result().toString(), false, System.currentTimeMillis() - startTime); @@ -333,12 +332,11 @@ private String doScrape() { // Log slow scrapes (> 1 second) if (scrapeDuration > 1000) { EcsLogger.warn("com.artipie.metrics.AsyncMetricsVerticle") - .message("Slow metrics scrape detected") + .message(String.format("Slow metrics scrape detected: %d bytes", result.getBytes(StandardCharsets.UTF_8).length)) .eventCategory("metrics") .eventAction("scrape") .eventOutcome("slow") .field("event.duration", scrapeDuration) - .field("metrics.size.bytes", result.getBytes(StandardCharsets.UTF_8).length) .log(); } diff --git a/artipie-main/src/main/java/com/artipie/micrometer/MicrometerSlice.java b/artipie-main/src/main/java/com/artipie/micrometer/MicrometerSlice.java index 21fc2c920..24a4b11c3 100644 --- a/artipie-main/src/main/java/com/artipie/micrometer/MicrometerSlice.java +++ b/artipie-main/src/main/java/com/artipie/micrometer/MicrometerSlice.java @@ -9,6 +9,7 @@ import com.artipie.http.ResponseBuilder; import com.artipie.http.Response; import com.artipie.http.Slice; +import com.artipie.http.log.EcsLogger; import com.artipie.http.rq.RequestLine; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -101,8 +102,11 @@ public CompletableFuture response(final RequestLine line, final Header .ifPresent(contentLength -> { try { responseBody.record(Long.parseLong(contentLength)); - } catch (NumberFormatException ignored) { - // Skip if Content-Length is invalid + } catch (final NumberFormatException ex) { + EcsLogger.debug("com.artipie.metrics") + .message("Invalid Content-Length header value") + .error(ex) + .log(); } }); // Pass response through unchanged - no body wrapping diff --git a/artipie-main/src/main/java/com/artipie/proxy/OfflineAwareSlice.java b/artipie-main/src/main/java/com/artipie/proxy/OfflineAwareSlice.java new file mode 100644 index 000000000..e759c0a9b --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/proxy/OfflineAwareSlice.java @@ -0,0 +1,81 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.proxy; + +import com.artipie.asto.Content; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.Slice; +import com.artipie.http.rq.RequestLine; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Offline-aware slice wrapper. + * When offline mode is enabled, returns 503 for requests that would hit upstream, + * serving only from local cache. + * + * @since 1.20.13 + */ +public final class OfflineAwareSlice implements Slice { + + /** + * Wrapped proxy slice. + */ + private final Slice origin; + + /** + * Offline flag. + */ + private final AtomicBoolean offline; + + /** + * Ctor. + * @param origin Wrapped slice + */ + public OfflineAwareSlice(final Slice origin) { + this.origin = Objects.requireNonNull(origin, "origin"); + this.offline = new AtomicBoolean(false); + } + + @Override + public CompletableFuture response( + final RequestLine line, final Headers headers, final Content body + ) { + if (this.offline.get()) { + return CompletableFuture.completedFuture( + ResponseBuilder.unavailable() + .textBody("Repository is in offline mode") + .build() + ); + } + return this.origin.response(line, headers, body); + } + + /** + * Enable offline mode. + */ + public void goOffline() { + this.offline.set(true); + } + + /** + * Disable offline mode. + */ + public void goOnline() { + this.offline.set(false); + } + + /** + * Check if offline mode is enabled. + * @return True if offline + */ + public boolean isOffline() { + return this.offline.get(); + } +} diff --git a/artipie-main/src/main/java/com/artipie/scheduling/ArtipieQuartzConnectionProvider.java b/artipie-main/src/main/java/com/artipie/scheduling/ArtipieQuartzConnectionProvider.java new file mode 100644 index 000000000..bc9ad0e04 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/scheduling/ArtipieQuartzConnectionProvider.java @@ -0,0 +1,60 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.scheduling; + +import java.sql.Connection; +import java.sql.SQLException; +import javax.sql.DataSource; +import org.quartz.utils.ConnectionProvider; + +/** + * Quartz {@link ConnectionProvider} backed by an existing {@link DataSource}. + *

+ * Allows Quartz JDBC job store to reuse the same connection pool (HikariCP) + * that Artipie uses for its artifacts database, eliminating the need for + * Quartz to manage its own connection pool. + *

+ * This provider is registered programmatically via + * {@link org.quartz.utils.DBConnectionManager#addConnectionProvider(String, ConnectionProvider)} + * before the Quartz scheduler is created. + * + * @since 1.20.13 + */ +public final class ArtipieQuartzConnectionProvider implements ConnectionProvider { + + /** + * The data source name used in Quartz configuration properties. + * Must match the value of {@code org.quartz.jobStore.dataSource}. + */ + public static final String DS_NAME = "artipieDS"; + + /** + * Underlying data source (typically HikariCP). + */ + private final DataSource dataSource; + + /** + * Ctor. + * @param dataSource Existing data source to delegate to + */ + public ArtipieQuartzConnectionProvider(final DataSource dataSource) { + this.dataSource = dataSource; + } + + @Override + public Connection getConnection() throws SQLException { + return this.dataSource.getConnection(); + } + + @Override + public void shutdown() throws SQLException { + // HikariCP manages its own lifecycle; nothing to do here. + } + + @Override + public void initialize() throws SQLException { + // Already initialized via Artipie's HikariCP setup. + } +} diff --git a/artipie-main/src/main/java/com/artipie/scheduling/MetadataEventQueues.java b/artipie-main/src/main/java/com/artipie/scheduling/MetadataEventQueues.java index 8d8f220dd..736633d52 100644 --- a/artipie-main/src/main/java/com/artipie/scheduling/MetadataEventQueues.java +++ b/artipie-main/src/main/java/com/artipie/scheduling/MetadataEventQueues.java @@ -6,13 +6,15 @@ import com.artipie.ArtipieException; import com.artipie.goproxy.GoProxyPackageProcessor; -import com.artipie.gradle.GradleProxyPackageProcessor; + import com.artipie.maven.MavenProxyPackageProcessor; import com.artipie.npm.events.NpmProxyPackageProcessor; import com.artipie.pypi.PyProxyPackageProcessor; import com.artipie.composer.http.proxy.ComposerProxyPackageProcessor; import com.artipie.settings.repo.RepoConfig; import com.artipie.http.log.EcsLogger; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -20,6 +22,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.LinkedBlockingQueue; import org.quartz.JobDataMap; import org.quartz.JobKey; import org.quartz.SchedulerException; @@ -66,6 +69,11 @@ public final class MetadataEventQueues { */ private final QuartzService quartz; + /** + * Optional meter registry for metrics. + */ + private final Optional registry; + /** * Ctor. * @@ -74,11 +82,32 @@ public final class MetadataEventQueues { */ public MetadataEventQueues( final Queue queue, final QuartzService quartz + ) { + this(queue, quartz, Optional.empty()); + } + + /** + * Ctor. + * + * @param queue Artifact events queue + * @param quartz Quartz service + * @param registry Optional meter registry for queue depth metrics + */ + public MetadataEventQueues( + final Queue queue, final QuartzService quartz, + final Optional registry ) { this.queue = queue; this.queues = new ConcurrentHashMap<>(); this.quartz = quartz; this.keys = new ConcurrentHashMap<>(); + this.registry = registry; + this.registry.ifPresent( + reg -> Gauge.builder("artipie.events.queue.size", queue, Queue::size) + .tag("type", "events") + .description("Size of the artifact events queue") + .register(reg) + ); } /** @@ -91,6 +120,15 @@ public Queue eventQueue() { /** * Obtain queue for proxy adapter repository. + *

+ * Thread-safety note: concurrent calls for the same config.name() are safe because + * {@link ConcurrentHashMap#computeIfAbsent} guarantees the mapping function executes + * exactly once per key. The initial {@code this.queues.get()} check is a fast-path + * optimization; if two threads both see null, both enter the if-block, but only one + * thread's lambda will execute inside computeIfAbsent. The other thread receives the + * already-created queue. The {@code this.keys.put()} call inside the lambda also + * executes exactly once per key, so no duplicate jobs are scheduled. + *

* @param config Repository config * @return Queue for proxy events */ @@ -103,14 +141,31 @@ public Optional> proxyEventQueues(final RepoConfig con final Queue events = this.queues.computeIfAbsent( config.name(), key -> { - final Queue res = new ConcurrentLinkedQueue<>(); + final Queue res = + new LinkedBlockingQueue<>(10_000); final JobDataMap data = new JobDataMap(); - data.put("packages", res); - data.put("storage", config.storage()); - data.put("events", this.queue); final ProxyRepoType type = ProxyRepoType.type(config.type()); - if (type == ProxyRepoType.NPM_PROXY) { - data.put(MetadataEventQueues.HOST, artipieHost(config)); + if (this.quartz.isClustered()) { + final String prefix = config.name() + "-proxy-"; + final String pkgKey = prefix + "packages"; + final String stoKey = prefix + "storage"; + final String evtKey = prefix + "events"; + JobDataRegistry.register(pkgKey, res); + JobDataRegistry.register(stoKey, config.storage()); + JobDataRegistry.register(evtKey, this.queue); + data.put("packages_key", pkgKey); + data.put("storage_key", stoKey); + data.put("events_key", evtKey); + if (type == ProxyRepoType.NPM_PROXY) { + data.put(MetadataEventQueues.HOST, artipieHost(config)); + } + } else { + data.put("packages", res); + data.put("storage", config.storage()); + data.put("events", this.queue); + if (type == ProxyRepoType.NPM_PROXY) { + data.put(MetadataEventQueues.HOST, artipieHost(config)); + } } final int threads = Math.max(1, settingsIntValue(config, "threads_count")); final int interval = Math.max( @@ -131,6 +186,13 @@ public Optional> proxyEventQueues(final RepoConfig con } catch (final SchedulerException err) { throw new ArtipieException(err); } + this.registry.ifPresent( + reg -> Gauge.builder( + "artipie.proxy.queue.size", res, Queue::size + ).tag("repo", config.name()) + .description("Size of proxy artifact event queue") + .register(reg) + ); return res; } ); @@ -213,7 +275,7 @@ Class job() { GRADLE_PROXY { @Override Class job() { - return GradleProxyPackageProcessor.class; + return MavenProxyPackageProcessor.class; } }, diff --git a/artipie-main/src/main/java/com/artipie/scheduling/QuartzSchema.java b/artipie-main/src/main/java/com/artipie/scheduling/QuartzSchema.java new file mode 100644 index 000000000..6fd4bbf93 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/scheduling/QuartzSchema.java @@ -0,0 +1,319 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.scheduling; + +import com.artipie.ArtipieException; +import com.artipie.http.log.EcsLogger; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import javax.sql.DataSource; + +/** + * Creates the Quartz JDBC job store schema (QRTZ_* tables) in PostgreSQL. + *

+ * Uses {@code CREATE TABLE IF NOT EXISTS} so it is safe to call on every + * startup. The DDL matches the official Quartz 2.3.x {@code tables_postgres.sql} + * shipped inside the {@code quartz-2.3.2.jar}. + * + * @since 1.20.13 + */ +public final class QuartzSchema { + + /** + * Data source to create the schema in. + */ + private final DataSource dataSource; + + /** + * Ctor. + * @param dataSource Data source for the target PostgreSQL database + */ + public QuartzSchema(final DataSource dataSource) { + this.dataSource = dataSource; + } + + /** + * Create all QRTZ_* tables and indexes if they do not already exist. + * @throws ArtipieException If DDL execution fails + */ + public void create() { + try (Connection conn = this.dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + QuartzSchema.createTables(stmt); + QuartzSchema.createIndexes(stmt); + EcsLogger.info("com.artipie.scheduling") + .message("Quartz JDBC schema created or verified") + .eventCategory("scheduling") + .eventAction("schema_create") + .eventOutcome("success") + .log(); + } catch (final SQLException error) { + throw new ArtipieException( + "Failed to create Quartz JDBC schema", error + ); + } + } + + /** + * Execute all CREATE TABLE IF NOT EXISTS statements. + * Order matters because of foreign key references. + * @param stmt JDBC statement + * @throws SQLException On SQL error + */ + @SuppressWarnings("PMD.ExcessiveMethodLength") + private static void createTables(final Statement stmt) throws SQLException { + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_JOB_DETAILS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " JOB_NAME VARCHAR(200) NOT NULL,", + " JOB_GROUP VARCHAR(200) NOT NULL,", + " DESCRIPTION VARCHAR(250) NULL,", + " JOB_CLASS_NAME VARCHAR(250) NOT NULL,", + " IS_DURABLE BOOL NOT NULL,", + " IS_NONCONCURRENT BOOL NOT NULL,", + " IS_UPDATE_DATA BOOL NOT NULL,", + " REQUESTS_RECOVERY BOOL NOT NULL,", + " JOB_DATA BYTEA NULL,", + " PRIMARY KEY (SCHED_NAME, JOB_NAME, JOB_GROUP)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_TRIGGERS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " TRIGGER_NAME VARCHAR(200) NOT NULL,", + " TRIGGER_GROUP VARCHAR(200) NOT NULL,", + " JOB_NAME VARCHAR(200) NOT NULL,", + " JOB_GROUP VARCHAR(200) NOT NULL,", + " DESCRIPTION VARCHAR(250) NULL,", + " NEXT_FIRE_TIME BIGINT NULL,", + " PREV_FIRE_TIME BIGINT NULL,", + " PRIORITY INTEGER NULL,", + " TRIGGER_STATE VARCHAR(16) NOT NULL,", + " TRIGGER_TYPE VARCHAR(8) NOT NULL,", + " START_TIME BIGINT NOT NULL,", + " END_TIME BIGINT NULL,", + " CALENDAR_NAME VARCHAR(200) NULL,", + " MISFIRE_INSTR SMALLINT NULL,", + " JOB_DATA BYTEA NULL,", + " PRIMARY KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP),", + " FOREIGN KEY (SCHED_NAME, JOB_NAME, JOB_GROUP)", + " REFERENCES QRTZ_JOB_DETAILS (SCHED_NAME, JOB_NAME, JOB_GROUP)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_SIMPLE_TRIGGERS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " TRIGGER_NAME VARCHAR(200) NOT NULL,", + " TRIGGER_GROUP VARCHAR(200) NOT NULL,", + " REPEAT_COUNT BIGINT NOT NULL,", + " REPEAT_INTERVAL BIGINT NOT NULL,", + " TIMES_TRIGGERED BIGINT NOT NULL,", + " PRIMARY KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP),", + " FOREIGN KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + " REFERENCES QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_CRON_TRIGGERS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " TRIGGER_NAME VARCHAR(200) NOT NULL,", + " TRIGGER_GROUP VARCHAR(200) NOT NULL,", + " CRON_EXPRESSION VARCHAR(120) NOT NULL,", + " TIME_ZONE_ID VARCHAR(80),", + " PRIMARY KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP),", + " FOREIGN KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + " REFERENCES QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_SIMPROP_TRIGGERS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " TRIGGER_NAME VARCHAR(200) NOT NULL,", + " TRIGGER_GROUP VARCHAR(200) NOT NULL,", + " STR_PROP_1 VARCHAR(512) NULL,", + " STR_PROP_2 VARCHAR(512) NULL,", + " STR_PROP_3 VARCHAR(512) NULL,", + " INT_PROP_1 INT NULL,", + " INT_PROP_2 INT NULL,", + " LONG_PROP_1 BIGINT NULL,", + " LONG_PROP_2 BIGINT NULL,", + " DEC_PROP_1 NUMERIC(13, 4) NULL,", + " DEC_PROP_2 NUMERIC(13, 4) NULL,", + " BOOL_PROP_1 BOOL NULL,", + " BOOL_PROP_2 BOOL NULL,", + " PRIMARY KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP),", + " FOREIGN KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + " REFERENCES QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_BLOB_TRIGGERS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " TRIGGER_NAME VARCHAR(200) NOT NULL,", + " TRIGGER_GROUP VARCHAR(200) NOT NULL,", + " BLOB_DATA BYTEA NULL,", + " PRIMARY KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP),", + " FOREIGN KEY (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + " REFERENCES QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_CALENDARS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " CALENDAR_NAME VARCHAR(200) NOT NULL,", + " CALENDAR BYTEA NOT NULL,", + " PRIMARY KEY (SCHED_NAME, CALENDAR_NAME)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_PAUSED_TRIGGER_GRPS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " TRIGGER_GROUP VARCHAR(200) NOT NULL,", + " PRIMARY KEY (SCHED_NAME, TRIGGER_GROUP)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_FIRED_TRIGGERS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " ENTRY_ID VARCHAR(95) NOT NULL,", + " TRIGGER_NAME VARCHAR(200) NOT NULL,", + " TRIGGER_GROUP VARCHAR(200) NOT NULL,", + " INSTANCE_NAME VARCHAR(200) NOT NULL,", + " FIRED_TIME BIGINT NOT NULL,", + " SCHED_TIME BIGINT NOT NULL,", + " PRIORITY INTEGER NOT NULL,", + " STATE VARCHAR(16) NOT NULL,", + " JOB_NAME VARCHAR(200) NULL,", + " JOB_GROUP VARCHAR(200) NULL,", + " IS_NONCONCURRENT BOOL NULL,", + " REQUESTS_RECOVERY BOOL NULL,", + " PRIMARY KEY (SCHED_NAME, ENTRY_ID)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_SCHEDULER_STATE (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " INSTANCE_NAME VARCHAR(200) NOT NULL,", + " LAST_CHECKIN_TIME BIGINT NOT NULL,", + " CHECKIN_INTERVAL BIGINT NOT NULL,", + " PRIMARY KEY (SCHED_NAME, INSTANCE_NAME)", + ")" + ) + ); + stmt.executeUpdate( + String.join( + "\n", + "CREATE TABLE IF NOT EXISTS QRTZ_LOCKS (", + " SCHED_NAME VARCHAR(120) NOT NULL,", + " LOCK_NAME VARCHAR(40) NOT NULL,", + " PRIMARY KEY (SCHED_NAME, LOCK_NAME)", + ")" + ) + ); + } + + /** + * Create performance indexes. Uses CREATE INDEX IF NOT EXISTS so + * the call is idempotent. + * @param stmt JDBC statement + * @throws SQLException On SQL error + */ + @SuppressWarnings("PMD.ExcessiveMethodLength") + private static void createIndexes(final Statement stmt) throws SQLException { + // Indexes on QRTZ_JOB_DETAILS + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_J_REQ_RECOVERY ON QRTZ_JOB_DETAILS (SCHED_NAME, REQUESTS_RECOVERY)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_J_GRP ON QRTZ_JOB_DETAILS (SCHED_NAME, JOB_GROUP)" + ); + // Indexes on QRTZ_TRIGGERS + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_J ON QRTZ_TRIGGERS (SCHED_NAME, JOB_NAME, JOB_GROUP)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_JG ON QRTZ_TRIGGERS (SCHED_NAME, JOB_GROUP)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_C ON QRTZ_TRIGGERS (SCHED_NAME, CALENDAR_NAME)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_G ON QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_GROUP)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_STATE ON QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_STATE)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_N_STATE ON QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP, TRIGGER_STATE)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_N_G_STATE ON QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_GROUP, TRIGGER_STATE)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_NEXT_FIRE_TIME ON QRTZ_TRIGGERS (SCHED_NAME, NEXT_FIRE_TIME)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_NFT_ST ON QRTZ_TRIGGERS (SCHED_NAME, TRIGGER_STATE, NEXT_FIRE_TIME)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_NFT_MISFIRE ON QRTZ_TRIGGERS (SCHED_NAME, MISFIRE_INSTR, NEXT_FIRE_TIME)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_NFT_ST_MISFIRE ON QRTZ_TRIGGERS (SCHED_NAME, MISFIRE_INSTR, NEXT_FIRE_TIME, TRIGGER_STATE)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_T_NFT_ST_MISFIRE_GRP ON QRTZ_TRIGGERS (SCHED_NAME, MISFIRE_INSTR, NEXT_FIRE_TIME, TRIGGER_GROUP, TRIGGER_STATE)" + ); + // Indexes on QRTZ_FIRED_TRIGGERS + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_FT_TRIG_INST_NAME ON QRTZ_FIRED_TRIGGERS (SCHED_NAME, INSTANCE_NAME)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_FT_INST_JOB_REQ_RCVRY ON QRTZ_FIRED_TRIGGERS (SCHED_NAME, INSTANCE_NAME, REQUESTS_RECOVERY)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_FT_J_G ON QRTZ_FIRED_TRIGGERS (SCHED_NAME, JOB_NAME, JOB_GROUP)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_FT_JG ON QRTZ_FIRED_TRIGGERS (SCHED_NAME, JOB_GROUP)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_FT_T_G ON QRTZ_FIRED_TRIGGERS (SCHED_NAME, TRIGGER_NAME, TRIGGER_GROUP)" + ); + stmt.executeUpdate( + "CREATE INDEX IF NOT EXISTS IDX_QRTZ_FT_TG ON QRTZ_FIRED_TRIGGERS (SCHED_NAME, TRIGGER_GROUP)" + ); + } +} diff --git a/artipie-main/src/main/java/com/artipie/scheduling/QuartzService.java b/artipie-main/src/main/java/com/artipie/scheduling/QuartzService.java index 8a017abba..932bbe8ca 100644 --- a/artipie-main/src/main/java/com/artipie/scheduling/QuartzService.java +++ b/artipie-main/src/main/java/com/artipie/scheduling/QuartzService.java @@ -9,11 +9,14 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Properties; import java.util.Queue; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import javax.sql.DataSource; import org.quartz.CronScheduleBuilder; import org.quartz.Job; import org.quartz.JobBuilder; @@ -27,48 +30,125 @@ import org.quartz.Trigger; import org.quartz.TriggerBuilder; import org.quartz.impl.StdSchedulerFactory; +import org.quartz.utils.DBConnectionManager; /** - * Start quarts scheduling service. + * Quartz scheduling service. + *

+ * Supports two modes: + *

    + *
  • RAM mode (default, no-arg constructor) -- uses in-memory RAMJobStore. + * Suitable for single-instance deployments.
    • + *
  • JDBC mode (DataSource constructor) -- uses {@code JobStoreTX} backed by + * PostgreSQL. Enables Quartz clustering so multiple Artipie instances coordinate + * job execution through the database and avoid duplicate scheduling.
    • + *
+ * * @since 1.3 */ public final class QuartzService { + /** + * Scheduler instance name shared across all clustered nodes. + */ + private static final String SCHED_NAME = "ArtipieScheduler"; + /** * Quartz scheduler. */ private final Scheduler scheduler; /** - * Ctor. + * Whether this service is backed by JDBC (clustered mode). + */ + private final boolean clustered; + + /** + * Flag to prevent double-shutdown of the Quartz scheduler. + * @since 1.20.13 + */ + private final AtomicBoolean stopped = new AtomicBoolean(false); + + /** + * Ctor for RAM-based (non-clustered) scheduler. + * Uses the default Quartz configuration with in-memory RAMJobStore. */ @SuppressWarnings("PMD.ConstructorOnlyInitializesOrCallOtherConstructors") public QuartzService() { try { this.scheduler = new StdSchedulerFactory().getScheduler(); - Runtime.getRuntime().addShutdownHook( - new Thread() { - @Override - public void run() { - try { - QuartzService.this.scheduler.shutdown(); - } catch (final SchedulerException error) { - EcsLogger.error("com.artipie.scheduling") - .message("Failed to shutdown Quartz scheduler") - .eventCategory("scheduling") - .eventAction("scheduler_shutdown") - .eventOutcome("failure") - .error(error) - .log(); - } - } - } + this.clustered = false; + this.addShutdownHook(); + } catch (final SchedulerException error) { + throw new ArtipieException(error); + } + } + + /** + * Ctor for JDBC-backed clustered scheduler. + *

+ * Creates the Quartz schema (QRTZ_* tables) if they do not exist, + * registers a {@link ArtipieQuartzConnectionProvider} wrapping the given + * DataSource, and configures Quartz to use {@code JobStoreTX} with + * PostgreSQL delegate and clustering enabled. + * + * @param dataSource PostgreSQL data source (typically HikariCP) + */ + @SuppressWarnings("PMD.ConstructorOnlyInitializesOrCallOtherConstructors") + public QuartzService(final DataSource dataSource) { + try { + // 1. Create QRTZ_* tables if they don't exist + new QuartzSchema(dataSource).create(); + // 2. Register our ConnectionProvider with Quartz's DBConnectionManager + DBConnectionManager.getInstance().addConnectionProvider( + ArtipieQuartzConnectionProvider.DS_NAME, + new ArtipieQuartzConnectionProvider(dataSource) ); + // 3. Build JDBC properties for Quartz + final Properties props = QuartzService.jdbcProperties(); + final StdSchedulerFactory factory = new StdSchedulerFactory(); + factory.initialize(props); + this.scheduler = factory.getScheduler(); + this.clustered = true; + // 4. Clear stale jobs from previous runs. In JDBC mode, jobs + // persist across restarts but their in-memory JobDataRegistry + // entries are lost. Old jobs would fire with null dependencies, + // fail, and loop indefinitely if not cleaned up. + this.scheduler.clear(); + this.addShutdownHook(); + EcsLogger.info("com.artipie.scheduling") + .message("Quartz JDBC clustering enabled (scheduler: " + + QuartzService.SCHED_NAME + ")") + .eventCategory("scheduling") + .eventAction("jdbc_cluster_init") + .eventOutcome("success") + .log(); } catch (final SchedulerException error) { throw new ArtipieException(error); } } + /** + * Returns whether this service is running in clustered JDBC mode. + * @return True if JDBC-backed clustering is enabled + */ + public boolean isClustered() { + return this.clustered; + } + + /** + * Checks whether the Quartz scheduler is running. + * @return True if started, not shutdown, and not in standby mode + */ + public boolean isRunning() { + try { + return this.scheduler.isStarted() && !this.scheduler.isShutdown() + && !this.scheduler.isInStandbyMode(); + } catch (final SchedulerException ex) { + return false; + } + } + /** * Adds event processor to the quarts job. The job is repeating forever every * given seconds. Jobs are run in parallel, if several consumers are passed, consumer for job. @@ -91,8 +171,19 @@ public Queue addPeriodicEventsProcessor( final int count = this.parallelJobs(consumer.size()); for (int item = 0; item < count; item = item + 1) { final JobDataMap data = new JobDataMap(); - data.put("elements", queue); - data.put("action", Objects.requireNonNull(consumer.get(item))); + if (this.clustered) { + final String queueKey = "elements-" + id; + final String actionKey = "action-" + id + "-" + item; + JobDataRegistry.register(queueKey, queue); + JobDataRegistry.register( + actionKey, Objects.requireNonNull(consumer.get(item)) + ); + data.put("elements_key", queueKey); + data.put("action_key", actionKey); + } else { + data.put("elements", queue); + data.put("action", Objects.requireNonNull(consumer.get(item))); + } this.scheduler.scheduleJob( JobBuilder.newJob(EventsProcessor.class).setJobData(data).withIdentity( QuartzService.jobId(id, item), EventsProcessor.class.getSimpleName() @@ -206,13 +297,93 @@ public void start() { * Stop scheduler. */ public void stop() { - try { - this.scheduler.shutdown(true); - } catch (final SchedulerException exc) { - throw new ArtipieException(exc); + if (this.stopped.compareAndSet(false, true)) { + try { + this.scheduler.shutdown(true); + } catch (final SchedulerException exc) { + throw new ArtipieException(exc); + } } } + /** + * Registers a JVM shutdown hook that gracefully shuts down the scheduler. + */ + private void addShutdownHook() { + Runtime.getRuntime().addShutdownHook( + new Thread() { + @Override + public void run() { + if (QuartzService.this.stopped.compareAndSet(false, true)) { + try { + QuartzService.this.scheduler.shutdown(); + } catch (final SchedulerException error) { + EcsLogger.error("com.artipie.scheduling") + .message("Failed to shutdown Quartz scheduler") + .eventCategory("scheduling") + .eventAction("scheduler_shutdown") + .eventOutcome("failure") + .error(error) + .log(); + } + } + } + } + ); + } + + /** + * Build Quartz properties for JDBC-backed clustered mode. + * @return Properties for StdSchedulerFactory + */ + private static Properties jdbcProperties() { + final Properties props = new Properties(); + // Scheduler identity + props.setProperty( + "org.quartz.scheduler.instanceName", QuartzService.SCHED_NAME + ); + props.setProperty( + "org.quartz.scheduler.instanceId", "AUTO" + ); + // Thread pool + props.setProperty( + "org.quartz.threadPool.class", + "org.quartz.simpl.SimpleThreadPool" + ); + props.setProperty( + "org.quartz.threadPool.threadCount", "10" + ); + props.setProperty( + "org.quartz.threadPool.threadPriority", "5" + ); + // JobStore - JDBC with PostgreSQL + props.setProperty( + "org.quartz.jobStore.class", + "org.quartz.impl.jdbcjobstore.JobStoreTX" + ); + props.setProperty( + "org.quartz.jobStore.driverDelegateClass", + "org.quartz.impl.jdbcjobstore.PostgreSQLDelegate" + ); + props.setProperty( + "org.quartz.jobStore.dataSource", + ArtipieQuartzConnectionProvider.DS_NAME + ); + props.setProperty( + "org.quartz.jobStore.tablePrefix", "QRTZ_" + ); + props.setProperty( + "org.quartz.jobStore.isClustered", "true" + ); + props.setProperty( + "org.quartz.jobStore.clusterCheckinInterval", "15000" + ); + props.setProperty( + "org.quartz.jobStore.misfireThreshold", "60000" + ); + return props; + } + /** * Checks if scheduler thread pool size allows to handle given `requested` amount * of parallel jobs. If thread pool size is smaller than `requested` value, diff --git a/artipie-main/src/main/java/com/artipie/scheduling/TempFileCleanupJob.java b/artipie-main/src/main/java/com/artipie/scheduling/TempFileCleanupJob.java new file mode 100644 index 000000000..2de828166 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/scheduling/TempFileCleanupJob.java @@ -0,0 +1,262 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.scheduling; + +import com.artipie.http.log.EcsLogger; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +/** + * Quartz job that periodically scans a directory for stale temporary files + * created during proxy cache operations and deletes them. + *

+ * Artipie creates temp files in several places: + *

    + *
  • {@code DiskCacheStorage} - UUID-named files in {@code .tmp/} subdirectory
    • + *
  • {@code StreamThroughCache} - files with prefix {@code artipie-stc-} and + * suffix {@code .tmp}
    • + *
  • Other operations that may leave {@code artipie-*} prefixed files in the + * system temp directory
    • + *
+ *

+ * These temp files can accumulate if processes crash before cleanup. This job + * walks the configured directory recursively and deletes files matching known + * temp file patterns that are older than a configurable age threshold. + *

+ * Configuration via {@link JobDataMap}: + *

    + *
  • {@code cleanupDir} - {@link Path} or {@link String} path to the directory + * to scan (required)
    • + *
  • {@code maxAgeMinutes} - {@link Long} maximum file age in minutes before + * deletion (default: 60)
    • + *
+ *

+ * Example scheduling with {@link QuartzService}: + * 

{@code
+ * JobDataMap data = new JobDataMap();
+ * data.put("cleanupDir", Path.of("/tmp"));
+ * data.put("maxAgeMinutes", 60L);
+ * quartzService.schedulePeriodicJob(
+ *     3600, 1, TempFileCleanupJob.class, data
+ * );
+ * }
+ * + * @since 1.20.13 + */ +public final class TempFileCleanupJob implements Job { + + /** + * Key for the cleanup directory in the {@link JobDataMap}. + */ + public static final String CLEANUP_DIR_KEY = "cleanupDir"; + + /** + * Key for the maximum file age in minutes in the {@link JobDataMap}. + */ + public static final String MAX_AGE_MINUTES_KEY = "maxAgeMinutes"; + + /** + * Default maximum age in minutes for temp files before they are deleted. + */ + static final long DEFAULT_MAX_AGE_MINUTES = 60L; + + @Override + public void execute(final JobExecutionContext context) throws JobExecutionException { + final JobDataMap data = context.getMergedJobDataMap(); + final Path dir = resolveCleanupDir(data); + final long max = data.containsKey(MAX_AGE_MINUTES_KEY) + ? data.getLong(MAX_AGE_MINUTES_KEY) + : DEFAULT_MAX_AGE_MINUTES; + cleanup(dir, max); + } + + /** + * Performs the temp file cleanup for the given directory with the given max age. + * This method contains the core cleanup logic and can be called directly + * for testing without requiring a Quartz execution context. + * + * @param dir Directory to scan for stale temp files, or null if not configured + * @param maxAgeMinutes Maximum file age in minutes before deletion + */ + static void cleanup(final Path dir, final long maxAgeMinutes) { + if (dir == null) { + EcsLogger.warn("com.artipie.scheduling") + .message("TempFileCleanupJob: no cleanupDir configured, skipping") + .eventCategory("scheduling") + .eventAction("temp_cleanup") + .eventOutcome("failure") + .log(); + return; + } + if (!Files.isDirectory(dir)) { + EcsLogger.debug("com.artipie.scheduling") + .message( + String.format( + "TempFileCleanupJob: directory does not exist: %s", dir + ) + ) + .eventCategory("scheduling") + .eventAction("temp_cleanup") + .eventOutcome("failure") + .log(); + return; + } + final long cutoff = Instant.now() + .minusMillis(TimeUnit.MINUTES.toMillis(maxAgeMinutes)) + .toEpochMilli(); + final AtomicInteger deleted = new AtomicInteger(0); + final AtomicInteger failed = new AtomicInteger(0); + try { + Files.walkFileTree(dir, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile( + final Path file, final BasicFileAttributes attrs + ) { + if (isTempFile(file) && isStale(attrs, cutoff)) { + try { + Files.deleteIfExists(file); + deleted.incrementAndGet(); + EcsLogger.debug("com.artipie.scheduling") + .message( + String.format( + "TempFileCleanupJob: deleted stale temp file: %s", + file + ) + ) + .eventCategory("scheduling") + .eventAction("temp_cleanup_delete") + .eventOutcome("success") + .log(); + } catch (final IOException ex) { + failed.incrementAndGet(); + EcsLogger.warn("com.artipie.scheduling") + .message( + String.format( + "TempFileCleanupJob: failed to delete: %s", file + ) + ) + .eventCategory("scheduling") + .eventAction("temp_cleanup_delete") + .eventOutcome("failure") + .error(ex) + .log(); + } + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed( + final Path file, final IOException exc + ) { + EcsLogger.warn("com.artipie.scheduling") + .message( + String.format( + "TempFileCleanupJob: cannot access file: %s", file + ) + ) + .eventCategory("scheduling") + .eventAction("temp_cleanup") + .eventOutcome("failure") + .error(exc) + .log(); + return FileVisitResult.CONTINUE; + } + }); + } catch (final IOException ex) { + EcsLogger.error("com.artipie.scheduling") + .message( + String.format( + "TempFileCleanupJob: error walking directory: %s", dir + ) + ) + .eventCategory("scheduling") + .eventAction("temp_cleanup") + .eventOutcome("failure") + .error(ex) + .log(); + } + EcsLogger.info("com.artipie.scheduling") + .message( + String.format( + "TempFileCleanupJob: completed scan of %s, deleted %d stale temp files, %d failures", + dir, deleted.get(), failed.get() + ) + ) + .eventCategory("scheduling") + .eventAction("temp_cleanup") + .eventOutcome("success") + .log(); + } + + /** + * Determines whether a file matches known Artipie temp file patterns. + *

+ * Patterns matched: + *

    + *
  • Files ending with {@code .tmp} (e.g., {@code artipie-stc-*.tmp})
    • + *
  • Files with names starting with {@code artipie-cache-}
    • + *
  • Files with names starting with {@code artipie-stc-}
    • + *
  • Files inside a directory named {@code .tmp} (DiskCacheStorage pattern)
    • + *
  • Files containing {@code .part-} in the name (failed partial writes)
    • + *
+ * + * @param file Path to check + * @return True if the file matches a known temp file pattern + */ + static boolean isTempFile(final Path file) { + final String name = file.getFileName().toString(); + final boolean intmpdir = file.getParent() != null + && file.getParent().getFileName() != null + && ".tmp".equals(file.getParent().getFileName().toString()); + return name.endsWith(".tmp") + || name.startsWith("artipie-cache-") + || name.startsWith("artipie-stc-") + || intmpdir + || name.contains(".part-"); + } + + /** + * Checks whether file attributes indicate the file is older than the cutoff time. + * + * @param attrs File attributes + * @param cutoff Cutoff time in epoch milliseconds + * @return True if the file's last modified time is before the cutoff + */ + private static boolean isStale(final BasicFileAttributes attrs, final long cutoff) { + return attrs.lastModifiedTime().toMillis() < cutoff; + } + + /** + * Resolves the cleanup directory from the job data map. + * Accepts both {@link Path} and {@link String} values. + * + * @param data Job data map + * @return Resolved path, or null if not configured + */ + private static Path resolveCleanupDir(final JobDataMap data) { + final Object raw = data.get(CLEANUP_DIR_KEY); + final Path result; + if (raw instanceof Path) { + result = (Path) raw; + } else if (raw instanceof String) { + result = Path.of((String) raw); + } else { + result = null; + } + return result; + } +} diff --git a/artipie-main/src/main/java/com/artipie/security/policy/CachedDbPolicy.java b/artipie-main/src/main/java/com/artipie/security/policy/CachedDbPolicy.java new file mode 100644 index 000000000..57bee72ff --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/security/policy/CachedDbPolicy.java @@ -0,0 +1,375 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.security.policy; + +import com.artipie.ArtipieException; +import com.artipie.asto.misc.Cleanable; +import com.artipie.asto.misc.UncheckedFunc; +import com.artipie.asto.misc.UncheckedSupplier; +import com.artipie.http.auth.AuthUser; +import com.artipie.http.log.EcsLogger; +import com.artipie.security.perms.EmptyPermissions; +import com.artipie.security.perms.PermissionConfig; +import com.artipie.security.perms.PermissionsLoader; +import com.artipie.security.perms.User; +import com.artipie.security.perms.UserPermissions; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.io.StringReader; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonValue; +import javax.sql.DataSource; + +/** + * Database-backed policy implementation. Reads user roles and role permissions + * from PostgreSQL and uses Caffeine cache to avoid hitting the database on + * every request. + *

+ * Drop-in replacement for {@link CachedYamlPolicy} when a database is + * configured. Users get permissions exclusively through roles (no individual + * user-level permissions in the DB model). + * + * @since 1.21 + */ +@SuppressWarnings("PMD.AvoidDuplicateLiterals") +public final class CachedDbPolicy implements Policy, Cleanable { + + /** + * Permissions factories. + */ + private static final PermissionsLoader FACTORIES = new PermissionsLoader(); + + /** + * Cache for usernames and {@link UserPermissions}. + */ + private final Cache cache; + + /** + * Cache for usernames and user with roles. + */ + private final Cache users; + + /** + * Cache for role name and role permissions. + */ + private final Cache roles; + + /** + * Database data source. + */ + private final DataSource source; + + /** + * Ctor with default cache settings. + * @param source Database data source + */ + public CachedDbPolicy(final DataSource source) { + this( + Caffeine.newBuilder() + .maximumSize(10_000) + .expireAfterAccess(Duration.ofMinutes(3)) + .recordStats() + .build(), + Caffeine.newBuilder() + .maximumSize(10_000) + .expireAfterAccess(Duration.ofMinutes(3)) + .recordStats() + .build(), + Caffeine.newBuilder() + .maximumSize(1_000) + .expireAfterAccess(Duration.ofMinutes(3)) + .recordStats() + .build(), + source + ); + } + + /** + * Primary ctor. + * @param cache Cache for usernames and {@link UserPermissions} + * @param users Cache for username and user roles + * @param roles Cache for role name and role permissions + * @param source Database data source + */ + public CachedDbPolicy( + final Cache cache, + final Cache users, + final Cache roles, + final DataSource source + ) { + this.cache = cache; + this.users = users; + this.roles = roles; + this.source = source; + } + + @Override + public UserPermissions getPermissions(final AuthUser user) { + return this.cache.get(user.name(), key -> { + try { + return this.createUserPermissions(user).call(); + } catch (final Exception err) { + EcsLogger.error("com.artipie.security") + .message("Failed to get user permissions from DB") + .eventCategory("security") + .eventAction("permissions_get") + .eventOutcome("failure") + .field("user.name", user.name()) + .error(err) + .log(); + throw new ArtipieException(err); + } + }); + } + + @Override + public void invalidate(final String key) { + if (this.cache.getIfPresent(key) != null || this.users.getIfPresent(key) != null) { + this.cache.invalidate(key); + this.users.invalidate(key); + } else { + this.roles.invalidate(key); + } + } + + @Override + public void invalidateAll() { + this.cache.invalidateAll(); + this.users.invalidateAll(); + this.roles.invalidateAll(); + } + + /** + * Create {@link UserPermissions} callable for cache loading. + * @param user Authenticated user + * @return Callable that creates UserPermissions + */ + private Callable createUserPermissions(final AuthUser user) { + return () -> new UserPermissions( + new UncheckedSupplier<>( + () -> this.users.get(user.name(), key -> new DbUser(this.source, user)) + ), + new UncheckedFunc<>( + role -> this.roles.get( + role, key -> CachedDbPolicy.rolePermissions(this.source, key) + ) + ) + ); + } + + /** + * Load role permissions from database. + * @param ds Data source + * @param role Role name + * @return Permissions of the role + */ + static PermissionCollection rolePermissions(final DataSource ds, final String role) { + final String sql = "SELECT permissions, enabled FROM roles WHERE name = ?"; + try (Connection conn = ds.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, role); + final ResultSet rs = ps.executeQuery(); + if (!rs.next()) { + return EmptyPermissions.INSTANCE; + } + if (!rs.getBoolean("enabled")) { + return EmptyPermissions.INSTANCE; + } + final String permsJson = rs.getString("permissions"); + if (permsJson == null || permsJson.isEmpty()) { + return EmptyPermissions.INSTANCE; + } + return readPermissionsFromJson( + Json.createReader(new StringReader(permsJson)).readObject() + ); + } catch (final Exception ex) { + EcsLogger.error("com.artipie.security") + .message("Failed to read role permissions from DB") + .eventCategory("security") + .eventAction("role_permissions_read") + .eventOutcome("failure") + .field("user.roles", role) + .error(ex) + .log(); + return EmptyPermissions.INSTANCE; + } + } + + /** + * Parse permissions from the JSON stored in the DB permissions column. + * The DB stores the full API body, e.g.: + * {@code {"permissions": {"api_search_permissions": ["read"], ...}}} + * @param stored The JSON object from the permissions column + * @return Permission collection + */ + private static PermissionCollection readPermissionsFromJson(final JsonObject stored) { + final JsonObject all; + if (stored.containsKey("permissions")) { + all = stored.getJsonObject("permissions"); + } else { + all = stored; + } + if (all == null || all.isEmpty()) { + return EmptyPermissions.INSTANCE; + } + final Permissions res = new Permissions(); + for (final String type : all.keySet()) { + final JsonValue perms = all.get(type); + final PermissionConfig config; + if (perms instanceof JsonObject) { + config = new PermissionConfig.FromJsonObject((JsonObject) perms); + } else if (perms instanceof javax.json.JsonArray) { + config = new PermissionConfig.FromJsonArray((javax.json.JsonArray) perms); + } else { + config = new PermissionConfig.FromJsonObject( + Json.createObjectBuilder().build() + ); + } + Collections.list(FACTORIES.newObject(type, config).elements()) + .forEach(res::add); + } + return res; + } + + /** + * User loaded from database. + * @since 1.21 + */ + static final class DbUser implements User { + + /** + * User individual permissions (always empty for DB users). + */ + private final PermissionCollection perms; + + /** + * User roles. + */ + private final Collection uroles; + + /** + * Ctor. + * @param ds Data source + * @param user Authenticated user + */ + DbUser(final DataSource ds, final AuthUser user) { + final UserRecord rec = DbUser.loadFromDb(ds, user.name()); + if (rec.disabled) { + this.perms = EmptyPermissions.INSTANCE; + this.uroles = Collections.emptyList(); + } else { + this.perms = EmptyPermissions.INSTANCE; + final List rlist = new ArrayList<>(rec.roles); + if (user.authContext() != null && !user.authContext().isEmpty()) { + rlist.add(String.format("default/%s", user.authContext())); + } + this.uroles = rlist; + } + } + + @Override + public PermissionCollection perms() { + return this.perms; + } + + @Override + public Collection roles() { + return this.uroles; + } + + /** + * Load user record from database. + * @param ds Data source + * @param username Username + * @return User record with enabled status and role names + */ + private static UserRecord loadFromDb(final DataSource ds, final String username) { + final String sql = String.join(" ", + "SELECT u.enabled,", + "COALESCE(json_agg(r.name) FILTER (WHERE r.name IS NOT NULL), '[]') AS roles", + "FROM users u", + "LEFT JOIN user_roles ur ON u.id = ur.user_id", + "LEFT JOIN roles r ON ur.role_id = r.id", + "WHERE u.username = ?", + "GROUP BY u.id" + ); + try (Connection conn = ds.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, username); + final ResultSet rs = ps.executeQuery(); + if (!rs.next()) { + EcsLogger.warn("com.artipie.security") + .message("User not found in DB for policy lookup") + .eventCategory("security") + .eventAction("user_lookup") + .eventOutcome("failure") + .field("user.name", username) + .log(); + return new UserRecord(true, Collections.emptyList()); + } + final boolean enabled = rs.getBoolean("enabled"); + final List roles = new ArrayList<>(); + final String rolesJson = rs.getString("roles"); + if (rolesJson != null) { + final javax.json.JsonArray arr = Json.createReader( + new StringReader(rolesJson) + ).readArray(); + for (int i = 0; i < arr.size(); i++) { + roles.add(arr.getString(i)); + } + } + return new UserRecord(!enabled, roles); + } catch (final Exception ex) { + EcsLogger.error("com.artipie.security") + .message("Failed to load user from DB for policy") + .eventCategory("security") + .eventAction("user_lookup") + .eventOutcome("failure") + .field("user.name", username) + .error(ex) + .log(); + return new UserRecord(true, Collections.emptyList()); + } + } + + /** + * Simple record for user data from DB query. + * @since 1.21 + */ + private static final class UserRecord { + /** + * Whether user is disabled. + */ + final boolean disabled; + + /** + * User's role names. + */ + final List roles; + + /** + * Ctor. + * @param disabled Whether user is disabled + * @param roles User's role names + */ + UserRecord(final boolean disabled, final List roles) { + this.disabled = disabled; + this.roles = roles; + } + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/settings/ArtipieSecurity.java b/artipie-main/src/main/java/com/artipie/settings/ArtipieSecurity.java index 97c991151..f121b4f44 100644 --- a/artipie-main/src/main/java/com/artipie/settings/ArtipieSecurity.java +++ b/artipie-main/src/main/java/com/artipie/settings/ArtipieSecurity.java @@ -7,11 +7,13 @@ import com.amihaiemil.eoyaml.YamlMapping; import com.artipie.asto.Storage; import com.artipie.http.auth.Authentication; +import com.artipie.security.policy.CachedDbPolicy; import com.artipie.security.policy.PoliciesLoader; import com.artipie.security.policy.Policy; import com.artipie.security.policy.YamlPolicyConfig; import com.artipie.settings.cache.CachedUsers; import java.util.Optional; +import javax.sql.DataSource; /** * Artipie security: authentication and permissions policy. @@ -78,8 +80,23 @@ class FromYaml implements ArtipieSecurity { */ public FromYaml(final YamlMapping settings, final Authentication auth, final Optional asto) { + this(settings, auth, asto, null); + } + + /** + * Ctor with optional database source. When a DataSource is provided, + * {@link CachedDbPolicy} is used instead of YAML-backed policy. + * @param settings Yaml settings + * @param auth Authentication instance + * @param asto Policy storage + * @param dataSource Database data source, nullable + */ + public FromYaml(final YamlMapping settings, final Authentication auth, + final Optional asto, final DataSource dataSource) { this.auth = auth; - this.plc = FromYaml.initPolicy(settings); + this.plc = dataSource != null + ? new CachedDbPolicy(dataSource) + : FromYaml.initPolicy(settings); this.asto = asto; } diff --git a/artipie-main/src/main/java/com/artipie/settings/JwtSettings.java b/artipie-main/src/main/java/com/artipie/settings/JwtSettings.java index d384f30c4..2ef325dd9 100644 --- a/artipie-main/src/main/java/com/artipie/settings/JwtSettings.java +++ b/artipie-main/src/main/java/com/artipie/settings/JwtSettings.java @@ -5,6 +5,7 @@ package com.artipie.settings; import com.amihaiemil.eoyaml.YamlMapping; +import com.artipie.http.log.EcsLogger; import java.util.Optional; /** @@ -110,8 +111,11 @@ public static JwtSettings fromYaml(final YamlMapping meta) { if (expirySeconds <= 0) { expirySeconds = DEFAULT_EXPIRY_SECONDS; } - } catch (final NumberFormatException ignored) { - // Use default + } catch (final NumberFormatException ex) { + EcsLogger.warn("com.artipie.settings") + .message("Invalid JWT expiry-seconds value, using default") + .error(ex) + .log(); } } String secret = jwt.string("secret"); diff --git a/artipie-main/src/main/java/com/artipie/settings/RepoData.java b/artipie-main/src/main/java/com/artipie/settings/RepoData.java index 746ffa54a..0ec119d45 100644 --- a/artipie-main/src/main/java/com/artipie/settings/RepoData.java +++ b/artipie-main/src/main/java/com/artipie/settings/RepoData.java @@ -18,9 +18,12 @@ import com.artipie.cache.StoragesCache; import com.artipie.http.log.EcsLogger; +import com.artipie.misc.Json2Yaml; +import com.artipie.settings.repo.CrudRepoSettings; import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import javax.json.JsonStructure; /** * Repository data management. @@ -105,7 +108,7 @@ public CompletionStage deleteArtifact(final RepositoryName rname, final .eventAction("artifact_delete") .eventOutcome("success") .field("repository.name", repo) - .field("artifact.path", artifactPath) + .field("file.path", artifactPath) .field("files.count", keys.size()) .log(); return true; @@ -121,7 +124,7 @@ public CompletionStage deleteArtifact(final RepositoryName rname, final .eventAction("artifact_delete") .eventOutcome("success") .field("repository.name", repo) - .field("artifact.path", artifactPath) + .field("file.path", artifactPath) .log(); return true; }); @@ -211,11 +214,11 @@ public CompletionStage move(final RepositoryName rname, final RepositoryNa } /** - * Obtain storage from repository settings. + * Obtain storage from repository settings (YAML config file). * @param rname Repository name * @return Abstract storage */ - private CompletionStage repoStorage(final RepositoryName rname) { + public CompletionStage repoStorage(final RepositoryName rname) { return new ConfigFile(String.format("%s.yaml", rname.toString())) .valueFrom(this.configStorage) .thenCompose(Content::asStringFuture) @@ -252,4 +255,40 @@ private CompletionStage repoStorage(final RepositoryName rname) { }); } + + /** + * Obtain storage with DB fallback. Tries YAML config first, + * then falls back to reading from CrudRepoSettings (DB). + * @param rname Repository name + * @param crs Repository settings CRUD (DB fallback) + * @return Abstract storage + */ + public CompletionStage repoStorage( + final RepositoryName rname, final CrudRepoSettings crs + ) { + return this.repoStorage(rname).exceptionallyCompose(err -> { + if (crs == null || !crs.exists(rname)) { + return CompletableFuture.failedFuture(err); + } + return CompletableFuture.supplyAsync(() -> { + final JsonStructure config = crs.value(rname); + if (config == null) { + throw new IllegalStateException("Repository not found: " + rname); + } + final javax.json.JsonObject jobj = config.asJsonObject(); + final javax.json.JsonObject repo = jobj.containsKey("repo") + ? jobj.getJsonObject("repo") : jobj; + if (!repo.containsKey(RepoData.STORAGE)) { + throw new IllegalStateException( + "No storage config in repository: " + rname + ); + } + final javax.json.JsonObject storageJson = + repo.getJsonObject(RepoData.STORAGE); + final YamlMapping storageYaml = + new Json2Yaml().apply(storageJson.toString()); + return this.storagesCache.storage(storageYaml); + }); + }); + } } diff --git a/artipie-main/src/main/java/com/artipie/settings/Settings.java b/artipie-main/src/main/java/com/artipie/settings/Settings.java index c0c772c5d..ab5512e01 100644 --- a/artipie-main/src/main/java/com/artipie/settings/Settings.java +++ b/artipie-main/src/main/java/com/artipie/settings/Settings.java @@ -8,8 +8,10 @@ import com.amihaiemil.eoyaml.YamlSequence; import com.artipie.api.ssl.KeyStore; import com.artipie.asto.Storage; +import com.artipie.cache.ValkeyConnection; import com.artipie.cooldown.CooldownSettings; import com.artipie.http.client.HttpClientSettings; +import com.artipie.index.ArtifactIndex; import com.artipie.scheduling.MetadataEventQueues; import com.artipie.settings.cache.ArtipieCaches; import java.util.Optional; @@ -144,4 +146,20 @@ default Duration httpServerRequestTimeout() { default JwtSettings jwtSettings() { return new JwtSettings(); } + + /** + * Artifact search index. + * @return Artifact index (NOP if indexing is disabled) + */ + default ArtifactIndex artifactIndex() { + return ArtifactIndex.NOP; + } + + /** + * Optional Valkey connection for cache operations. + * @return Valkey connection if configured + */ + default Optional valkeyConnection() { + return Optional.empty(); + } } diff --git a/artipie-main/src/main/java/com/artipie/settings/SettingsFromPath.java b/artipie-main/src/main/java/com/artipie/settings/SettingsFromPath.java index e5344e692..0cf479e79 100644 --- a/artipie-main/src/main/java/com/artipie/settings/SettingsFromPath.java +++ b/artipie-main/src/main/java/com/artipie/settings/SettingsFromPath.java @@ -44,6 +44,19 @@ public SettingsFromPath(final Path path) { * @throws IOException On IO error */ public Settings find(final QuartzService quartz) throws IOException { + return this.find(quartz, java.util.Optional.empty()); + } + + /** + * Searches settings by the provided path, reusing a pre-created DataSource. + * @param quartz Quartz service + * @param dataSource Shared DataSource to avoid duplicate connection pools + * @return Artipie settings + * @throws IOException On IO error + * @since 1.20.13 + */ + public Settings find(final QuartzService quartz, + final java.util.Optional dataSource) throws IOException { boolean initialize = Boolean.parseBoolean(System.getenv("ARTIPIE_INIT")); if (!Files.exists(this.path)) { new JavaResource("example/artipie.yaml").copy(this.path); @@ -51,7 +64,7 @@ public Settings find(final QuartzService quartz) throws IOException { } final Settings settings = new YamlSettings( Yaml.createYamlInput(this.path.toFile()).readYamlMapping(), - this.path.getParent(), quartz + this.path.getParent(), quartz, dataSource ); final BlockingStorage bsto = new BlockingStorage(settings.configStorage()); final Key init = new Key.From(".artipie", "initialized"); @@ -104,6 +117,7 @@ private static void copyResources( final Path tmp = Files.createTempFile( Path.of(res).getFileName().toString(), ".tmp" ); + tmp.toFile().deleteOnExit(); new JavaResource(String.format("example/%s/%s", dir, res)).copy(tmp); bsto.save(new Key.From(res), Files.readAllBytes(tmp)); Files.delete(tmp); diff --git a/artipie-main/src/main/java/com/artipie/settings/YamlSettings.java b/artipie-main/src/main/java/com/artipie/settings/YamlSettings.java index 38d2c5d1e..e7be15213 100644 --- a/artipie-main/src/main/java/com/artipie/settings/YamlSettings.java +++ b/artipie-main/src/main/java/com/artipie/settings/YamlSettings.java @@ -13,10 +13,15 @@ import com.artipie.asto.Key; import com.artipie.asto.Storage; import com.artipie.asto.SubStorage; +import com.artipie.asto.misc.Cleanable; import com.artipie.asto.factory.Config; import com.artipie.asto.factory.StoragesLoader; +import com.artipie.auth.AuthFromDb; import com.artipie.auth.AuthFromEnv; +import com.artipie.cache.CacheInvalidationPubSub; import com.artipie.cache.GlobalCacheConfig; +import com.artipie.cache.NegativeCacheConfig; +import com.artipie.cache.PublishingCleanable; import com.artipie.cache.StoragesCache; import com.artipie.cache.ValkeyConnection; import com.artipie.cooldown.CooldownSettings; @@ -30,10 +35,14 @@ import com.artipie.scheduling.ArtifactEvent; import com.artipie.scheduling.MetadataEventQueues; import com.artipie.scheduling.QuartzService; +import com.artipie.security.policy.CachedYamlPolicy; import com.artipie.settings.cache.ArtipieCaches; import com.artipie.settings.cache.CachedUsers; import com.artipie.settings.cache.GuavaFiltersCache; +import com.artipie.settings.cache.PublishingFiltersCache; import com.artipie.http.log.EcsLogger; +import com.artipie.index.ArtifactIndex; +import com.artipie.index.DbArtifactIndex; import org.quartz.SchedulerException; import javax.sql.DataSource; @@ -149,11 +158,34 @@ public final class YamlSettings implements Settings { */ private final JwtSettings jwtSettings; + /** + * Artifact index (PostgreSQL-backed). + */ + private final ArtifactIndex artifactIndex; + /** * Path to artipie.yaml config file. */ private final Path configFilePath; + /** + * Redis pub/sub for cross-instance cache invalidation, or null if Valkey is not configured. + * @since 1.20.13 + */ + private final CacheInvalidationPubSub cachePubSub; + + /** + * Valkey connection for proper cleanup on shutdown, or null if Valkey is not configured. + * @since 1.20.13 + */ + private final ValkeyConnection valkeyConn; + + /** + * Guard flag to make {@link #close()} idempotent without spurious error logs. + * @since 1.20.13 + */ + private volatile boolean closed; + /** * Tracked storages for proper cleanup. * Thread-safe list to track all storage instances created by this settings. @@ -171,8 +203,26 @@ public final class YamlSettings implements Settings { * @param path Path to the folder with yaml settings file * @param quartz Quartz service */ - @SuppressWarnings("PMD.ConstructorOnlyInitializesOrCallOtherConstructors") public YamlSettings(final YamlMapping content, final Path path, final QuartzService quartz) { + this(content, path, quartz, Optional.empty()); + } + + /** + * Ctor with optional pre-created DataSource. + *

+ * When a shared DataSource is provided, it is reused instead of creating a + * new connection pool. This allows VertxMain to share one HikariCP pool + * between Quartz JDBC clustering and artifact operations. + * + * @param content YAML file content. + * @param path Path to the folder with yaml settings file + * @param quartz Quartz service + * @param shared Pre-created DataSource to reuse, or empty to create a new one + * @since 1.20.13 + */ + @SuppressWarnings("PMD.ConstructorOnlyInitializesOrCallOtherConstructors") + public YamlSettings(final YamlMapping content, final Path path, + final QuartzService quartz, final Optional shared) { // Config file can be artipie.yaml or artipie.yml this.configFilePath = YamlSettings.findConfigFile(path); this.meta = content.yamlMapping("meta"); @@ -183,23 +233,70 @@ public YamlSettings(final YamlMapping content, final Path path, final QuartzServ // Parse JWT settings first - needed for auth cache TTL capping this.jwtSettings = JwtSettings.fromYaml(this.meta()); final Optional valkey = YamlSettings.initValkey(this.meta()); + this.valkeyConn = valkey.orElse(null); // Initialize global cache config for all adapters GlobalCacheConfig.initialize(valkey); // Initialize unified negative cache config - com.artipie.cache.NegativeCacheConfig.initialize(this.meta().yamlMapping("caches")); + NegativeCacheConfig.initialize(this.meta().yamlMapping("caches")); // Initialize cooldown metadata cache config FilteredMetadataCacheConfig.initialize(this.meta().yamlMapping("caches")); - final CachedUsers auth = YamlSettings.initAuth(this.meta(), valkey, this.jwtSettings); - this.security = new ArtipieSecurity.FromYaml( - this.meta(), auth, new PolicyStorage(this.meta()).parse() + // Initialize database early so AuthFromDb can be used in auth chain + if (shared.isPresent()) { + this.artifactsDb = shared; + } else { + this.artifactsDb = YamlSettings.initArtifactsDb(this.meta()); + } + final CachedUsers auth = YamlSettings.initAuth( + this.meta(), valkey, this.jwtSettings, this.artifactsDb.orElse(null) ); - this.acach = new ArtipieCaches.All( - auth, new StoragesCache(), this.security.policy(), new GuavaFiltersCache() + this.security = new ArtipieSecurity.FromYaml( + this.meta(), auth, new PolicyStorage(this.meta()).parse(), + this.artifactsDb.orElse(null) ); + // Initialize cross-instance cache invalidation via Redis pub/sub + if (valkey.isPresent()) { + final CacheInvalidationPubSub ps = + new CacheInvalidationPubSub(valkey.get()); + this.cachePubSub = ps; + ps.register("auth", auth); + final GuavaFiltersCache filters = new GuavaFiltersCache(); + ps.register("filters", filters); + final Cleanable policyCache; + if (this.security.policy() instanceof Cleanable) { + policyCache = (Cleanable) this.security.policy(); + ps.register("policy", policyCache); + } + this.acach = new ArtipieCaches.All( + new PublishingCleanable(auth, ps, "auth"), + new StoragesCache(), + this.security.policy(), + new PublishingFiltersCache(filters, ps) + ); + } else { + this.cachePubSub = null; + this.acach = new ArtipieCaches.All( + auth, new StoragesCache(), this.security.policy(), new GuavaFiltersCache() + ); + } this.mctx = new MetricsContext(this.meta()); this.lctx = new LoggingContext(this.meta()); this.cooldown = YamlCooldownSettings.fromMeta(this.meta()); - this.artifactsDb = YamlSettings.initArtifactsDb(this.meta()); + // Initialize artifact index + final YamlMapping indexConfig = this.meta.yamlMapping("artifact_index"); + final boolean indexEnabled = indexConfig != null + && "true".equals(indexConfig.string("enabled")); + if (indexEnabled && this.artifactsDb.isPresent()) { + this.artifactIndex = new DbArtifactIndex(this.artifactsDb.get()); + } else if (indexEnabled) { + throw new IllegalStateException( + "artifact_index.enabled=true requires artifacts_database to be configured" + ); + } else if (this.artifactsDb.isPresent()) { + // Auto-enable DB-backed index when database is configured + this.artifactIndex = new DbArtifactIndex(this.artifactsDb.get()); + } else { + this.artifactIndex = ArtifactIndex.NOP; + } this.events = this.artifactsDb.flatMap( db -> YamlSettings.initArtifactsEvents(this.meta(), quartz, db) ); @@ -292,6 +389,11 @@ public Optional artifactsDatabase() { return this.artifactsDb; } + @Override + public Optional valkeyConnection() { + return Optional.ofNullable(this.valkeyConn); + } + @Override public PrefixesConfig prefixes() { return this.prefixesConfig; @@ -302,13 +404,31 @@ public JwtSettings jwtSettings() { return this.jwtSettings; } + @Override + public ArtifactIndex artifactIndex() { + return this.artifactIndex; + } + @Override public void close() { + if (this.closed) { + return; + } + this.closed = true; EcsLogger.info("com.artipie.settings") .message("Closing YamlSettings and cleaning up storage resources") .eventCategory("configuration") .eventAction("settings_close") .log(); + // Close ordering is critical — dependencies flow downward: + // 1. Tracked storages (may use DataSource / Valkey indirectly) + // 2. Artifact index (uses DataSource via its executor) + // 3. Cache pub/sub (uses ValkeyConnection's pub/sub connections) + // 4. HikariDataSource (safe after index executor drained) + // 5. ValkeyConnection (safe after pub/sub closed) + // 6. Clear tracked storages list + // Note: VertxMain.stop() closes HTTP servers before calling this, + // so in-flight requests should have drained by the time we get here. for (final Storage storage : this.trackedStorages) { try { // Try to close via factory first (preferred method) @@ -342,6 +462,83 @@ public void close() { .log(); } } + // Close artifact index + if (this.artifactIndex != null && this.artifactIndex != ArtifactIndex.NOP) { + try { + this.artifactIndex.close(); + EcsLogger.info("com.artipie.settings") + .message("Closed artifact index") + .eventCategory("configuration") + .eventAction("index_close") + .eventOutcome("success") + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie.settings") + .message("Failed to close artifact index") + .eventCategory("configuration") + .eventAction("index_close") + .eventOutcome("failure") + .error(e) + .log(); + } + } + // Close cache invalidation pub/sub + if (this.cachePubSub != null) { + try { + this.cachePubSub.close(); + } catch (final Exception e) { + EcsLogger.error("com.artipie.settings") + .message("Failed to close cache invalidation pub/sub") + .eventCategory("configuration") + .eventAction("pubsub_close") + .eventOutcome("failure") + .error(e) + .log(); + } + } + // Close artifacts database connection pool + if (this.artifactsDb.isPresent()) { + final javax.sql.DataSource ds = this.artifactsDb.get(); + if (ds instanceof AutoCloseable) { + try { + ((AutoCloseable) ds).close(); + EcsLogger.info("com.artipie.settings") + .message("Closed artifacts database connection pool") + .eventCategory("configuration") + .eventAction("database_close") + .eventOutcome("success") + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie.settings") + .message("Failed to close artifacts database connection pool") + .eventCategory("configuration") + .eventAction("database_close") + .eventOutcome("failure") + .error(e) + .log(); + } + } + } + // Close Valkey connection pool + if (this.valkeyConn != null) { + try { + this.valkeyConn.close(); + EcsLogger.info("com.artipie.settings") + .message("Closed Valkey connection") + .eventCategory("configuration") + .eventAction("valkey_close") + .eventOutcome("success") + .log(); + } catch (final Exception e) { + EcsLogger.error("com.artipie.settings") + .message("Failed to close Valkey connection") + .eventCategory("configuration") + .eventAction("valkey_close") + .eventOutcome("failure") + .error(e) + .log(); + } + } this.trackedStorages.clear(); EcsLogger.info("com.artipie.settings") .message("YamlSettings cleanup complete") @@ -357,15 +554,15 @@ public void close() { * @return Storage type string (e.g., "s3", "fs") or null if unknown */ private String detectStorageType(final Storage storage) { - final String className = storage.getClass().getSimpleName().toLowerCase(); + Storage target = storage; + if (target instanceof com.artipie.http.misc.DispatchedStorage) { + target = ((com.artipie.http.misc.DispatchedStorage) target).unwrap(); + } + final String className = target.getClass().getSimpleName().toLowerCase(); if (className.contains("s3")) { return "s3"; } else if (className.contains("file")) { return "fs"; - } else if (className.contains("etcd")) { - return "etcd"; - } else if (className.contains("redis")) { - return "redis"; } return null; } @@ -494,46 +691,65 @@ private static Optional initValkey(final YamlMapping settings) } /** - * Initialise authentication. If `credentials` section is absent or empty, - * {@link AuthFromEnv} is used. + * Initialise authentication. When a database is available, {@link AuthFromDb} + * is used as the primary authenticator. File-based and other providers from + * the YAML credentials section are added as fallbacks. * @param settings Yaml settings * @param valkey Optional Valkey connection for L2 cache * @param jwtSettings JWT settings for cache TTL capping + * @param dataSource Database data source (nullable) * @return Authentication + * @checkstyle ParameterNumberCheck (5 lines) */ private static CachedUsers initAuth( final YamlMapping settings, final Optional valkey, - final JwtSettings jwtSettings + final JwtSettings jwtSettings, + final DataSource dataSource ) { Authentication res; - final YamlSequence creds = settings.yamlSequence(YamlSettings.NODE_CREDENTIALS); - if (creds == null || creds.isEmpty()) { + if (dataSource != null) { + // Database is the primary source of truth for user credentials + res = new AuthFromDb(dataSource); EcsLogger.info("com.artipie.security") - .message("Credentials yaml section is absent or empty, using AuthFromEnv()") + .message("Using AuthFromDb as primary authenticator") .eventCategory("authentication") .eventAction("auth_init") - .field("event.provider", "env") + .field("event.provider", "db") .log(); - res = new AuthFromEnv(); } else { + res = new AuthFromEnv(); + } + // Add YAML-configured providers as fallbacks (SSO, env, etc.) + final YamlSequence creds = settings.yamlSequence(YamlSettings.NODE_CREDENTIALS); + if (creds != null && !creds.isEmpty()) { final AuthLoader loader = new AuthLoader(); - final List auths = creds.values().stream().map( - node -> node.asMapping().string(YamlSettings.NODE_TYPE) - ).map(type -> loader.newObject(type, settings)).toList(); - res = auths.get(0); - for (final Authentication auth : auths.subList(1, auths.size())) { - res = new Authentication.Joined(res, auth); + for (final YamlNode node : creds.values()) { + final String type = node.asMapping().string(YamlSettings.NODE_TYPE); + // Skip "artipie" file-based auth when DB is primary + if (dataSource != null && "artipie".equals(type)) { + continue; + } + try { + final Authentication auth = loader.newObject(type, settings); + res = new Authentication.Joined(res, auth); + } catch (final Exception ex) { + EcsLogger.warn("com.artipie.security") + .message("Failed to load auth provider: " + type) + .eventCategory("authentication") + .eventAction("auth_init") + .eventOutcome("failure") + .error(ex) + .log(); + } } } // Create CachedUsers with Valkey connection and JWT settings for TTL capping if (valkey.isPresent()) { EcsLogger.info("com.artipie.settings") - .message("Initializing auth cache with Valkey L2 cache and JWT TTL cap") + .message(String.format("Initializing auth cache with Valkey L2 cache and JWT TTL cap: expires=%s, expirySeconds=%d", jwtSettings.expires(), jwtSettings.expirySeconds())) .eventCategory("authentication") .eventAction("auth_cache_init") - .field("jwt_expires", jwtSettings.expires()) - .field("jwt_expiry_seconds", jwtSettings.expirySeconds()) .log(); return new CachedUsers(res, valkey.get(), jwtSettings); } else { diff --git a/artipie-main/src/main/java/com/artipie/settings/cache/ArtipieCaches.java b/artipie-main/src/main/java/com/artipie/settings/cache/ArtipieCaches.java index 45d59139d..6fe17bfab 100644 --- a/artipie-main/src/main/java/com/artipie/settings/cache/ArtipieCaches.java +++ b/artipie-main/src/main/java/com/artipie/settings/cache/ArtipieCaches.java @@ -6,6 +6,7 @@ import com.artipie.asto.misc.Cleanable; import com.artipie.cache.StoragesCache; +import com.artipie.security.policy.CachedDbPolicy; import com.artipie.security.policy.CachedYamlPolicy; import com.artipie.security.policy.Policy; @@ -103,6 +104,8 @@ public Cleanable policyCache() { final Cleanable res; if (this.policy instanceof CachedYamlPolicy) { res = (CachedYamlPolicy) this.policy; + } else if (this.policy instanceof CachedDbPolicy) { + res = (CachedDbPolicy) this.policy; } else { res = new Cleanable<>() { @Override diff --git a/artipie-main/src/main/java/com/artipie/settings/cache/CachedUsers.java b/artipie-main/src/main/java/com/artipie/settings/cache/CachedUsers.java index 26df8755a..08273ca11 100644 --- a/artipie-main/src/main/java/com/artipie/settings/cache/CachedUsers.java +++ b/artipie-main/src/main/java/com/artipie/settings/cache/CachedUsers.java @@ -113,11 +113,10 @@ public CachedUsers( ); EcsLogger.info("com.artipie.settings.cache") - .message("Auth cache initialized - JWT-as-password bypasses cache") + .message(String.format("Auth cache initialized - JWT-as-password bypasses cache: basicAuthTtl=%ds, jwtExpiry=%ds", + this.ttl.toSeconds(), jwtSettings != null ? jwtSettings.expirySeconds() : -1)) .eventCategory("cache") .eventAction("init") - .field("basic_auth_ttl_seconds", this.ttl.toSeconds()) - .field("jwt_expiry_seconds", jwtSettings != null ? jwtSettings.expirySeconds() : -1) .log(); // L1: Hot data cache for direct Basic Auth only diff --git a/artipie-main/src/main/java/com/artipie/settings/cache/PublishingFiltersCache.java b/artipie-main/src/main/java/com/artipie/settings/cache/PublishingFiltersCache.java new file mode 100644 index 000000000..eff3c07ad --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/settings/cache/PublishingFiltersCache.java @@ -0,0 +1,62 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.settings.cache; + +import com.amihaiemil.eoyaml.YamlMapping; +import com.artipie.cache.CacheInvalidationPubSub; +import com.artipie.http.filter.Filters; +import java.util.Optional; + +/** + * {@link FiltersCache} decorator that publishes invalidation events + * to Redis pub/sub for cross-instance cache invalidation. + * + * @since 1.20.13 + */ +public final class PublishingFiltersCache implements FiltersCache { + + /** + * Inner cache to delegate to. + */ + private final FiltersCache inner; + + /** + * Pub/sub channel. + */ + private final CacheInvalidationPubSub pubsub; + + /** + * Ctor. + * @param inner Local filters cache + * @param pubsub Redis pub/sub channel + */ + public PublishingFiltersCache(final FiltersCache inner, + final CacheInvalidationPubSub pubsub) { + this.inner = inner; + this.pubsub = pubsub; + } + + @Override + public Optional filters(final String reponame, final YamlMapping repoyaml) { + return this.inner.filters(reponame, repoyaml); + } + + @Override + public long size() { + return this.inner.size(); + } + + @Override + public void invalidate(final String reponame) { + this.inner.invalidate(reponame); + this.pubsub.publish("filters", reponame); + } + + @Override + public void invalidateAll() { + this.inner.invalidateAll(); + this.pubsub.publishAll("filters"); + } +} diff --git a/artipie-main/src/main/java/com/artipie/settings/repo/CrudRepoSettings.java b/artipie-main/src/main/java/com/artipie/settings/repo/CrudRepoSettings.java index 3e37ae338..1134863e2 100644 --- a/artipie-main/src/main/java/com/artipie/settings/repo/CrudRepoSettings.java +++ b/artipie-main/src/main/java/com/artipie/settings/repo/CrudRepoSettings.java @@ -48,6 +48,16 @@ public interface CrudRepoSettings { */ void save(RepositoryName rname, JsonStructure value); + /** + * Add new repository with actor tracking. + * @param rname Repository name + * @param value New repository settings + * @param actor Username performing the action + */ + default void save(RepositoryName rname, JsonStructure value, String actor) { + save(rname, value); + } + /** * Remove repository. * @param rname Repository name diff --git a/artipie-main/src/main/java/com/artipie/settings/repo/DualCrudRepoSettings.java b/artipie-main/src/main/java/com/artipie/settings/repo/DualCrudRepoSettings.java new file mode 100644 index 000000000..15e35c264 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/settings/repo/DualCrudRepoSettings.java @@ -0,0 +1,108 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.settings.repo; + +import com.artipie.api.RepositoryName; +import com.artipie.http.log.EcsLogger; +import java.util.Collection; +import javax.json.JsonStructure; + +/** + * Composite CrudRepoSettings that writes to both DB (primary) and + * YAML storage (secondary). Reads come from the primary. + * The secondary write ensures MapRepositories picks up the config + * from YAML files so the upload/download path can resolve repos. + * @since 1.21.0 + */ +public final class DualCrudRepoSettings implements CrudRepoSettings { + + private final CrudRepoSettings primary; + private final CrudRepoSettings secondary; + + /** + * Ctor. + * @param primary Primary (DB) repo settings + * @param secondary Secondary (YAML) repo settings + */ + public DualCrudRepoSettings( + final CrudRepoSettings primary, final CrudRepoSettings secondary + ) { + this.primary = primary; + this.secondary = secondary; + } + + @Override + public Collection listAll() { + return this.primary.listAll(); + } + + @Override + public Collection list(final String uname) { + return this.primary.list(uname); + } + + @Override + public boolean exists(final RepositoryName rname) { + return this.primary.exists(rname); + } + + @Override + public JsonStructure value(final RepositoryName name) { + return this.primary.value(name); + } + + @Override + public void save(final RepositoryName rname, final JsonStructure value) { + this.save(rname, value, null); + } + + @Override + public void save(final RepositoryName rname, final JsonStructure value, + final String actor) { + this.primary.save(rname, value, actor); + try { + this.secondary.save(rname, value); + } catch (final Exception ex) { + EcsLogger.warn("com.artipie.settings.repo") + .message("Failed to save repo config to secondary (YAML)") + .field("repository.name", rname.toString()) + .error(ex) + .log(); + } + } + + @Override + public void delete(final RepositoryName rname) { + this.primary.delete(rname); + try { + this.secondary.delete(rname); + } catch (final Exception ex) { + EcsLogger.warn("com.artipie.settings.repo") + .message("Failed to delete repo config from secondary (YAML)") + .field("repository.name", rname.toString()) + .error(ex) + .log(); + } + } + + @Override + public void move(final RepositoryName rname, final RepositoryName newrname) { + this.primary.move(rname, newrname); + try { + this.secondary.move(rname, newrname); + } catch (final Exception ex) { + EcsLogger.warn("com.artipie.settings.repo") + .message("Failed to move repo config in secondary (YAML)") + .field("repository.name", rname.toString()) + .error(ex) + .log(); + } + } + + @Override + public boolean hasSettingsDuplicates(final RepositoryName rname) { + return this.primary.hasSettingsDuplicates(rname); + } +} diff --git a/artipie-main/src/main/java/com/artipie/settings/repo/RepoConfig.java b/artipie-main/src/main/java/com/artipie/settings/repo/RepoConfig.java index 58c7eef04..07c410370 100644 --- a/artipie-main/src/main/java/com/artipie/settings/repo/RepoConfig.java +++ b/artipie-main/src/main/java/com/artipie/settings/repo/RepoConfig.java @@ -257,6 +257,43 @@ public Optional settings() { return Optional.ofNullable(this.repoYaml().yamlMapping("settings")); } + /** + * Group routing rules for directing requests to specific members + * based on path prefix or pattern matching. + * + * @return List of routing rules or empty list if not specified. + */ + public List routingRules() { + final YamlSequence seq = this.repoYaml.yamlSequence("routing"); + if (seq == null) { + return Collections.emptyList(); + } + final List rules = new ArrayList<>(seq.size()); + seq.forEach(node -> { + if (node instanceof YamlMapping mapping) { + final String member = mapping.string("member"); + if (member == null || member.isEmpty()) { + throw new IllegalStateException("routing rule missing 'member' field"); + } + final String prefix = mapping.string("prefix"); + final String pattern = mapping.string("pattern"); + if (prefix != null && !prefix.isEmpty()) { + rules.add(new com.artipie.group.RoutingRule.PathPrefix(member, prefix)); + } else if (pattern != null && !pattern.isEmpty()) { + rules.add(new com.artipie.group.RoutingRule.PathPattern(member, pattern)); + } else { + throw new IllegalStateException( + "routing rule for member '" + member + + "' must have 'prefix' or 'pattern'" + ); + } + } else { + throw new IllegalStateException("`routing` element is not mapping in group config"); + } + }); + return rules; + } + public Optional httpClientSettings() { final YamlMapping client = this.repoYaml().yamlMapping("http_client"); return client != null ? Optional.of(HttpClientSettings.from(client)) : Optional.empty(); diff --git a/artipie-main/src/main/java/com/artipie/settings/users/DualCrudUsers.java b/artipie-main/src/main/java/com/artipie/settings/users/DualCrudUsers.java new file mode 100644 index 000000000..0c8043190 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/settings/users/DualCrudUsers.java @@ -0,0 +1,90 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.settings.users; + +import java.util.Optional; +import javax.json.JsonArray; +import javax.json.JsonObject; + +/** + * Composite CrudUsers that delegates to a primary (DB) and secondary (YAML) + * implementation. Writes go to both; reads come from primary. + * This ensures DB-backed users also get YAML files for the YAML-based + * policy to resolve roles and permissions. + * + * @since 1.21 + */ +public final class DualCrudUsers implements CrudUsers { + + /** + * Primary user storage (DB). + */ + private final CrudUsers primary; + + /** + * Secondary user storage (YAML policy files). + */ + private final CrudUsers secondary; + + /** + * Ctor. + * @param primary Primary (DB) user storage + * @param secondary Secondary (YAML) user storage + */ + public DualCrudUsers(final CrudUsers primary, final CrudUsers secondary) { + this.primary = primary; + this.secondary = secondary; + } + + @Override + public JsonArray list() { + return this.primary.list(); + } + + @Override + public Optional get(final String uname) { + return this.primary.get(uname); + } + + @Override + public void addOrUpdate(final JsonObject info, final String uname) { + this.primary.addOrUpdate(info, uname); + try { + this.secondary.addOrUpdate(info, uname); + } catch (final Exception ignored) { + // Best-effort: YAML write failure should not break DB operation + } + } + + @Override + public void remove(final String uname) { + this.primary.remove(uname); + try { + this.secondary.remove(uname); + } catch (final Exception ignored) { + // Best-effort + } + } + + @Override + public void disable(final String uname) { + this.primary.disable(uname); + } + + @Override + public void enable(final String uname) { + this.primary.enable(uname); + } + + @Override + public void alterPassword(final String uname, final JsonObject info) { + this.primary.alterPassword(uname, info); + try { + this.secondary.alterPassword(uname, info); + } catch (final Exception ignored) { + // Best-effort + } + } +} diff --git a/artipie-main/src/main/java/com/artipie/webhook/WebhookConfig.java b/artipie-main/src/main/java/com/artipie/webhook/WebhookConfig.java new file mode 100644 index 000000000..21a13f548 --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/webhook/WebhookConfig.java @@ -0,0 +1,61 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.webhook; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Webhook configuration. + * + * @param url Webhook endpoint URL + * @param secret HMAC-SHA256 signing secret (nullable) + * @param events List of event types to deliver (e.g., "artifact.published", "artifact.deleted") + * @param repos Optional list of repo names to filter (empty = all repos) + * @since 1.20.13 + */ +public record WebhookConfig( + String url, + String secret, + List events, + List repos +) { + + /** + * Ctor. + */ + public WebhookConfig { + Objects.requireNonNull(url, "url"); + events = events != null ? List.copyOf(events) : List.of(); + repos = repos != null ? List.copyOf(repos) : List.of(); + } + + /** + * Check if this webhook should receive the given event type. + * @param eventType Event type (e.g., "artifact.published") + * @return True if this webhook should receive it + */ + public boolean matchesEvent(final String eventType) { + return this.events.isEmpty() || this.events.contains(eventType); + } + + /** + * Check if this webhook should receive events for the given repo. + * @param repoName Repository name + * @return True if this webhook should receive events for this repo + */ + public boolean matchesRepo(final String repoName) { + return this.repos.isEmpty() || this.repos.contains(repoName); + } + + /** + * Get the signing secret if configured. + * @return Optional secret + */ + public Optional signingSecret() { + return Optional.ofNullable(this.secret); + } +} diff --git a/artipie-main/src/main/java/com/artipie/webhook/WebhookDispatcher.java b/artipie-main/src/main/java/com/artipie/webhook/WebhookDispatcher.java new file mode 100644 index 000000000..7637542be --- /dev/null +++ b/artipie-main/src/main/java/com/artipie/webhook/WebhookDispatcher.java @@ -0,0 +1,155 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.webhook; + +import com.artipie.http.log.EcsLogger; +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.WebClient; +import io.vertx.ext.web.client.WebClientOptions; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.HexFormat; +import java.util.List; +import java.util.Objects; + +/** + * Dispatches webhook events to configured endpoints. + * Supports HMAC-SHA256 signing and async delivery with retry. + * + * @since 1.20.13 + */ +@SuppressWarnings("PMD.AvoidCatchingGenericException") +public final class WebhookDispatcher { + + /** + * Max retry attempts per webhook delivery. + */ + private static final int MAX_RETRIES = 3; + + /** + * Webhook configurations. + */ + private final List webhooks; + + /** + * Vert.x web client for async HTTP. + */ + private final WebClient client; + + /** + * Ctor. + * @param webhooks Webhook configurations + * @param vertx Vert.x instance + */ + public WebhookDispatcher(final List webhooks, final Vertx vertx) { + this.webhooks = Objects.requireNonNull(webhooks, "webhooks"); + final WebClientOptions opts = new WebClientOptions() + .setConnectTimeout(5000) + .setIdleTimeout(10); + this.client = WebClient.create(vertx, opts); + } + + /** + * Dispatch an artifact event to all matching webhooks. + * + * @param eventType Event type (e.g., "artifact.published", "artifact.deleted") + * @param repoName Repository name + * @param artifactPath Artifact path + * @param repoType Repository type + */ + public void dispatch( + final String eventType, + final String repoName, + final String artifactPath, + final String repoType + ) { + final JsonObject payload = new JsonObject() + .put("event", eventType) + .put("timestamp", Instant.now().toString()) + .put("repository", new JsonObject() + .put("name", repoName) + .put("type", repoType)) + .put("artifact", new JsonObject() + .put("path", artifactPath)); + for (final WebhookConfig webhook : this.webhooks) { + if (webhook.matchesEvent(eventType) && webhook.matchesRepo(repoName)) { + this.deliverAsync(webhook, payload, 0); + } + } + } + + /** + * Deliver payload to a webhook endpoint with retry. + */ + private void deliverAsync( + final WebhookConfig webhook, final JsonObject payload, final int attempt + ) { + final String body = payload.encode(); + final io.vertx.ext.web.client.HttpRequest request = + this.client.postAbs(webhook.url()) + .putHeader("Content-Type", "application/json") + .putHeader("X-Artipie-Event", payload.getString("event")); + webhook.signingSecret().ifPresent(secret -> { + final String signature = computeHmac(body, secret); + request.putHeader("X-Artipie-Signature", "sha256=" + signature); + }); + request.sendBuffer(Buffer.buffer(body), ar -> { + if (ar.succeeded() && ar.result().statusCode() < 300) { + EcsLogger.debug("com.artipie.webhook") + .message("Webhook delivered") + .eventCategory("webhook") + .eventAction("deliver") + .eventOutcome("success") + .field("url.full", webhook.url()) + .field("event.type", payload.getString("event")) + .log(); + } else if (attempt < MAX_RETRIES) { + final long delay = (long) Math.pow(2, attempt) * 1000L; + io.vertx.core.Vertx.currentContext().owner().setTimer(delay, id -> + this.deliverAsync(webhook, payload, attempt + 1) + ); + } else { + final String error = ar.succeeded() + ? "HTTP " + ar.result().statusCode() + : ar.cause().getMessage(); + EcsLogger.warn("com.artipie.webhook") + .message("Webhook delivery failed after retries") + .eventCategory("webhook") + .eventAction("deliver") + .eventOutcome("failure") + .field("url.full", webhook.url()) + .field("error.message", error) + .log(); + } + }); + } + + /** + * Compute HMAC-SHA256 signature. + * @param payload Payload string + * @param secret Signing secret + * @return Hex-encoded HMAC signature + */ + static String computeHmac(final String payload, final String secret) { + try { + final Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec( + secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256" + )); + return HexFormat.of().formatHex( + mac.doFinal(payload.getBytes(StandardCharsets.UTF_8)) + ); + } catch (final NoSuchAlgorithmException | InvalidKeyException ex) { + throw new IllegalStateException("Failed to compute HMAC-SHA256", ex); + } + } +} diff --git a/artipie-main/src/main/resources/db/migration/V100__create_settings_tables.sql b/artipie-main/src/main/resources/db/migration/V100__create_settings_tables.sql new file mode 100644 index 000000000..d307377b7 --- /dev/null +++ b/artipie-main/src/main/resources/db/migration/V100__create_settings_tables.sql @@ -0,0 +1,95 @@ +-- V100__create_settings_tables.sql +-- Settings layer tables for Artipie Web UI +-- Uses V100 to avoid numbering conflicts with potential artifact table migrations + +CREATE TABLE IF NOT EXISTS repositories ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL UNIQUE, + type VARCHAR(50) NOT NULL, + config JSONB NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_by VARCHAR(255), + updated_by VARCHAR(255) +); + +CREATE INDEX IF NOT EXISTS idx_repositories_type ON repositories (type); +CREATE INDEX IF NOT EXISTS idx_repositories_enabled ON repositories (enabled); + +CREATE TABLE IF NOT EXISTS users ( + id SERIAL PRIMARY KEY, + username VARCHAR(255) NOT NULL UNIQUE, + password_hash VARCHAR(255), + email VARCHAR(255), + enabled BOOLEAN NOT NULL DEFAULT TRUE, + auth_provider VARCHAR(50) NOT NULL DEFAULT 'artipie', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_users_enabled ON users (enabled); +CREATE INDEX IF NOT EXISTS idx_users_auth_provider ON users (auth_provider); + +CREATE TABLE IF NOT EXISTS roles ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL UNIQUE, + permissions JSONB NOT NULL DEFAULT '{}'::jsonb, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS user_roles ( + user_id INT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + role_id INT NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + PRIMARY KEY (user_id, role_id) +); + +CREATE TABLE IF NOT EXISTS storage_aliases ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + repo_name VARCHAR(255), + config JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (name, repo_name) +); + +-- Partial unique index for global aliases (repo_name IS NULL). +-- PostgreSQL UNIQUE constraints treat NULLs as distinct, so without this +-- two rows ('default', NULL) would both be allowed. +CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_aliases_global_unique + ON storage_aliases (name) WHERE repo_name IS NULL; + +CREATE INDEX IF NOT EXISTS idx_storage_aliases_repo ON storage_aliases (repo_name); + +CREATE TABLE IF NOT EXISTS settings ( + key VARCHAR(255) PRIMARY KEY, + value JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_by VARCHAR(255) +); + +CREATE TABLE IF NOT EXISTS auth_providers ( + id SERIAL PRIMARY KEY, + type VARCHAR(50) NOT NULL UNIQUE, + priority INT NOT NULL DEFAULT 0, + config JSONB NOT NULL DEFAULT '{}'::jsonb, + enabled BOOLEAN NOT NULL DEFAULT TRUE +); + +CREATE TABLE IF NOT EXISTS audit_log ( + id BIGSERIAL PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + actor VARCHAR(255), + action VARCHAR(50) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_name VARCHAR(255), + old_value JSONB, + new_value JSONB +); + +CREATE INDEX IF NOT EXISTS idx_audit_log_created ON audit_log (created_at); +CREATE INDEX IF NOT EXISTS idx_audit_log_resource ON audit_log (resource_type, resource_name); +CREATE INDEX IF NOT EXISTS idx_audit_log_actor ON audit_log (actor); diff --git a/artipie-main/src/main/resources/db/migration/V101__create_user_tokens_table.sql b/artipie-main/src/main/resources/db/migration/V101__create_user_tokens_table.sql new file mode 100644 index 000000000..04f044800 --- /dev/null +++ b/artipie-main/src/main/resources/db/migration/V101__create_user_tokens_table.sql @@ -0,0 +1,15 @@ +-- V101__create_user_tokens_table.sql +-- API tokens issued to users, with expiry tracking and revocation support + +CREATE TABLE IF NOT EXISTS user_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + username VARCHAR(255) NOT NULL, + label VARCHAR(255) NOT NULL DEFAULT 'API Token', + token_hash VARCHAR(255) NOT NULL, + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + revoked BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE INDEX IF NOT EXISTS idx_user_tokens_username ON user_tokens (username); +CREATE INDEX IF NOT EXISTS idx_user_tokens_revoked ON user_tokens (revoked) WHERE revoked = FALSE; diff --git a/artipie-main/src/main/resources/swagger-ui/swagger-initializer.js b/artipie-main/src/main/resources/swagger-ui/swagger-initializer.js index 7d65d4bb5..fdd772f19 100644 --- a/artipie-main/src/main/resources/swagger-ui/swagger-initializer.js +++ b/artipie-main/src/main/resources/swagger-ui/swagger-initializer.js @@ -10,7 +10,7 @@ window.onload = function() { {url: "./yaml/users.yaml", name: "Users"}, {url: "./yaml/roles.yaml", name: "Roles"}, {url: "./yaml/settings.yaml", name: "Settings"}, - {url: "./yaml/cache.yaml", name: "Cache & Health"}, + {url: "./yaml/search.yaml", name: "Search"}, ], dom_id: '#swagger-ui', deepLinking: true, diff --git a/artipie-main/src/main/resources/swagger-ui/yaml/cache.yaml b/artipie-main/src/main/resources/swagger-ui/yaml/cache.yaml deleted file mode 100644 index a04e6cded..000000000 --- a/artipie-main/src/main/resources/swagger-ui/yaml/cache.yaml +++ /dev/null @@ -1,427 +0,0 @@ -openapi: "3.0.0" -info: - version: 1.0.0 - title: Artipie - OpenAPI 3.0 - description: - This is Artipie Server based on the OpenAPI 3.0 specification. - license: - name: MIT -externalDocs: - description: Find out more about Artipie - url: https://github.com/artipie -tags: - - name: cache - description: Cache management operations - - name: health - description: Health check endpoints -paths: - /api/health: - get: - summary: Health check endpoint - description: Returns the health status of the Artipie server - operationId: healthCheck - tags: - - health - security: [] - responses: - '200': - description: Server is healthy - content: - application/json: - schema: - $ref: '#/components/schemas/HealthStatus' - example: - status: "ok" - /api/cache/negative/groups: - get: - summary: List all registered groups - description: Returns a list of all group repository names that have negative cache instances registered - operationId: listCacheGroups - tags: - - cache - security: - - bearerAuth: [ ] - responses: - '200': - description: List of registered group names - content: - application/json: - schema: - $ref: '#/components/schemas/GroupsList' - example: - groups: ["npm_group", "maven_group", "pypi_group"] - count: 3 - '401': - $ref: '#/components/responses/UnauthorizedError' - default: - description: unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - /api/cache/negative/group/{groupName}/stats: - get: - summary: Get cache statistics for a group - description: Returns L1 cache size and whether two-tier caching (L2 Valkey/Redis) is enabled - operationId: getCacheStats - tags: - - cache - parameters: - - name: groupName - in: path - required: true - description: Name of the group repository - schema: - type: string - example: "npm_group" - security: - - bearerAuth: [ ] - responses: - '200': - description: Cache statistics for the group - content: - application/json: - schema: - $ref: '#/components/schemas/CacheStats' - example: - group: "npm_group" - l1Size: 1250 - twoTier: true - '401': - $ref: '#/components/responses/UnauthorizedError' - '404': - description: Group not found in cache registry - content: - application/json: - schema: - $ref: '#/components/schemas/GroupNotFoundError' - example: - error: "Group not found in cache registry" - group: "unknown_group" - default: - description: unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - /api/cache/negative/group/{groupName}: - delete: - summary: Clear all negative cache for a group - description: | - Removes all negative cache entries (cached 404 responses) for the specified group repository. - This clears both L1 (in-memory) and L2 (Valkey/Redis) caches if configured. - Use this when you know packages have been published and want to force re-fetching. - operationId: clearGroupCache - tags: - - cache - parameters: - - name: groupName - in: path - required: true - description: Name of the group repository to clear cache for - schema: - type: string - example: "npm_group" - security: - - bearerAuth: [ ] - responses: - '200': - description: Cache cleared successfully - content: - application/json: - schema: - $ref: '#/components/schemas/CacheClearResponse' - example: - status: "cleared" - group: "npm_group" - '401': - $ref: '#/components/responses/UnauthorizedError' - '500': - description: Failed to clear cache - content: - application/json: - schema: - $ref: '#/components/schemas/CacheOperationError' - default: - description: unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - /api/cache/negative/group/{groupName}/package: - delete: - summary: Invalidate specific package cache in a group - description: | - Removes negative cache entries for a specific package path within a group repository. - Use this after publishing a new package to ensure it's immediately available. - The path should match the package identifier (e.g., "@scope/package-name" for NPM). - operationId: invalidatePackageInGroup - tags: - - cache - parameters: - - name: groupName - in: path - required: true - description: Name of the group repository - schema: - type: string - example: "npm_group" - requestBody: - required: true - description: Package path to invalidate - content: - application/json: - schema: - $ref: '#/components/schemas/PackagePathRequest' - example: - path: "@scope/package-name" - security: - - bearerAuth: [ ] - responses: - '200': - description: Package cache invalidated successfully - content: - application/json: - schema: - $ref: '#/components/schemas/PackageInvalidateResponse' - example: - status: "invalidated" - group: "npm_group" - package: "@scope/package-name" - '400': - description: Invalid request (missing path or invalid JSON) - content: - application/json: - schema: - $ref: '#/components/schemas/InvalidRequestError' - example: - error: "Missing 'path' in request body" - example: "{\"path\": \"@scope/package-name\"}" - '401': - $ref: '#/components/responses/UnauthorizedError' - '500': - description: Failed to invalidate cache - content: - application/json: - schema: - $ref: '#/components/schemas/CacheOperationError' - default: - description: unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - /api/cache/negative/package: - delete: - summary: Invalidate package cache globally - description: | - Removes negative cache entries for a specific package path across ALL group repositories. - This is useful when a package is published to a member repository that is used by multiple groups. - The operation affects all registered group caches. - operationId: invalidatePackageGlobally - tags: - - cache - requestBody: - required: true - description: Package path to invalidate globally - content: - application/json: - schema: - $ref: '#/components/schemas/PackagePathRequest' - example: - path: "@scope/package-name" - security: - - bearerAuth: [ ] - responses: - '200': - description: Package cache invalidated in all groups - content: - application/json: - schema: - $ref: '#/components/schemas/GlobalInvalidateResponse' - example: - status: "invalidated" - package: "@scope/package-name" - groupsAffected: ["npm_group", "npm_mirror", "all_packages"] - '400': - description: Invalid request (missing path or invalid JSON) - content: - application/json: - schema: - $ref: '#/components/schemas/InvalidRequestError' - example: - error: "Missing 'path' in request body" - example: "{\"path\": \"@scope/package-name\"}" - '401': - $ref: '#/components/responses/UnauthorizedError' - '500': - description: Failed to invalidate cache - content: - application/json: - schema: - $ref: '#/components/schemas/CacheOperationError' - default: - description: unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" -components: - securitySchemes: - bearerAuth: - type: http - scheme: bearer - bearerFormat: JWT - schemas: - Error: - type: object - required: - - code - - message - properties: - code: - type: integer - format: int32 - message: - type: string - HealthStatus: - type: object - required: - - status - properties: - status: - type: string - description: Health status indicator - enum: ["ok", "degraded", "unhealthy"] - GroupsList: - type: object - required: - - groups - - count - properties: - groups: - type: array - items: - type: string - description: List of registered group repository names - count: - type: integer - description: Number of registered groups - CacheStats: - type: object - required: - - group - - l1Size - - twoTier - properties: - group: - type: string - description: Group repository name - l1Size: - type: integer - format: int64 - description: Number of entries in L1 (in-memory) cache - twoTier: - type: boolean - description: Whether L2 (Valkey/Redis) caching is enabled - GroupNotFoundError: - type: object - required: - - error - - group - properties: - error: - type: string - description: Error message - group: - type: string - description: Group name that was not found - CacheClearResponse: - type: object - required: - - status - - group - properties: - status: - type: string - description: Operation status - enum: ["cleared"] - group: - type: string - description: Group repository name - PackagePathRequest: - type: object - required: - - path - properties: - path: - type: string - description: | - Package path to invalidate. Format depends on repository type: - - NPM: "@scope/package-name" or "package-name" - - Maven: "com/example/artifact" - - PyPI: "package-name" - PackageInvalidateResponse: - type: object - required: - - status - - group - - package - properties: - status: - type: string - description: Operation status - enum: ["invalidated"] - group: - type: string - description: Group repository name - package: - type: string - description: Package path that was invalidated - GlobalInvalidateResponse: - type: object - required: - - status - - package - - groupsAffected - properties: - status: - type: string - description: Operation status - enum: ["invalidated"] - package: - type: string - description: Package path that was invalidated - groupsAffected: - type: array - items: - type: string - description: List of group names where cache was invalidated - InvalidRequestError: - type: object - required: - - error - properties: - error: - type: string - description: Error message - example: - type: string - description: Example of correct request format - CacheOperationError: - type: object - required: - - error - - message - properties: - error: - type: string - description: Error type - message: - type: string - description: Detailed error message - responses: - UnauthorizedError: - description: "Access token is missing or invalid" -security: - - bearerAuth: [] diff --git a/artipie-main/src/main/resources/swagger-ui/yaml/repo.yaml b/artipie-main/src/main/resources/swagger-ui/yaml/repo.yaml index a512bd8d8..d040967f8 100644 --- a/artipie-main/src/main/resources/swagger-ui/yaml/repo.yaml +++ b/artipie-main/src/main/resources/swagger-ui/yaml/repo.yaml @@ -3,7 +3,7 @@ info: version: 1.0.0 title: Artipie - OpenAPI 3.0 description: - This is Atripie Server based on the OpenAPI 3.0 specification. + This is Artipie Server based on the OpenAPI 3.0 specification. license: name: MIT externalDocs: @@ -16,6 +16,7 @@ paths: /api/v1/repository/list: get: summary: List all repositories. + description: Returns the names of all configured repositories. Requires read permission on the repository API. operationId: listAll tags: - repository @@ -41,6 +42,10 @@ paths: /api/v1/repository/{rname}: get: summary: Get repository settings + description: | + Returns the full configuration for a repository, including type, storage, + permissions, and any proxy/group settings. Returns 404 if the repository + does not exist, or 409 if duplicate configuration files are found. operationId: getRepo tags: - repository @@ -64,10 +69,22 @@ paths: - $ref: '#/components/schemas/FullRepository' '400': description: Wrong repository name + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '404': description: Repository not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '409': description: Repository has settings duplicates + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -78,6 +95,7 @@ paths: $ref: "#/components/schemas/Error" head: summary: Checks if repository settings exist + description: Lightweight check for repository existence. Returns 200 if the repository exists, 404 otherwise. operationId: existRepo tags: - repository @@ -95,10 +113,22 @@ paths: description: Repository exists '400': description: Wrong repository name + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '404': description: Repository not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '409': description: Repository has settings duplicates + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -109,6 +139,10 @@ paths: $ref: "#/components/schemas/Error" put: summary: Create or update repository + description: | + Creates a new repository or updates an existing one. The request body must + contain the repository configuration including type and storage. Requires + create permission for new repositories or update permission for existing ones. operationId: createOrUpdateRepo tags: - repository @@ -133,6 +167,16 @@ paths: description: Repository was created or updated '400': description: Wrong repository name + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + '403': + description: Insufficient permissions + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -143,6 +187,7 @@ paths: $ref: "#/components/schemas/Error" delete: summary: Remove repository + description: Permanently removes a repository and its configuration. The repository storage data is not deleted. Requires delete permission. operationId: removeRepo tags: - repository @@ -160,8 +205,16 @@ paths: description: Remove a repository with name {rname} '400': description: Wrong repository name + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '404': description: Repository not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -173,6 +226,7 @@ paths: /api/v1/repository/{rname}/move: put: summary: Move repository + description: Renames/moves a repository by changing its configuration key. Returns 404 if the source repository does not exist, or 409 if the target name conflicts. operationId: moveRepo tags: - repository @@ -197,10 +251,22 @@ paths: description: Repository moved successfully '400': description: Wrong repository name + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '404': description: Repository not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '409': description: Repository has settings duplicates + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -236,10 +302,18 @@ paths: description: Cooldown removed '400': description: Invalid request + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' '404': description: Repository not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" default: description: unexpected error content: @@ -266,10 +340,18 @@ paths: description: All cooldown entries removed '400': description: Invalid repository configuration + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' '404': description: Repository not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" default: description: unexpected error content: @@ -308,10 +390,18 @@ paths: description: Artifact deleted successfully '400': description: Invalid request (missing path) + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' '404': description: Repository or artifact not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" default: description: unexpected error content: @@ -350,10 +440,18 @@ paths: description: Package folder deleted successfully '400': description: Invalid request (missing path) + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' '404': description: Repository or package folder not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" default: description: unexpected error content: @@ -456,6 +554,10 @@ paths: description: Alias was removed successfully '404': description: Alias does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -541,6 +643,10 @@ paths: description: Alias was removed successfully '404': description: Alias does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -820,5 +926,9 @@ components: responses: UnauthorizedError: description: "Access token is missing or invalid" + content: + application/json: + schema: + $ref: "#/components/schemas/Error" security: - bearerAuth: [] diff --git a/artipie-main/src/main/resources/swagger-ui/yaml/roles.yaml b/artipie-main/src/main/resources/swagger-ui/yaml/roles.yaml index cd4919fc6..fba86e36b 100644 --- a/artipie-main/src/main/resources/swagger-ui/yaml/roles.yaml +++ b/artipie-main/src/main/resources/swagger-ui/yaml/roles.yaml @@ -3,7 +3,7 @@ info: version: 1.0.0 title: Artipie - OpenAPI 3.0 description: - This is Atripie Server based on the OpenAPI 3.0 specification. + This is Artipie Server based on the OpenAPI 3.0 specification. license: name: MIT externalDocs: @@ -16,6 +16,7 @@ paths: /api/v1/roles: get: summary: List all roles. + description: Returns all configured roles with their permissions and enabled status. Requires read permission on the role API. operationId: listAllRoles tags: - roles @@ -41,6 +42,7 @@ paths: /api/v1/roles/{role}: get: summary: Get role info. + description: Returns the configuration for a specific role including its permissions and enabled status. Returns 404 if the role does not exist. operationId: getRole tags: - roles @@ -62,6 +64,10 @@ paths: $ref: "#/components/schemas/Role" '404': description: Role does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -72,6 +78,7 @@ paths: $ref: "#/components/schemas/Error" put: summary: Create or replace role. + description: Creates a new role or replaces an existing one. The request body must contain the role permissions. Requires create or update permission. operationId: putRole tags: - roles @@ -95,6 +102,12 @@ paths: description: Role successfully added '401': $ref: '#/components/responses/UnauthorizedError' + '403': + description: Insufficient permissions + content: + application/json: + schema: + $ref: "#/components/schemas/Error" default: description: unexpected error content: @@ -103,6 +116,7 @@ paths: $ref: "#/components/schemas/Error" delete: summary: Delete role info. + description: Permanently removes a role. Users assigned to this role will lose its permissions. Returns 404 if the role does not exist. operationId: deleteRole tags: - roles @@ -120,6 +134,10 @@ paths: description: Role removed successfully '404': description: Role does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -131,6 +149,7 @@ paths: /api/v1/roles/{role}/disable: post: summary: Disable role. + description: Disables a role without deleting it. Users assigned to this role will temporarily lose its permissions. Returns 404 if the role does not exist. operationId: disable tags: - roles @@ -148,6 +167,10 @@ paths: description: Role disabled successfully '404': description: Role does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -159,6 +182,7 @@ paths: /api/v1/roles/{role}/enable: post: summary: Enable role. + description: Re-enables a previously disabled role. Users assigned to this role will regain its permissions. Returns 404 if the role does not exist. operationId: enable tags: - roles @@ -176,6 +200,10 @@ paths: description: Role enabled successfully '404': description: Role does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -295,6 +323,16 @@ components: enum: ["read", "write"] description: Permissions for cache REST API operations example: ["read", "write"] + api_search_permissions: + oneOf: + - type: string + enum: ["*"] + - type: array + items: + type: string + enum: ["read", "write"] + description: Permissions for search REST API operations + example: ["read"] adapter_basic_permissions: type: object description: Repository-specific artifact permissions @@ -313,5 +351,9 @@ components: responses: UnauthorizedError: description: "Access token is missing or invalid" + content: + application/json: + schema: + $ref: "#/components/schemas/Error" security: - bearerAuth: [] \ No newline at end of file diff --git a/artipie-main/src/main/resources/swagger-ui/yaml/search.yaml b/artipie-main/src/main/resources/swagger-ui/yaml/search.yaml new file mode 100644 index 000000000..23464ab32 --- /dev/null +++ b/artipie-main/src/main/resources/swagger-ui/yaml/search.yaml @@ -0,0 +1,305 @@ +openapi: "3.0.0" +info: + version: 1.0.0 + title: Artipie - OpenAPI 3.0 + description: Artipie artifact search API + license: + name: MIT +externalDocs: + description: Find out more about Artipie + url: https://github.com/artipie +tags: + - name: search + description: Artifact search operations +paths: + /api/v1/search: + get: + summary: Full-text search across all indexed artifacts + description: | + Searches the artifact index using full-text queries. Returns matching artifacts + with metadata including repository name, type, path, version, and size. + Results are paginated using size and from parameters. + operationId: searchArtifacts + tags: + - search + parameters: + - name: q + in: query + required: true + description: Search query string + schema: + type: string + example: "spring-boot" + - name: size + in: query + required: false + description: Maximum number of results (default 20, max 100) + schema: + type: integer + default: 20 + minimum: 1 + maximum: 100 + - name: from + in: query + required: false + description: Starting offset for pagination (default 0) + schema: + type: integer + default: 0 + minimum: 0 + security: + - bearerAuth: [ ] + responses: + '200': + description: Search results + content: + application/json: + schema: + $ref: '#/components/schemas/SearchResponse' + example: + items: + - repo_type: "maven" + repo_name: "maven-central" + artifact_path: "org/springframework/spring-boot/3.2.0/spring-boot-3.2.0.jar" + artifact_name: "spring-boot" + version: "3.2.0" + size: 1548234 + owner: "admin" + total_hits: 42 + offset: 0 + '400': + description: Missing or invalid query parameter + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + '401': + $ref: '#/components/responses/UnauthorizedError' + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /api/v1/search/locate: + get: + summary: Locate repositories containing an artifact + description: | + Finds which repositories contain an artifact at the specified path. + Useful for group repository resolution and dependency analysis. + operationId: locateArtifact + tags: + - search + parameters: + - name: path + in: query + required: true + description: Artifact path to locate + schema: + type: string + example: "org/springframework/spring-boot/3.2.0/spring-boot-3.2.0.jar" + security: + - bearerAuth: [ ] + responses: + '200': + description: List of repository names containing the artifact + content: + application/json: + schema: + $ref: '#/components/schemas/LocateResponse' + example: + repositories: ["maven-central", "maven-local"] + count: 2 + '400': + description: Missing path parameter + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + '401': + $ref: '#/components/responses/UnauthorizedError' + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /api/v1/search/reindex: + post: + summary: Trigger full reindex of all artifacts + description: | + Triggers a full rebuild of the search index from the artifact database. + This is an admin-only operation that runs asynchronously. + Use this after data migration or to recover from index corruption. + operationId: reindexArtifacts + tags: + - search + security: + - bearerAuth: [ ] + responses: + '202': + description: Reindex started + content: + application/json: + schema: + $ref: '#/components/schemas/ReindexResponse' + example: + status: "started" + message: "Full reindex initiated" + '401': + $ref: '#/components/responses/UnauthorizedError' + '403': + description: Insufficient permissions (requires write permission) + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /api/v1/search/stats: + get: + summary: Get artifact index statistics + description: | + Returns statistics about the artifact search index including + document count, warmup status, and directory type. + operationId: getIndexStats + tags: + - search + security: + - bearerAuth: [ ] + responses: + '200': + description: Index statistics + content: + application/json: + schema: + $ref: '#/components/schemas/IndexStats' + '401': + $ref: '#/components/responses/UnauthorizedError' + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + schemas: + Error: + type: object + required: + - code + - message + properties: + code: + type: integer + format: int32 + message: + type: string + SearchResponse: + type: object + required: + - items + - total_hits + - offset + properties: + items: + type: array + items: + $ref: '#/components/schemas/ArtifactResult' + total_hits: + type: integer + format: int64 + description: Total number of matching artifacts + offset: + type: integer + description: Current pagination offset + LocateResponse: + type: object + required: + - repositories + - count + properties: + repositories: + type: array + items: + type: string + description: Repository names containing the artifact + count: + type: integer + description: Number of repositories found + ArtifactResult: + type: object + required: + - repo_type + - repo_name + - artifact_path + properties: + repo_type: + type: string + description: Repository type (maven, npm, pypi, etc.) + repo_name: + type: string + description: Repository name + artifact_path: + type: string + description: Full artifact path within the repository + artifact_name: + type: string + description: Artifact name (tokenized for search) + version: + type: string + description: Artifact version + size: + type: integer + format: int64 + description: Artifact size in bytes + created_at: + type: string + format: date-time + description: When the artifact was indexed + owner: + type: string + description: User who published the artifact + ReindexResponse: + type: object + required: + - status + properties: + status: + type: string + description: Operation status + enum: ["started"] + message: + type: string + description: Human-readable status message + IndexStats: + type: object + properties: + documents: + type: integer + format: int64 + description: Total number of indexed documents + warmedUp: + type: boolean + description: Whether the index warmup has completed + directoryType: + type: string + description: Lucene directory implementation type + responses: + UnauthorizedError: + description: "Access token is missing or invalid" + content: + application/json: + schema: + $ref: "#/components/schemas/Error" +security: + - bearerAuth: [] diff --git a/artipie-main/src/main/resources/swagger-ui/yaml/settings.yaml b/artipie-main/src/main/resources/swagger-ui/yaml/settings.yaml index 40dd71287..a1a0fdcb1 100644 --- a/artipie-main/src/main/resources/swagger-ui/yaml/settings.yaml +++ b/artipie-main/src/main/resources/swagger-ui/yaml/settings.yaml @@ -3,7 +3,7 @@ info: version: 1.0.0 title: Artipie - OpenAPI 3.0 description: - This is Atripie Server based on the OpenAPI 3.0 specification. + This is Artipie Server based on the OpenAPI 3.0 specification. license: name: MIT externalDocs: @@ -13,6 +13,37 @@ tags: - name: settings description: Operations about settings paths: + /api/v1/dashboard: + get: + summary: Admin dashboard statistics + description: | + Returns aggregate statistics for the Artipie instance including + server port, version, and repository count. Useful for admin + dashboard UIs. + operationId: getDashboard + tags: + - settings + security: + - bearerAuth: [ ] + responses: + '200': + description: Dashboard statistics + content: + application/json: + schema: + $ref: '#/components/schemas/Dashboard' + example: + port: 8080 + version: "1.20.13" + repositories: 42 + '401': + $ref: '#/components/responses/UnauthorizedError' + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" /api/v1/settings/port: get: summary: Artipie server-side port (repositories default port). @@ -146,6 +177,26 @@ components: example: ["org1", "org2", "team-alpha"] minItems: 0 uniqueItems: true + Dashboard: + type: object + description: Admin dashboard statistics + required: + - port + - version + properties: + port: + type: integer + format: int32 + description: Artipie server port + example: 8080 + version: + type: string + description: Artipie version + example: "1.20.13" + repositories: + type: integer + description: Total number of configured repositories + example: 42 Error: type: object description: Error response @@ -159,4 +210,11 @@ components: description: HTTP status code message: type: string - description: Human-readable error message \ No newline at end of file + description: Human-readable error message + responses: + UnauthorizedError: + description: "Access token is missing or invalid" + content: + application/json: + schema: + $ref: "#/components/schemas/Error" \ No newline at end of file diff --git a/artipie-main/src/main/resources/swagger-ui/yaml/token-gen.yaml b/artipie-main/src/main/resources/swagger-ui/yaml/token-gen.yaml index 5acc74d32..14a994c27 100644 --- a/artipie-main/src/main/resources/swagger-ui/yaml/token-gen.yaml +++ b/artipie-main/src/main/resources/swagger-ui/yaml/token-gen.yaml @@ -3,7 +3,7 @@ info: version: 1.0.0 title: Artipie - OpenAPI 3.0 description: - This is Atripie Server based on the OpenAPI 3.0 specification. + This is Artipie Server based on the OpenAPI 3.0 specification. license: name: MIT externalDocs: @@ -15,7 +15,7 @@ tags: paths: /api/v1/oauth/token: post: - summary: Obtain JWT auth token . + summary: Obtain JWT auth token operationId: getJwtToken tags: - oauth @@ -34,6 +34,16 @@ paths: $ref: "#/components/schemas/Token" '401': description: User and password pair is not valid + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + '400': + description: Invalid request body + content: + application/json: + schema: + $ref: "#/components/schemas/Error" default: description: unexpected error content: @@ -41,6 +51,18 @@ paths: schema: $ref: "#/components/schemas/Error" components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + responses: + UnauthorizedError: + description: "Access token is missing or invalid" + content: + application/json: + schema: + $ref: "#/components/schemas/Error" schemas: OAuthUser: type: object diff --git a/artipie-main/src/main/resources/swagger-ui/yaml/users.yaml b/artipie-main/src/main/resources/swagger-ui/yaml/users.yaml index 8ce821e0a..1c0e3c739 100644 --- a/artipie-main/src/main/resources/swagger-ui/yaml/users.yaml +++ b/artipie-main/src/main/resources/swagger-ui/yaml/users.yaml @@ -3,7 +3,7 @@ info: version: 1.0.0 title: Artipie - OpenAPI 3.0 description: - This is Atripie Server based on the OpenAPI 3.0 specification. + This is Artipie Server based on the OpenAPI 3.0 specification. license: name: MIT externalDocs: @@ -13,6 +13,46 @@ tags: - name: users description: Operations about users paths: + /api/v1/users/me: + get: + summary: Get current authenticated user info and effective permissions + description: | + Returns information about the currently authenticated user, including + their username, authentication context, profile details, and a summary + of their effective permissions across all API categories. This endpoint + is essential for UI integration to determine what features to display. + operationId: getCurrentUser + tags: + - users + security: + - bearerAuth: [ ] + responses: + '200': + description: Current user info with effective permissions + content: + application/json: + schema: + $ref: '#/components/schemas/CurrentUser' + example: + name: "john.doe" + context: "artipie" + email: "john.doe@example.com" + groups: ["developers", "readers"] + permissions: + api_repository_permissions: true + api_user_permissions: false + api_role_permissions: false + api_alias_permissions: true + api_cache_permissions: true + api_search_permissions: true + '401': + $ref: '#/components/responses/UnauthorizedError' + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" /api/v1/users: get: summary: List all users. @@ -62,6 +102,10 @@ paths: $ref: "#/components/schemas/User" '404': description: User does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -95,6 +139,12 @@ paths: description: User successfully added '401': $ref: '#/components/responses/UnauthorizedError' + '403': + description: Insufficient permissions + content: + application/json: + schema: + $ref: "#/components/schemas/Error" default: description: unexpected error content: @@ -120,6 +170,10 @@ paths: description: User removed successfully '404': description: User does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -154,6 +208,10 @@ paths: description: Password changed successfully '404': description: User does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -182,6 +240,10 @@ paths: description: User disabled successfully '404': description: User does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -210,6 +272,10 @@ paths: description: User enabled successfully '404': description: User does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/Error" '401': $ref: '#/components/responses/UnauthorizedError' default: @@ -329,8 +395,62 @@ components: description: Password encoding type for the new password enum: ["plain", "sha256"] example: "sha256" + CurrentUser: + type: object + description: Current authenticated user info with effective permissions + required: + - name + properties: + name: + type: string + description: Authenticated username + example: "john.doe" + context: + type: string + description: Authentication context (e.g., artipie, okta) + example: "artipie" + email: + type: string + format: email + description: User email address (if available) + example: "john.doe@example.com" + groups: + type: array + items: + type: string + description: User's role/group memberships + example: ["developers", "readers"] + permissions: + type: object + description: | + Effective permission summary. Each key indicates whether the user + has READ access to that API category. UI can use this to show/hide + menu items and features. + properties: + api_repository_permissions: + type: boolean + description: Can manage repositories + api_user_permissions: + type: boolean + description: Can manage users + api_role_permissions: + type: boolean + description: Can manage roles + api_alias_permissions: + type: boolean + description: Can manage storage aliases + api_cache_permissions: + type: boolean + description: Can manage caches + api_search_permissions: + type: boolean + description: Can use search responses: UnauthorizedError: description: "Access token is missing or invalid" + content: + application/json: + schema: + $ref: "#/components/schemas/Error" security: - bearerAuth: [] \ No newline at end of file diff --git a/artipie-main/src/test/java/com/artipie/api/RestApiServerBase.java b/artipie-main/src/test/java/com/artipie/api/RestApiServerBase.java index ba3e842f2..209e89fb3 100644 --- a/artipie-main/src/test/java/com/artipie/api/RestApiServerBase.java +++ b/artipie-main/src/test/java/com/artipie/api/RestApiServerBase.java @@ -217,7 +217,9 @@ vertx, new JWTAuthOptions().addPubSecKey( ), Optional.empty(), NoopCooldownService.INSTANCE, - this.settings() + this.settings(), + com.artipie.index.ArtifactIndex.NOP, + null ), context.succeedingThenComplete() ); diff --git a/artipie-main/src/test/java/com/artipie/api/perms/RestApiPermissionTest.java b/artipie-main/src/test/java/com/artipie/api/perms/RestApiPermissionTest.java index 93b0bd6bd..4e7382067 100644 --- a/artipie-main/src/test/java/com/artipie/api/perms/RestApiPermissionTest.java +++ b/artipie-main/src/test/java/com/artipie/api/perms/RestApiPermissionTest.java @@ -177,6 +177,55 @@ void permissionsWithSeveralActionsWorksCorrect() { ), new IsEqual<>(true) ); + final ApiSearchPermission search = new ApiSearchPermission(Set.of("read")); + MatcherAssert.assertThat( + "Search implies read", + search.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)), + new IsEqual<>(true) + ); + MatcherAssert.assertThat( + "Search not implies write", + search.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.WRITE)), + new IsEqual<>(false) + ); + final ApiSearchPermission searchAll = new ApiSearchPermission(Set.of("*")); + MatcherAssert.assertThat( + "Search wildcard implies write", + searchAll.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.WRITE)), + new IsEqual<>(true) + ); + } + + @ParameterizedTest + @EnumSource(ApiSearchPermission.SearchAction.class) + void searchPermissionWorksCorrect(final ApiSearchPermission.SearchAction action) { + MatcherAssert.assertThat( + "All implies any other action", + new ApiSearchPermission(ApiSearchPermission.SearchAction.ALL).implies( + new ApiSearchPermission(action) + ), + new IsEqual<>(true) + ); + if (action != ApiSearchPermission.SearchAction.ALL) { + MatcherAssert.assertThat( + "Any other action does not imply all", + new ApiSearchPermission(action).implies( + new ApiSearchPermission(ApiSearchPermission.SearchAction.ALL) + ), + new IsEqual<>(false) + ); + for (final ApiSearchPermission.SearchAction item + : ApiSearchPermission.SearchAction.values()) { + if (item != action) { + MatcherAssert.assertThat( + "Action not implies other action", + new ApiSearchPermission(action) + .implies(new ApiSearchPermission(item)), + new IsEqual<>(false) + ); + } + } + } } @Test @@ -196,6 +245,11 @@ void notImpliesOtherClassPermission() { .implies(new ApiUserPermission(ApiUserPermission.UserAction.ALL)), new IsEqual<>(false) ); + MatcherAssert.assertThat( + new ApiSearchPermission(ApiSearchPermission.SearchAction.ALL) + .implies(new ApiRolePermission(ApiRolePermission.RoleAction.ALL)), + new IsEqual<>(false) + ); } } diff --git a/artipie-main/src/test/java/com/artipie/api/v1/ApiResponseTest.java b/artipie-main/src/test/java/com/artipie/api/v1/ApiResponseTest.java new file mode 100644 index 000000000..1cb48ac54 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/ApiResponseTest.java @@ -0,0 +1,71 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.Test; +import java.util.List; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +class ApiResponseTest { + + @Test + void createsErrorResponse() { + final JsonObject err = ApiResponse.error(404, "NOT_FOUND", "Repo not found"); + assertThat(err.getInteger("status"), is(404)); + assertThat(err.getString("error"), is("NOT_FOUND")); + assertThat(err.getString("message"), is("Repo not found")); + } + + @Test + void createsPaginatedResponse() { + final JsonArray items = new JsonArray().add("a").add("b"); + final JsonObject page = ApiResponse.paginated(items, 0, 20, 42); + assertThat(page.getJsonArray("items").size(), is(2)); + assertThat(page.getInteger("page"), is(0)); + assertThat(page.getInteger("size"), is(20)); + assertThat(page.getInteger("total"), is(42)); + assertThat(page.getBoolean("hasMore"), is(true)); + } + + @Test + void paginatedHasMoreFalseOnLastPage() { + final JsonArray items = new JsonArray().add("x"); + final JsonObject page = ApiResponse.paginated(items, 2, 20, 41); + assertThat(page.getBoolean("hasMore"), is(false)); + } + + @Test + void slicesList() { + final List all = List.of("a", "b", "c", "d", "e"); + final JsonArray items = ApiResponse.sliceToArray(all, 1, 2); + assertThat(items.size(), is(2)); + assertThat(items.getString(0), is("c")); + assertThat(items.getString(1), is("d")); + } + + @Test + void slicesListBeyondEnd() { + final List all = List.of("a", "b"); + final JsonArray items = ApiResponse.sliceToArray(all, 1, 20); + assertThat(items.size(), is(0)); + } + + @Test + void clampsPageSize() { + assertThat(ApiResponse.clampSize(200), is(100)); + assertThat(ApiResponse.clampSize(-5), is(20)); + assertThat(ApiResponse.clampSize(50), is(50)); + } + + @Test + void parsesIntParam() { + assertThat(ApiResponse.intParam("10", 20), is(10)); + assertThat(ApiResponse.intParam(null, 20), is(20)); + assertThat(ApiResponse.intParam("abc", 20), is(20)); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/ArtifactHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/ArtifactHandlerTest.java new file mode 100644 index 000000000..8a0278a72 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/ArtifactHandlerTest.java @@ -0,0 +1,67 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.junit5.VertxTestContext; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link ArtifactHandler}. + */ +public final class ArtifactHandlerTest extends AsyncApiTestBase { + + /** + * Valid repo body: maven-proxy with fs storage. + */ + private static final JsonObject VALID_BODY = new JsonObject() + .put( + "repo", + new JsonObject() + .put("type", "maven-proxy") + .put("storage", new JsonObject().put("type", "fs").put("path", "/tmp")) + ); + + @Test + void treeEndpointReturns200(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: create the repo so it exists + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/myrepo") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(VALID_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, put.statusCode()); + // Step 2: call the tree endpoint + final HttpResponse res = client + .get(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/myrepo/tree") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body.getJsonArray("items"), "Response must have 'items' array"); + ctx.completeNow(); + } + + @Test + void artifactDetailRequiresPath(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/repositories/myrepo/artifact", + res -> Assertions.assertEquals(400, res.statusCode()) + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/AsyncApiTestBase.java b/artipie-main/src/test/java/com/artipie/api/v1/AsyncApiTestBase.java new file mode 100644 index 000000000..502727637 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/AsyncApiTestBase.java @@ -0,0 +1,209 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import com.artipie.asto.Storage; +import com.artipie.asto.memory.InMemoryStorage; +import com.artipie.cooldown.NoopCooldownService; +import com.artipie.http.auth.AuthUser; +import com.artipie.http.auth.Authentication; +import com.artipie.index.ArtifactIndex; +import com.artipie.nuget.RandomFreePort; +import com.artipie.security.policy.Policy; +import com.artipie.settings.ArtipieSecurity; +import com.artipie.test.TestArtipieCaches; +import com.artipie.test.TestSettings; +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.core.net.NetClient; +import io.vertx.ext.auth.PubSecKeyOptions; +import io.vertx.ext.auth.jwt.JWTAuth; +import io.vertx.ext.auth.jwt.JWTAuthOptions; +import io.vertx.ext.web.client.HttpRequest; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Test base for AsyncApiVerticle integration tests. + */ +@ExtendWith(VertxExtension.class) +public class AsyncApiTestBase { + + /** + * Test timeout in seconds. + */ + static final long TEST_TIMEOUT = Duration.ofSeconds(5).toSeconds(); + + /** + * Service host. + */ + static final String HOST = "localhost"; + + /** + * Hardcoded JWT token for test user "artipie" with context "test". + * Issued with HS256, secret "some secret", no expiry. + */ + static final String TEST_TOKEN = + "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9" + + ".eyJzdWIiOiJhcnRpcGllIiwiY29udGV4dCI6InRlc3QiLCJpYXQiOjE2ODIwODgxNTh9" + + ".QjQPLQ0tQFbiRIWpE-GUtUFXvUXvXP4p7va_DOBHjTM"; + + /** + * Server port. + */ + private int port; + + @BeforeEach + final void setUp(final Vertx vertx, final VertxTestContext ctx) throws Exception { + this.port = new RandomFreePort().value(); + final Storage storage = new InMemoryStorage(); + final ArtipieSecurity security = new ArtipieSecurity() { + @Override + public Authentication authentication() { + return (name, pswd) -> Optional.of(new AuthUser("artipie", "test")); + } + + @Override + public Policy policy() { + return Policy.FREE; + } + + @Override + public Optional policyStorage() { + return Optional.of(storage); + } + }; + final JWTAuth jwt = JWTAuth.create( + vertx, new JWTAuthOptions().addPubSecKey( + new PubSecKeyOptions().setAlgorithm("HS256").setBuffer("some secret") + ) + ); + vertx.deployVerticle( + new AsyncApiVerticle( + new TestArtipieCaches(), + storage, + this.port, + security, + Optional.empty(), + jwt, + Optional.empty(), + NoopCooldownService.INSTANCE, + new TestSettings(), + ArtifactIndex.NOP, + null + ), + ctx.succeedingThenComplete() + ); + this.waitServer(vertx); + } + + /** + * Get test server port. + * @return The port int value + */ + final int port() { + return this.port; + } + + /** + * Perform HTTP request with test token. + * @param vertx Vertx instance + * @param ctx Test context + * @param method HTTP method + * @param path Request path + * @param assertion Response assertion + * @throws Exception On error + */ + final void request(final Vertx vertx, final VertxTestContext ctx, + final HttpMethod method, final String path, + final Consumer> assertion) throws Exception { + this.request(vertx, ctx, method, path, null, assertion); + } + + /** + * Perform HTTP request with test token and body. + * @param vertx Vertx instance + * @param ctx Test context + * @param method HTTP method + * @param path Request path + * @param body Request body (nullable) + * @param assertion Response assertion + * @throws Exception On error + */ + final void request(final Vertx vertx, final VertxTestContext ctx, + final HttpMethod method, final String path, final JsonObject body, + final Consumer> assertion) throws Exception { + this.request(vertx, ctx, method, path, body, TEST_TOKEN, assertion); + } + + /** + * Perform HTTP request with specified token and body. + * @param vertx Vertx instance + * @param ctx Test context + * @param method HTTP method + * @param path Request path + * @param body Request body (nullable) + * @param token JWT token (nullable for no auth) + * @param assertion Response assertion + * @throws Exception On error + */ + final void request(final Vertx vertx, final VertxTestContext ctx, + final HttpMethod method, final String path, final JsonObject body, + final String token, + final Consumer> assertion) throws Exception { + final HttpRequest req = WebClient.create(vertx) + .request(method, this.port, HOST, path); + if (token != null) { + req.bearerTokenAuthentication(token); + } + final var future = body != null ? req.sendJsonObject(body) : req.send(); + future.onSuccess(res -> { + assertion.accept(res); + ctx.completeNow(); + }) + .onFailure(ctx::failNow) + .toCompletionStage().toCompletableFuture() + .get(TEST_TIMEOUT, TimeUnit.SECONDS); + } + + /** + * Wait for server to be available on the test port. + * @param vertx Vertx instance + */ + private void waitServer(final Vertx vertx) { + final AtomicReference ready = new AtomicReference<>(false); + final NetClient client = vertx.createNetClient(); + final long deadline = System.currentTimeMillis() + Duration.ofMinutes(1).toMillis(); + while (!ready.get() && System.currentTimeMillis() < deadline) { + client.connect(this.port, HOST, ar -> { + if (ar.succeeded()) { + ready.set(true); + } + }); + if (!ready.get()) { + try { + TimeUnit.MILLISECONDS.sleep(100); + } catch (final InterruptedException exc) { + break; + } + } + } + if (!ready.get()) { + Assertions.fail("Server not reachable on port " + this.port); + } + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/AsyncApiVerticleTest.java b/artipie-main/src/test/java/com/artipie/api/v1/AsyncApiVerticleTest.java new file mode 100644 index 000000000..932f1debf --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/AsyncApiVerticleTest.java @@ -0,0 +1,37 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +/** + * Tests for AsyncApiVerticle health endpoint and auth. + */ +class AsyncApiVerticleTest extends AsyncApiTestBase { + + @Test + void healthEndpointReturnsOk(final Vertx vertx, + final VertxTestContext ctx) throws Exception { + request(vertx, ctx, HttpMethod.GET, "/api/v1/health", null, null, + res -> { + assertThat(res.statusCode(), is(200)); + assertThat( + res.bodyAsJsonObject().getString("status"), is("ok") + ); + }); + } + + @Test + void returns401WithoutToken(final Vertx vertx, + final VertxTestContext ctx) throws Exception { + request(vertx, ctx, HttpMethod.GET, "/api/v1/repositories", null, null, + res -> assertThat(res.statusCode(), is(401))); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/AuthHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/AuthHandlerTest.java new file mode 100644 index 000000000..571fc9d57 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/AuthHandlerTest.java @@ -0,0 +1,69 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Tests for AuthHandler endpoints. + */ +class AuthHandlerTest extends AsyncApiTestBase { + + @Test + void tokenEndpointReturnsJwt(final Vertx vertx, + final VertxTestContext ctx) throws Exception { + final JsonObject body = new JsonObject() + .put("name", "artipie") + .put("pass", "secret"); + request(vertx, ctx, HttpMethod.POST, "/api/v1/auth/token", body, null, + res -> { + assertThat(res.statusCode(), is(200)); + final JsonObject json = res.bodyAsJsonObject(); + assertThat(json.getString("token"), notNullValue()); + }); + } + + @Test + void providersEndpointReturnsArray(final Vertx vertx, + final VertxTestContext ctx) throws Exception { + request(vertx, ctx, HttpMethod.GET, "/api/v1/auth/providers", null, null, + res -> { + assertThat(res.statusCode(), is(200)); + final JsonObject json = res.bodyAsJsonObject(); + assertThat(json.getJsonArray("providers"), notNullValue()); + assertThat(json.getJsonArray("providers").size() > 0, is(true)); + }); + } + + @Test + void meEndpointReturnsCurrentUser(final Vertx vertx, + final VertxTestContext ctx) throws Exception { + request(vertx, ctx, HttpMethod.GET, "/api/v1/auth/me", + res -> { + assertThat(res.statusCode(), is(200)); + final JsonObject json = res.bodyAsJsonObject(); + assertThat(json.getString("name"), is("artipie")); + }); + } + + @Test + void meEndpointReturnsPermissions(final Vertx vertx, + final VertxTestContext ctx) throws Exception { + request(vertx, ctx, HttpMethod.GET, "/api/v1/auth/me", + res -> { + assertThat(res.statusCode(), is(200)); + final JsonObject json = res.bodyAsJsonObject(); + assertThat(json.getJsonObject("permissions"), notNullValue()); + }); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/CooldownHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/CooldownHandlerTest.java new file mode 100644 index 000000000..280a897e1 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/CooldownHandlerTest.java @@ -0,0 +1,68 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link CooldownHandler}. + * @since 1.21.0 + */ +public final class CooldownHandlerTest extends AsyncApiTestBase { + + @Test + void overviewEndpointReturns200(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/cooldown/overview", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final io.vertx.core.json.JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull( + body.getJsonArray("repos"), + "Response must have 'repos' array" + ); + } + ); + } + + @Test + void blockedEndpointReturnsPaginated(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/cooldown/blocked", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final io.vertx.core.json.JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull( + body.getJsonArray("items"), + "Response must have 'items' array" + ); + Assertions.assertTrue( + body.containsKey("page"), + "Response must have 'page' field" + ); + Assertions.assertTrue( + body.containsKey("size"), + "Response must have 'size' field" + ); + Assertions.assertTrue( + body.containsKey("total"), + "Response must have 'total' field" + ); + Assertions.assertTrue( + body.containsKey("hasMore"), + "Response must have 'hasMore' field" + ); + } + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/DashboardHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/DashboardHandlerTest.java new file mode 100644 index 000000000..40ba41537 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/DashboardHandlerTest.java @@ -0,0 +1,80 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link DashboardHandler}. + */ +public final class DashboardHandlerTest extends AsyncApiTestBase { + + @Test + void statsReturnsRepoCount(final Vertx vertx, final VertxTestContext ctx) throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/dashboard/stats", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body); + Assertions.assertTrue( + body.containsKey("repo_count"), + "Response must contain repo_count" + ); + Assertions.assertEquals(0, body.getInteger("repo_count")); + Assertions.assertEquals(0, body.getInteger("artifact_count")); + Assertions.assertEquals("0", body.getString("total_storage")); + Assertions.assertEquals(0, body.getInteger("blocked_count")); + } + ); + } + + @Test + void requestsReturnsPlaceholder(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/dashboard/requests", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body); + Assertions.assertEquals("24h", body.getString("period")); + Assertions.assertNotNull( + body.getJsonArray("data"), + "Response must contain data array" + ); + } + ); + } + + @Test + void reposByTypeReturnsEmptyWhenNoRepos(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/dashboard/repos-by-type", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body); + Assertions.assertNotNull( + body.getJsonObject("types"), + "Response must contain types object" + ); + Assertions.assertTrue( + body.getJsonObject("types").isEmpty(), + "types should be empty when no repos exist" + ); + } + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/RepositoryHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/RepositoryHandlerTest.java new file mode 100644 index 000000000..98ab61901 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/RepositoryHandlerTest.java @@ -0,0 +1,145 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.junit5.VertxTestContext; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link RepositoryHandler}. + */ +public final class RepositoryHandlerTest extends AsyncApiTestBase { + + /** + * Valid repo body: maven-proxy with fs storage. + */ + private static final JsonObject VALID_BODY = new JsonObject() + .put( + "repo", + new JsonObject() + .put("type", "maven-proxy") + .put("storage", new JsonObject().put("type", "fs").put("path", "/tmp")) + ); + + @Test + void listReposReturnsPaginatedFormat(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/repositories", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body.getJsonArray("items")); + Assertions.assertTrue(body.containsKey("page")); + Assertions.assertTrue(body.containsKey("size")); + Assertions.assertTrue(body.containsKey("total")); + Assertions.assertTrue(body.containsKey("hasMore")); + } + ); + } + + @Test + void createRepoAndGet(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the repo + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/myrepo") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(VALID_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, put.statusCode()); + // Step 2: GET the repo + final HttpResponse get = client + .get(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/myrepo") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, get.statusCode()); + final String body = get.bodyAsString(); + Assertions.assertNotNull(body); + Assertions.assertFalse(body.isBlank()); + ctx.completeNow(); + } + + @Test + void headReturns200IfExists(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the repo + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/headrepo") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(VALID_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, put.statusCode()); + // Step 2: HEAD it + final HttpResponse head = client + .head(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/headrepo") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, head.statusCode()); + ctx.completeNow(); + } + + @Test + void headReturns404IfMissing(final Vertx vertx, final VertxTestContext ctx) throws Exception { + this.request( + vertx, ctx, + HttpMethod.HEAD, "/api/v1/repositories/nonexistent-repo-xyz", + res -> Assertions.assertEquals(404, res.statusCode()) + ); + } + + @Test + void deleteRepo(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the repo + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/deleteme") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(VALID_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, put.statusCode()); + // Step 2: DELETE it + final HttpResponse del = client + .delete(this.port(), AsyncApiTestBase.HOST, "/api/v1/repositories/deleteme") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, del.statusCode()); + ctx.completeNow(); + } + + @Test + void getRepoReturns404IfMissing(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/repositories/no-such-repo-abc", + res -> { + Assertions.assertEquals(404, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertEquals("NOT_FOUND", body.getString("error")); + Assertions.assertEquals(404, body.getInteger("status")); + Assertions.assertNotNull(body.getString("message")); + } + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/RoleHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/RoleHandlerTest.java new file mode 100644 index 000000000..20c238e31 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/RoleHandlerTest.java @@ -0,0 +1,93 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.junit5.VertxTestContext; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link RoleHandler}. + */ +public final class RoleHandlerTest extends AsyncApiTestBase { + + /** + * PUT body for creating a test role. + */ + private static final JsonObject ROLE_BODY = new JsonObject() + .put( + "permissions", + new JsonObject().put( + "api_repository", + new JsonObject().put("read", true).put("write", false) + ) + ); + + @Test + void listRolesReturnsPaginatedFormat(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/roles", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body.getJsonArray("items")); + Assertions.assertTrue(body.containsKey("page")); + Assertions.assertTrue(body.containsKey("size")); + Assertions.assertTrue(body.containsKey("total")); + Assertions.assertTrue(body.containsKey("hasMore")); + } + ); + } + + @Test + void getRoleReturns404WhenMissing(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/roles/nonexistent", + res -> { + Assertions.assertEquals(404, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertEquals("NOT_FOUND", body.getString("error")); + Assertions.assertEquals(404, body.getInteger("status")); + Assertions.assertNotNull(body.getString("message")); + } + ); + } + + @Test + void createAndGetRole(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the role + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/roles/testrole") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(ROLE_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(201, put.statusCode()); + // Step 2: GET the role + final HttpResponse get = client + .get(this.port(), AsyncApiTestBase.HOST, "/api/v1/roles/testrole") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, get.statusCode()); + final String body = get.bodyAsString(); + Assertions.assertNotNull(body); + Assertions.assertFalse(body.isBlank()); + ctx.completeNow(); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/SearchHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/SearchHandlerTest.java new file mode 100644 index 000000000..3e9505040 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/SearchHandlerTest.java @@ -0,0 +1,120 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link SearchHandler}. + * @since 1.21.0 + */ +public final class SearchHandlerTest extends AsyncApiTestBase { + + @Test + void searchRequiresQueryParam(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/search", + res -> Assertions.assertEquals(400, res.statusCode()) + ); + } + + @Test + void searchReturnsResults(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/search?q=test", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull( + body.getJsonArray("items"), + "Response must have 'items' array" + ); + Assertions.assertTrue( + body.containsKey("page"), + "Response must have 'page' field" + ); + Assertions.assertTrue( + body.containsKey("size"), + "Response must have 'size' field" + ); + Assertions.assertTrue( + body.containsKey("total"), + "Response must have 'total' field" + ); + Assertions.assertTrue( + body.containsKey("hasMore"), + "Response must have 'hasMore' field" + ); + } + ); + } + + @Test + void reindexReturns202(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.POST, "/api/v1/search/reindex", + res -> { + Assertions.assertEquals(202, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertEquals( + "started", body.getString("status"), + "Response status must be 'started'" + ); + } + ); + } + + @Test + void locateRequiresPathParam(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/search/locate", + res -> Assertions.assertEquals(400, res.statusCode()) + ); + } + + @Test + void locateReturnsRepositories(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/search/locate?path=com/example/lib/1.0/lib.jar", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull( + body.getJsonArray("repositories"), + "Response must have 'repositories' array" + ); + Assertions.assertTrue( + body.containsKey("count"), + "Response must have 'count' field" + ); + } + ); + } + + @Test + void statsReturnsJsonObject(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/search/stats", + res -> Assertions.assertEquals(200, res.statusCode()) + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/SettingsHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/SettingsHandlerTest.java new file mode 100644 index 000000000..2ed31a41b --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/SettingsHandlerTest.java @@ -0,0 +1,56 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link SettingsHandler}. + * + * @since 1.21 + */ +public final class SettingsHandlerTest extends AsyncApiTestBase { + + @Test + void getSettingsReturnsPort(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/settings", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body); + Assertions.assertTrue( + body.containsKey("port"), + "Response must contain 'port' field" + ); + } + ); + } + + @Test + void getSettingsReturnsVersion(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/settings", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body); + Assertions.assertNotNull( + body.getString("version"), + "Response must contain non-null 'version' field" + ); + } + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/StorageAliasHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/StorageAliasHandlerTest.java new file mode 100644 index 000000000..4550f0260 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/StorageAliasHandlerTest.java @@ -0,0 +1,100 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.junit5.VertxTestContext; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link StorageAliasHandler}. + */ +public final class StorageAliasHandlerTest extends AsyncApiTestBase { + + /** + * Sample alias configuration body. + */ + private static final JsonObject ALIAS_BODY = new JsonObject() + .put("type", "fs") + .put("path", "/var/artipie/data"); + + @Test + void listGlobalAliasesReturnsArray(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/storages", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonArray body = res.bodyAsJsonArray(); + Assertions.assertNotNull(body, "Response body must be a JSON array"); + } + ); + } + + @Test + void createAndListGlobalAlias(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the alias + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/storages/default") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(ALIAS_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, put.statusCode()); + // Step 2: GET list and verify alias appears + final HttpResponse get = client + .get(this.port(), AsyncApiTestBase.HOST, "/api/v1/storages") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, get.statusCode()); + final JsonArray aliases = get.bodyAsJsonArray(); + Assertions.assertNotNull(aliases); + final boolean found = aliases.stream() + .anyMatch(obj -> { + if (obj instanceof JsonObject) { + final String alias = ((JsonObject) obj).getString("name"); + return "default".equals(alias); + } + return false; + }); + Assertions.assertTrue(found, "Alias 'default' should appear in the list after creation"); + ctx.completeNow(); + } + + @Test + void deleteGlobalAlias(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the alias + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/storages/default") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(ALIAS_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, put.statusCode()); + // Step 2: DELETE the alias + final HttpResponse del = client + .delete(this.port(), AsyncApiTestBase.HOST, "/api/v1/storages/default") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, del.statusCode()); + ctx.completeNow(); + } +} diff --git a/artipie-main/src/test/java/com/artipie/api/v1/UserHandlerTest.java b/artipie-main/src/test/java/com/artipie/api/v1/UserHandlerTest.java new file mode 100644 index 000000000..f96331c35 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/api/v1/UserHandlerTest.java @@ -0,0 +1,111 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.api.v1; + +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.junit5.VertxTestContext; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for {@link UserHandler}. + */ +public final class UserHandlerTest extends AsyncApiTestBase { + + /** + * PUT body for creating a test user. + */ + private static final JsonObject USER_BODY = new JsonObject() + .put("pass", "secret123") + .put("type", "plain") + .put("email", "test@example.com"); + + @Test + void listUsersReturnsPaginatedFormat(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/users", + res -> { + Assertions.assertEquals(200, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertNotNull(body.getJsonArray("items")); + Assertions.assertTrue(body.containsKey("page")); + Assertions.assertTrue(body.containsKey("size")); + Assertions.assertTrue(body.containsKey("total")); + Assertions.assertTrue(body.containsKey("hasMore")); + } + ); + } + + @Test + void getUserReturns404WhenMissing(final Vertx vertx, final VertxTestContext ctx) + throws Exception { + this.request( + vertx, ctx, + HttpMethod.GET, "/api/v1/users/nonexistent", + res -> { + Assertions.assertEquals(404, res.statusCode()); + final JsonObject body = res.bodyAsJsonObject(); + Assertions.assertEquals("NOT_FOUND", body.getString("error")); + Assertions.assertEquals(404, body.getInteger("status")); + Assertions.assertNotNull(body.getString("message")); + } + ); + } + + @Test + void createAndGetUser(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the user + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/users/testuser") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(USER_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(201, put.statusCode()); + // Step 2: GET the user + final HttpResponse get = client + .get(this.port(), AsyncApiTestBase.HOST, "/api/v1/users/testuser") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, get.statusCode()); + final String body = get.bodyAsString(); + Assertions.assertNotNull(body); + Assertions.assertFalse(body.isBlank()); + ctx.completeNow(); + } + + @Test + void deleteUser(final Vertx vertx, final VertxTestContext ctx) throws Exception { + final WebClient client = WebClient.create(vertx); + // Step 1: PUT the user + final HttpResponse put = client + .put(this.port(), AsyncApiTestBase.HOST, "/api/v1/users/testuser") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .sendJsonObject(USER_BODY) + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(201, put.statusCode()); + // Step 2: DELETE the user + final HttpResponse del = client + .delete(this.port(), AsyncApiTestBase.HOST, "/api/v1/users/testuser") + .bearerTokenAuthentication(AsyncApiTestBase.TEST_TOKEN) + .send() + .toCompletionStage().toCompletableFuture() + .get(AsyncApiTestBase.TEST_TIMEOUT, TimeUnit.SECONDS); + Assertions.assertEquals(200, del.statusCode()); + ctx.completeNow(); + } +} diff --git a/artipie-main/src/test/java/com/artipie/cache/CacheInvalidationPubSubTest.java b/artipie-main/src/test/java/com/artipie/cache/CacheInvalidationPubSubTest.java new file mode 100644 index 000000000..69f3de0bc --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/cache/CacheInvalidationPubSubTest.java @@ -0,0 +1,294 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cache; + +import com.artipie.asto.misc.Cleanable; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.awaitility.Awaitility; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for {@link CacheInvalidationPubSub}. + * Uses a Testcontainers Valkey/Redis container. + * + * @since 1.20.13 + */ +@Testcontainers +@SuppressWarnings("PMD.AvoidDuplicateLiterals") +final class CacheInvalidationPubSubTest { + + /** + * Valkey container. + */ + @Container + @SuppressWarnings("rawtypes") + private static final GenericContainer VALKEY = + new GenericContainer<>("redis:7-alpine") + .withExposedPorts(6379); + + /** + * First Valkey connection (simulates instance A). + */ + private ValkeyConnection connA; + + /** + * Second Valkey connection (simulates instance B). + */ + private ValkeyConnection connB; + + /** + * Pub/sub for instance A. + */ + private CacheInvalidationPubSub pubsubA; + + /** + * Pub/sub for instance B. + */ + private CacheInvalidationPubSub pubsubB; + + @BeforeEach + void setUp() { + final String host = VALKEY.getHost(); + final int port = VALKEY.getMappedPort(6379); + this.connA = new ValkeyConnection(host, port, Duration.ofSeconds(5)); + this.connB = new ValkeyConnection(host, port, Duration.ofSeconds(5)); + this.pubsubA = new CacheInvalidationPubSub(this.connA); + this.pubsubB = new CacheInvalidationPubSub(this.connB); + } + + @AfterEach + void tearDown() { + if (this.pubsubA != null) { + this.pubsubA.close(); + } + if (this.pubsubB != null) { + this.pubsubB.close(); + } + if (this.connA != null) { + this.connA.close(); + } + if (this.connB != null) { + this.connB.close(); + } + } + + @Test + void invalidatesRemoteCacheForSpecificKey() { + final RecordingCleanable cache = new RecordingCleanable(); + this.pubsubB.register("auth", cache); + this.pubsubA.publish("auth", "user:alice"); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> MatcherAssert.assertThat( + "Instance B should have received invalidation for 'user:alice'", + cache.invalidated(), + Matchers.contains("user:alice") + ) + ); + } + + @Test + void selfMessagesAreIgnored() throws Exception { + final RecordingCleanable cache = new RecordingCleanable(); + this.pubsubA.register("auth", cache); + this.pubsubA.publish("auth", "user:bob"); + Thread.sleep(1000); + MatcherAssert.assertThat( + "Self-published messages should not trigger local invalidation", + cache.invalidated(), + Matchers.empty() + ); + } + + @Test + void invalidateAllBroadcastsToRemoteInstances() { + final RecordingCleanable cache = new RecordingCleanable(); + this.pubsubB.register("policy", cache); + this.pubsubA.publishAll("policy"); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> MatcherAssert.assertThat( + "Instance B should have received invalidateAll", + cache.invalidatedAll(), + Matchers.is(1) + ) + ); + } + + @Test + void unknownCacheTypeIsIgnored() throws Exception { + final RecordingCleanable cache = new RecordingCleanable(); + this.pubsubB.register("auth", cache); + this.pubsubA.publish("unknown-type", "some-key"); + Thread.sleep(1000); + MatcherAssert.assertThat( + "Unknown cache type should not trigger any invalidation", + cache.invalidated(), + Matchers.empty() + ); + MatcherAssert.assertThat( + "Unknown cache type should not trigger invalidateAll", + cache.invalidatedAll(), + Matchers.is(0) + ); + } + + @Test + void multipleKeysAreDeliveredInOrder() { + final RecordingCleanable cache = new RecordingCleanable(); + this.pubsubB.register("filters", cache); + this.pubsubA.publish("filters", "repo-one"); + this.pubsubA.publish("filters", "repo-two"); + this.pubsubA.publish("filters", "repo-three"); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> MatcherAssert.assertThat( + "All three keys should be delivered", + cache.invalidated(), + Matchers.contains("repo-one", "repo-two", "repo-three") + ) + ); + } + + @Test + void multipleCacheTypesAreRoutedCorrectly() { + final RecordingCleanable auth = new RecordingCleanable(); + final RecordingCleanable filters = new RecordingCleanable(); + this.pubsubB.register("auth", auth); + this.pubsubB.register("filters", filters); + this.pubsubA.publish("auth", "user:charlie"); + this.pubsubA.publish("filters", "repo-x"); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> { + MatcherAssert.assertThat( + "Auth cache should only receive auth key", + auth.invalidated(), + Matchers.contains("user:charlie") + ); + MatcherAssert.assertThat( + "Filters cache should only receive filters key", + filters.invalidated(), + Matchers.contains("repo-x") + ); + } + ); + } + + @Test + void closeStopsReceivingMessages() throws Exception { + final RecordingCleanable cache = new RecordingCleanable(); + this.pubsubB.register("auth", cache); + this.pubsubB.close(); + this.pubsubA.publish("auth", "user:after-close"); + Thread.sleep(1000); + MatcherAssert.assertThat( + "Closed instance should not receive messages", + cache.invalidated(), + Matchers.empty() + ); + this.pubsubB = null; + } + + @Test + void publishingCleanableDelegatesAndPublishes() { + final RecordingCleanable inner = new RecordingCleanable(); + final RecordingCleanable remote = new RecordingCleanable(); + this.pubsubB.register("auth", remote); + final PublishingCleanable wrapper = + new PublishingCleanable(inner, this.pubsubA, "auth"); + wrapper.invalidate("user:delta"); + MatcherAssert.assertThat( + "Inner cache should be invalidated directly", + inner.invalidated(), + Matchers.contains("user:delta") + ); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> MatcherAssert.assertThat( + "Remote cache should receive invalidation via pub/sub", + remote.invalidated(), + Matchers.contains("user:delta") + ) + ); + } + + @Test + void publishingCleanableInvalidateAllDelegatesAndPublishes() { + final RecordingCleanable inner = new RecordingCleanable(); + final RecordingCleanable remote = new RecordingCleanable(); + this.pubsubB.register("policy", remote); + final PublishingCleanable wrapper = + new PublishingCleanable(inner, this.pubsubA, "policy"); + wrapper.invalidateAll(); + MatcherAssert.assertThat( + "Inner cache should receive invalidateAll", + inner.invalidatedAll(), + Matchers.is(1) + ); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> MatcherAssert.assertThat( + "Remote cache should receive invalidateAll via pub/sub", + remote.invalidatedAll(), + Matchers.is(1) + ) + ); + } + + /** + * Recording implementation of {@link Cleanable} for test verification. + */ + private static final class RecordingCleanable implements Cleanable { + /** + * Keys that were invalidated. + */ + private final List keys; + + /** + * Count of invalidateAll calls. + */ + private int allCount; + + RecordingCleanable() { + this.keys = Collections.synchronizedList(new ArrayList<>(8)); + } + + @Override + public void invalidate(final String key) { + this.keys.add(key); + } + + @Override + public void invalidateAll() { + this.allCount += 1; + } + + List invalidated() { + return this.keys; + } + + int invalidatedAll() { + return this.allCount; + } + } +} diff --git a/artipie-main/src/test/java/com/artipie/cluster/DbNodeRegistryTest.java b/artipie-main/src/test/java/com/artipie/cluster/DbNodeRegistryTest.java new file mode 100644 index 000000000..d4c590fe3 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/cluster/DbNodeRegistryTest.java @@ -0,0 +1,242 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cluster; + +import com.artipie.db.PostgreSQLTestConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.List; +import javax.sql.DataSource; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.hamcrest.core.IsEqual; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Tests for {@link DbNodeRegistry}. + * + * @since 1.20.13 + */ +@SuppressWarnings("PMD.TooManyMethods") +@Testcontainers +class DbNodeRegistryTest { + + /** + * PostgreSQL test container. + */ + @Container + static final PostgreSQLContainer POSTGRES = PostgreSQLTestConfig.createContainer(); + + /** + * Data source for tests. + */ + private DataSource source; + + /** + * Registry under test. + */ + private DbNodeRegistry registry; + + @BeforeEach + void setUp() throws SQLException { + final HikariConfig config = new HikariConfig(); + config.setJdbcUrl(POSTGRES.getJdbcUrl()); + config.setUsername(POSTGRES.getUsername()); + config.setPassword(POSTGRES.getPassword()); + config.setMaximumPoolSize(5); + config.setPoolName("DbNodeRegistryTest-Pool"); + this.source = new HikariDataSource(config); + // Drop and recreate for clean state + try (Connection conn = this.source.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS artipie_nodes"); + } + this.registry = new DbNodeRegistry(this.source); + this.registry.createTable(); + } + + @Test + void createsTableWithoutError() throws SQLException { + // Table already created in setUp; calling again should be idempotent + this.registry.createTable(); + } + + @Test + void registersNode() throws SQLException { + final NodeRegistry.NodeInfo node = new NodeRegistry.NodeInfo( + "node-1", "host-1", Instant.now(), Instant.now() + ); + this.registry.register(node); + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat(live, Matchers.hasSize(1)); + MatcherAssert.assertThat( + live.get(0).nodeId(), + new IsEqual<>("node-1") + ); + MatcherAssert.assertThat( + live.get(0).hostname(), + new IsEqual<>("host-1") + ); + } + + @Test + void upsertUpdatesExistingNode() throws SQLException { + final Instant started = Instant.now(); + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-1", started, started) + ); + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-updated", started, started) + ); + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat(live, Matchers.hasSize(1)); + MatcherAssert.assertThat( + live.get(0).hostname(), + new IsEqual<>("host-updated") + ); + } + + @Test + void registersMultipleNodes() throws SQLException { + final Instant now = Instant.now(); + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-1", now, now) + ); + this.registry.register( + new NodeRegistry.NodeInfo("node-2", "host-2", now, now) + ); + this.registry.register( + new NodeRegistry.NodeInfo("node-3", "host-3", now, now) + ); + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat(live, Matchers.hasSize(3)); + } + + @Test + void heartbeatUpdatesTimestamp() throws SQLException { + final Instant now = Instant.now(); + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-1", now, now) + ); + this.registry.heartbeat("node-1"); + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat(live, Matchers.hasSize(1)); + } + + @Test + void deregisterSetsStatusToStopped() throws SQLException { + final Instant now = Instant.now(); + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-1", now, now) + ); + this.registry.deregister("node-1"); + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat( + "Deregistered node should not appear in live nodes", + live, Matchers.hasSize(0) + ); + } + + @Test + void liveNodesExcludesStaleNodes() throws SQLException { + final Instant now = Instant.now(); + // Register two nodes + this.registry.register( + new NodeRegistry.NodeInfo("node-fresh", "host-1", now, now) + ); + this.registry.register( + new NodeRegistry.NodeInfo("node-stale", "host-2", now, now) + ); + // Manually set one node's heartbeat to far in the past + try (Connection conn = this.source.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate( + "UPDATE artipie_nodes SET last_heartbeat = TIMESTAMP '2020-01-01 00:00:00'" + + " WHERE node_id = 'node-stale'" + ); + } + // Only the fresh node should appear with a 30s timeout + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat(live, Matchers.hasSize(1)); + MatcherAssert.assertThat( + live.get(0).nodeId(), + new IsEqual<>("node-fresh") + ); + } + + @Test + void evictStaleRemovesOldNodes() throws SQLException { + final Instant now = Instant.now(); + this.registry.register( + new NodeRegistry.NodeInfo("node-fresh", "host-1", now, now) + ); + this.registry.register( + new NodeRegistry.NodeInfo("node-stale", "host-2", now, now) + ); + // Manually set one node's heartbeat to far in the past + try (Connection conn = this.source.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate( + "UPDATE artipie_nodes SET last_heartbeat = TIMESTAMP '2020-01-01 00:00:00'" + + " WHERE node_id = 'node-stale'" + ); + } + final int evicted = this.registry.evictStale(30_000L); + MatcherAssert.assertThat(evicted, new IsEqual<>(1)); + // Only the fresh node should remain + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat(live, Matchers.hasSize(1)); + MatcherAssert.assertThat( + live.get(0).nodeId(), + new IsEqual<>("node-fresh") + ); + } + + @Test + void evictStaleReturnsZeroWhenNothingToEvict() throws SQLException { + final Instant now = Instant.now(); + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-1", now, now) + ); + final int evicted = this.registry.evictStale(30_000L); + MatcherAssert.assertThat(evicted, new IsEqual<>(0)); + } + + @Test + void heartbeatForUnknownNodeDoesNotFail() throws SQLException { + // Should not throw; logs a warning + this.registry.heartbeat("nonexistent-node"); + } + + @Test + void reRegisterAfterDeregister() throws SQLException { + final Instant now = Instant.now(); + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-1", now, now) + ); + this.registry.deregister("node-1"); + MatcherAssert.assertThat( + this.registry.liveNodes(30_000L), Matchers.hasSize(0) + ); + // Re-register should bring the node back + this.registry.register( + new NodeRegistry.NodeInfo("node-1", "host-1", now, now) + ); + final List live = this.registry.liveNodes(30_000L); + MatcherAssert.assertThat(live, Matchers.hasSize(1)); + MatcherAssert.assertThat( + live.get(0).nodeId(), + new IsEqual<>("node-1") + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/cluster/NodeRegistryTest.java b/artipie-main/src/test/java/com/artipie/cluster/NodeRegistryTest.java new file mode 100644 index 000000000..3d6500806 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/cluster/NodeRegistryTest.java @@ -0,0 +1,48 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.cluster; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.emptyString; + +/** + * Tests for {@link NodeRegistry}. + */ +class NodeRegistryTest { + + @Test + void registersSelfOnConstruction() { + final NodeRegistry reg = new NodeRegistry("node-1", "localhost"); + assertThat(reg.size(), equalTo(1)); + assertThat(reg.nodeId(), equalTo("node-1")); + } + + @Test + void autoGeneratesNodeId() { + final NodeRegistry reg = new NodeRegistry("localhost"); + assertThat(reg.nodeId(), not(emptyString())); + assertThat(reg.size(), equalTo(1)); + } + + @Test + void heartbeatUpdatesTimestamp() { + final NodeRegistry reg = new NodeRegistry("node-1", "localhost"); + reg.heartbeat(); + assertThat(reg.activeNodes(), hasSize(1)); + } + + @Test + void activeNodesReturnsSelf() { + final NodeRegistry reg = new NodeRegistry("node-1", "localhost"); + assertThat(reg.activeNodes(), hasSize(1)); + assertThat(reg.activeNodes().get(0).nodeId(), equalTo("node-1")); + assertThat(reg.activeNodes().get(0).hostname(), equalTo("localhost")); + } +} diff --git a/artipie-main/src/test/java/com/artipie/cooldown/JdbcCooldownServiceTest.java b/artipie-main/src/test/java/com/artipie/cooldown/JdbcCooldownServiceTest.java index 5c595f351..8357f5635 100644 --- a/artipie-main/src/test/java/com/artipie/cooldown/JdbcCooldownServiceTest.java +++ b/artipie-main/src/test/java/com/artipie/cooldown/JdbcCooldownServiceTest.java @@ -173,7 +173,11 @@ public CompletableFuture> dependencies(final String art }; this.service.evaluate(request, inspector).join(); this.service.unblockAll("npm-proxy", "npm", "eve").join(); - MatcherAssert.assertThat(this.status("npm", "main", "1.0.0"), Matchers.is("INACTIVE")); + MatcherAssert.assertThat( + "Record should be deleted after unblock", + this.recordExists("npm", "main", "1.0.0"), + Matchers.is(false) + ); } @Test @@ -234,6 +238,22 @@ private void insertArtifact( } } + private boolean recordExists(final String repo, final String artifact, final String version) { + try (Connection conn = this.dataSource.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT 1 FROM artifact_cooldowns WHERE repo_name = ? AND artifact = ? AND version = ?" + )) { + stmt.setString(1, repo); + stmt.setString(2, artifact); + stmt.setString(3, version); + try (ResultSet rs = stmt.executeQuery()) { + return rs.next(); + } + } catch (final SQLException err) { + throw new IllegalStateException(err); + } + } + private String status(final String repo, final String artifact, final String version) { try (Connection conn = this.dataSource.getConnection(); PreparedStatement stmt = conn.prepareStatement( diff --git a/artipie-main/src/test/java/com/artipie/db/DbManagerTest.java b/artipie-main/src/test/java/com/artipie/db/DbManagerTest.java new file mode 100644 index 000000000..fa80e2274 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/DbManagerTest.java @@ -0,0 +1,77 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db; + +import java.sql.Connection; +import java.sql.ResultSet; +import javax.sql.DataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link DbManager}. + * @since 1.0 + */ +@Testcontainers +class DbManagerTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + } + + @AfterAll + static void teardown() { + if (ds != null) { + ds.close(); + } + } + + @Test + void runsMigrationsAndCreatesSettingsTables() throws Exception { + DbManager.migrate(ds); + try (Connection conn = ds.getConnection()) { + assertTrue(tableExists(conn, "repositories")); + assertTrue(tableExists(conn, "users")); + assertTrue(tableExists(conn, "roles")); + assertTrue(tableExists(conn, "user_roles")); + assertTrue(tableExists(conn, "storage_aliases")); + assertTrue(tableExists(conn, "settings")); + assertTrue(tableExists(conn, "auth_providers")); + assertTrue(tableExists(conn, "audit_log")); + } + } + + @Test + void migrationsAreIdempotent() throws Exception { + DbManager.migrate(ds); + DbManager.migrate(ds); + try (Connection conn = ds.getConnection()) { + assertTrue(tableExists(conn, "repositories")); + } + } + + private static boolean tableExists(Connection conn, String table) throws Exception { + try (ResultSet rs = conn.getMetaData().getTables(null, "public", table, null)) { + return rs.next(); + } + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/DeadLetterWriterTest.java b/artipie-main/src/test/java/com/artipie/db/DeadLetterWriterTest.java new file mode 100644 index 000000000..61b35f103 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/DeadLetterWriterTest.java @@ -0,0 +1,87 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db; + +import com.artipie.scheduling.ArtifactEvent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; + +/** + * Tests for {@link DeadLetterWriter}. + * + * @since 1.20.13 + */ +final class DeadLetterWriterTest { + + @Test + void writesEventsToFile(@TempDir final Path tmp) throws Exception { + final DeadLetterWriter writer = new DeadLetterWriter(tmp.resolve("dead-letter")); + final List events = List.of( + new ArtifactEvent( + "maven", "my-repo", "owner1", + "com.example:artifact", "1.0.0", + 1024L, System.currentTimeMillis(), + ArtifactEvent.Type.INSERT + ) + ); + final Path file = writer.write(events, new RuntimeException("DB down"), 3); + assertThat("Dead letter file should exist", + String.valueOf(Files.exists(file)), containsString("true")); + final String content = Files.readString(file); + assertThat("Should contain repo name", content, containsString("my-repo")); + assertThat("Should contain artifact name", content, + containsString("com.example:artifact")); + assertThat("Should contain error", content, containsString("DB down")); + assertThat("Should contain retry count", content, containsString("3")); + } + + @Test + void createsDirectoryIfMissing(@TempDir final Path tmp) throws Exception { + final Path nested = tmp.resolve("a").resolve("b").resolve("dead-letter"); + final DeadLetterWriter writer = new DeadLetterWriter(nested); + final List events = List.of( + new ArtifactEvent( + "npm", "npm-proxy", "admin", + "@scope/pkg", "2.0.0", + 0L, System.currentTimeMillis(), + ArtifactEvent.Type.DELETE_VERSION + ) + ); + final Path file = writer.write(events, new RuntimeException("timeout"), 1); + assertThat("Nested directory should be created", + String.valueOf(Files.isDirectory(nested)), containsString("true")); + assertThat("File should exist", + String.valueOf(Files.exists(file)), containsString("true")); + } + + @Test + void handlesMultipleEvents(@TempDir final Path tmp) throws Exception { + final DeadLetterWriter writer = new DeadLetterWriter(tmp); + final List events = List.of( + new ArtifactEvent("maven", "r1", "u1", + "a1", "1.0", 100L, System.currentTimeMillis(), + ArtifactEvent.Type.INSERT), + new ArtifactEvent("maven", "r1", "u2", + "a2", "2.0", 200L, System.currentTimeMillis(), + ArtifactEvent.Type.INSERT), + new ArtifactEvent("docker", "r2", "u1", + "a3", "3.0", 300L, System.currentTimeMillis(), + ArtifactEvent.Type.DELETE_ALL) + ); + final Path file = writer.write(events, new RuntimeException("fail"), 2); + final String content = Files.readString(file); + assertThat("Should contain first event", content, containsString("a1")); + assertThat("Should contain second event", content, containsString("a2")); + assertThat("Should contain third event", content, containsString("a3")); + assertThat("Should contain event count", content, containsString("3")); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/SettingsLayerIntegrationTest.java b/artipie-main/src/test/java/com/artipie/db/SettingsLayerIntegrationTest.java new file mode 100644 index 000000000..8755a9325 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/SettingsLayerIntegrationTest.java @@ -0,0 +1,116 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db; + +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; +import com.artipie.api.RepositoryName; +import com.artipie.db.dao.*; +import com.artipie.db.migration.YamlToDbMigrator; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import java.nio.file.Files; +import java.nio.file.Path; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class SettingsLayerIntegrationTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + + @TempDir + Path configDir; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(3); + ds = new HikariDataSource(cfg); + } + + @AfterAll + static void teardown() { + if (ds != null) { ds.close(); } + } + + @BeforeEach + void clean() throws Exception { + DbManager.migrate(ds); + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM repositories"); + conn.createStatement().execute("DELETE FROM user_roles"); + conn.createStatement().execute("DELETE FROM users"); + conn.createStatement().execute("DELETE FROM roles"); + conn.createStatement().execute("DELETE FROM storage_aliases"); + conn.createStatement().execute("DELETE FROM settings"); + conn.createStatement().execute("DELETE FROM auth_providers"); + conn.createStatement().execute("DELETE FROM audit_log"); + } + } + + @Test + void flywayCreatesTables() throws Exception { + try (var conn = ds.getConnection(); + var rs = conn.getMetaData().getTables(null, "public", "repositories", null)) { + assertTrue(rs.next()); + } + } + + @Test + void migratorPopulatesFromYaml() throws Exception { + final Path repos = this.configDir.resolve("repo"); + Files.createDirectories(repos); + Files.writeString(repos.resolve("test-maven.yaml"), + "repo:\n type: maven-proxy\n storage: default"); + final YamlToDbMigrator migrator = new YamlToDbMigrator(ds, this.configDir.resolve("security"), repos); + assertTrue(migrator.migrate()); + final RepositoryDao repoDao = new RepositoryDao(ds); + assertTrue(repoDao.exists(new RepositoryName.Simple("test-maven"))); + } + + @Test + void crudOperationsWork() throws Exception { + final RepositoryDao repoDao = new RepositoryDao(ds); + repoDao.save(new RepositoryName.Simple("int-test-npm"), + Json.createObjectBuilder() + .add("repo", Json.createObjectBuilder() + .add("type", "npm-local") + .add("storage", "default")) + .build(), + "admin" + ); + assertTrue(repoDao.exists(new RepositoryName.Simple("int-test-npm"))); + final AuditLogDao audit = new AuditLogDao(ds); + audit.log("admin", "CREATE", "repository", "int-test-npm", null, + Json.createObjectBuilder().add("type", "npm-local").build()); + final SettingsDao settings = new SettingsDao(ds); + settings.put("int_test_key", Json.createObjectBuilder().add("v", 1).build(), "admin"); + assertTrue(settings.get("int_test_key").isPresent()); + } + + @Test + void migratorIsIdempotent() throws Exception { + final Path repos = this.configDir.resolve("repo"); + Files.createDirectories(repos); + final YamlToDbMigrator migrator = new YamlToDbMigrator(ds, this.configDir.resolve("security"), repos); + migrator.migrate(); + // Second call should skip + assertFalse(migrator.migrate()); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/dao/AuditLogDaoTest.java b/artipie-main/src/test/java/com/artipie/db/dao/AuditLogDaoTest.java new file mode 100644 index 000000000..13f168dfb --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/dao/AuditLogDaoTest.java @@ -0,0 +1,82 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.sql.Connection; +import java.sql.ResultSet; +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Testcontainers +class AuditLogDaoTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + AuditLogDao dao; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { ds.close(); } + } + + @BeforeEach + void init() { + this.dao = new AuditLogDao(ds); + } + + @Test + void logsCreateAction() throws Exception { + final JsonObject val = Json.createObjectBuilder() + .add("type", "maven-proxy").build(); + this.dao.log("admin", "CREATE", "repository", "maven-central", null, val); + try (Connection conn = ds.getConnection()) { + final ResultSet rs = conn.createStatement() + .executeQuery("SELECT * FROM audit_log WHERE resource_name = 'maven-central'"); + assertTrue(rs.next()); + assertEquals("admin", rs.getString("actor")); + assertEquals("CREATE", rs.getString("action")); + assertEquals("repository", rs.getString("resource_type")); + } + } + + @Test + void logsUpdateWithOldAndNewValues() throws Exception { + final JsonObject old = Json.createObjectBuilder().add("enabled", true).build(); + final JsonObject nw = Json.createObjectBuilder().add("enabled", false).build(); + this.dao.log("admin", "UPDATE", "user", "john", old, nw); + try (Connection conn = ds.getConnection()) { + final ResultSet rs = conn.createStatement() + .executeQuery("SELECT * FROM audit_log WHERE resource_name = 'john'"); + assertTrue(rs.next()); + assertEquals("UPDATE", rs.getString("action")); + } + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/dao/AuthProviderDaoTest.java b/artipie-main/src/test/java/com/artipie/db/dao/AuthProviderDaoTest.java new file mode 100644 index 000000000..e6dd64ebf --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/dao/AuthProviderDaoTest.java @@ -0,0 +1,121 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.util.List; +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class AuthProviderDaoTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + AuthProviderDao dao; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { ds.close(); } + } + + @BeforeEach + void init() throws Exception { + this.dao = new AuthProviderDao(ds); + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM auth_providers"); + } + } + + @Test + void putsAndListsProvider() { + final JsonObject config = Json.createObjectBuilder() + .add("realm", "artipie").build(); + this.dao.put("artipie", 1, config); + final List all = this.dao.list(); + assertEquals(1, all.size()); + assertEquals("artipie", all.get(0).getString("type")); + assertEquals(1, all.get(0).getInt("priority")); + } + + @Test + void upsertsExistingProviderByType() { + this.dao.put("keycloak", 1, Json.createObjectBuilder() + .add("url", "http://old").build()); + this.dao.put("keycloak", 2, Json.createObjectBuilder() + .add("url", "http://new").build()); + final List all = this.dao.list(); + assertEquals(1, all.size()); + assertEquals(2, all.get(0).getInt("priority")); + assertEquals("http://new", + all.get(0).getJsonObject("config").getString("url")); + } + + @Test + void listsEnabledOnly() { + this.dao.put("artipie", 1, Json.createObjectBuilder().build()); + this.dao.put("keycloak", 2, Json.createObjectBuilder().build()); + // Disable keycloak + final int kcId = this.dao.list().stream() + .filter(p -> p.getString("type").equals("keycloak")) + .findFirst().get().getInt("id"); + this.dao.disable(kcId); + final List enabled = this.dao.listEnabled(); + assertEquals(1, enabled.size()); + assertEquals("artipie", enabled.get(0).getString("type")); + } + + @Test + void enablesAndDisablesProvider() { + this.dao.put("okta", 1, Json.createObjectBuilder().build()); + final int id = this.dao.list().get(0).getInt("id"); + this.dao.disable(id); + assertFalse(this.dao.list().get(0).getBoolean("enabled")); + this.dao.enable(id); + assertTrue(this.dao.list().get(0).getBoolean("enabled")); + } + + @Test + void deletesProvider() { + this.dao.put("temp", 1, Json.createObjectBuilder().build()); + assertEquals(1, this.dao.list().size()); + final int id = this.dao.list().get(0).getInt("id"); + this.dao.delete(id); + assertEquals(0, this.dao.list().size()); + } + + @Test + void listsOrderedByPriority() { + this.dao.put("keycloak", 2, Json.createObjectBuilder().build()); + this.dao.put("artipie", 1, Json.createObjectBuilder().build()); + final List all = this.dao.list(); + assertEquals("artipie", all.get(0).getString("type")); + assertEquals("keycloak", all.get(1).getString("type")); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/dao/RepositoryDaoTest.java b/artipie-main/src/test/java/com/artipie/db/dao/RepositoryDaoTest.java new file mode 100644 index 000000000..3bd34d0f7 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/dao/RepositoryDaoTest.java @@ -0,0 +1,139 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.util.Collection; +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonStructure; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.artipie.api.RepositoryName; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class RepositoryDaoTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + RepositoryDao dao; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { ds.close(); } + } + + @BeforeEach + void init() throws Exception { + this.dao = new RepositoryDao(ds); + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM repositories"); + } + } + + @Test + void savesAndGetsRepository() { + final JsonObject config = Json.createObjectBuilder() + .add("repo", Json.createObjectBuilder() + .add("type", "maven-proxy") + .add("storage", "default")) + .build(); + this.dao.save(new RepositoryName.Simple("maven-central"), config, "admin"); + final JsonStructure result = this.dao.value(new RepositoryName.Simple("maven-central")); + assertNotNull(result); + assertEquals("maven-proxy", + result.asJsonObject().getJsonObject("repo").getString("type")); + } + + @Test + void existsReturnsTrueForExistingRepo() { + saveTestRepo("my-repo", "maven-proxy"); + assertTrue(this.dao.exists(new RepositoryName.Simple("my-repo"))); + } + + @Test + void existsReturnsFalseForMissingRepo() { + assertFalse(this.dao.exists(new RepositoryName.Simple("no-such-repo"))); + } + + @Test + void listsAllRepos() { + saveTestRepo("repo-a", "maven-proxy"); + saveTestRepo("repo-b", "npm-local"); + final Collection all = this.dao.listAll(); + assertEquals(2, all.size()); + assertTrue(all.contains("repo-a")); + assertTrue(all.contains("repo-b")); + } + + @Test + void deletesRepository() { + saveTestRepo("to-delete", "maven-local"); + assertTrue(this.dao.exists(new RepositoryName.Simple("to-delete"))); + this.dao.delete(new RepositoryName.Simple("to-delete")); + assertFalse(this.dao.exists(new RepositoryName.Simple("to-delete"))); + } + + @Test + void movesRepository() { + saveTestRepo("old-name", "docker-proxy"); + this.dao.move(new RepositoryName.Simple("old-name"), new RepositoryName.Simple("new-name")); + assertFalse(this.dao.exists(new RepositoryName.Simple("old-name"))); + assertTrue(this.dao.exists(new RepositoryName.Simple("new-name"))); + } + + @Test + void updatesExistingRepo() { + saveTestRepo("updatable", "maven-proxy"); + final JsonObject updated = Json.createObjectBuilder() + .add("repo", Json.createObjectBuilder() + .add("type", "maven-local") + .add("storage", "s3")) + .build(); + this.dao.save(new RepositoryName.Simple("updatable"), updated, "admin"); + final JsonStructure result = this.dao.value(new RepositoryName.Simple("updatable")); + assertEquals("maven-local", + result.asJsonObject().getJsonObject("repo").getString("type")); + } + + @Test + void throwsOnGetMissingRepo() { + assertThrows(IllegalStateException.class, + () -> this.dao.value(new RepositoryName.Simple("nope"))); + } + + private void saveTestRepo(final String name, final String type) { + this.dao.save( + new RepositoryName.Simple(name), + Json.createObjectBuilder() + .add("repo", Json.createObjectBuilder() + .add("type", type) + .add("storage", "default")) + .build(), + "admin" + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/dao/RoleDaoTest.java b/artipie-main/src/test/java/com/artipie/db/dao/RoleDaoTest.java new file mode 100644 index 000000000..a8fbaeb05 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/dao/RoleDaoTest.java @@ -0,0 +1,121 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.sql.DataSource; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class RoleDaoTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + RoleDao dao; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { ds.close(); } + } + + @BeforeEach + void init() throws Exception { + this.dao = new RoleDao(ds); + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM user_roles"); + conn.createStatement().execute("DELETE FROM roles"); + } + } + + @Test + void addsAndGetsRole() { + final JsonObject perms = Json.createObjectBuilder() + .add("adapter_basic_permissions", Json.createObjectBuilder() + .add("maven-repo", Json.createArrayBuilder().add("read").add("write"))) + .build(); + this.dao.addOrUpdate(perms, "developers"); + final Optional result = this.dao.get("developers"); + assertTrue(result.isPresent()); + assertEquals("developers", result.get().getString("name")); + assertTrue(result.get().containsKey("adapter_basic_permissions")); + } + + @Test + void listsRoles() { + addTestRole("readers"); + addTestRole("writers"); + final JsonArray list = this.dao.list(); + assertEquals(2, list.size()); + } + + @Test + void updatesExistingRole() { + addTestRole("updatable"); + final JsonObject newPerms = Json.createObjectBuilder() + .add("api_repository", Json.createArrayBuilder().add("read")) + .build(); + this.dao.addOrUpdate(newPerms, "updatable"); + final JsonObject role = this.dao.get("updatable").get(); + assertTrue(role.containsKey("api_repository")); + } + + @Test + void enablesAndDisablesRole() { + addTestRole("toggleable"); + this.dao.disable("toggleable"); + assertFalse(this.dao.get("toggleable").get().getBoolean("enabled")); + this.dao.enable("toggleable"); + assertTrue(this.dao.get("toggleable").get().getBoolean("enabled")); + } + + @Test + void removesRole() { + addTestRole("removable"); + assertTrue(this.dao.get("removable").isPresent()); + this.dao.remove("removable"); + assertTrue(this.dao.get("removable").isEmpty()); + } + + @Test + void returnsEmptyForMissingRole() { + assertTrue(this.dao.get("nonexistent").isEmpty()); + } + + private void addTestRole(final String name) { + this.dao.addOrUpdate( + Json.createObjectBuilder() + .add("adapter_basic_permissions", Json.createObjectBuilder() + .add("*", Json.createArrayBuilder().add("read"))) + .build(), + name + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/dao/SettingsDaoTest.java b/artipie-main/src/test/java/com/artipie/db/dao/SettingsDaoTest.java new file mode 100644 index 000000000..48ed81069 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/dao/SettingsDaoTest.java @@ -0,0 +1,95 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.util.Map; +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class SettingsDaoTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + SettingsDao dao; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { ds.close(); } + } + + @BeforeEach + void init() throws Exception { + this.dao = new SettingsDao(ds); + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM settings"); + } + } + + @Test + void putAndGetSetting() { + final JsonObject val = Json.createObjectBuilder().add("timeout", 120).build(); + this.dao.put("http_client", val, "admin"); + final Optional result = this.dao.get("http_client"); + assertTrue(result.isPresent()); + assertEquals(120, result.get().getInt("timeout")); + } + + @Test + void returnsEmptyForMissingKey() { + assertTrue(this.dao.get("nonexistent").isEmpty()); + } + + @Test + void updatesExistingKey() { + this.dao.put("port", Json.createObjectBuilder().add("value", 8080).build(), "admin"); + this.dao.put("port", Json.createObjectBuilder().add("value", 9090).build(), "admin"); + assertEquals(9090, this.dao.get("port").get().getInt("value")); + } + + @Test + void listsAllSettings() { + this.dao.put("key1", Json.createObjectBuilder().add("a", 1).build(), "admin"); + this.dao.put("key2", Json.createObjectBuilder().add("b", 2).build(), "admin"); + final Map all = this.dao.listAll(); + assertEquals(2, all.size()); + assertTrue(all.containsKey("key1")); + assertTrue(all.containsKey("key2")); + } + + @Test + void deletesKey() { + this.dao.put("temp", Json.createObjectBuilder().add("x", 1).build(), "admin"); + assertTrue(this.dao.get("temp").isPresent()); + this.dao.delete("temp"); + assertTrue(this.dao.get("temp").isEmpty()); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/dao/StorageAliasDaoTest.java b/artipie-main/src/test/java/com/artipie/db/dao/StorageAliasDaoTest.java new file mode 100644 index 000000000..1e682fffc --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/dao/StorageAliasDaoTest.java @@ -0,0 +1,121 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.util.List; +import javax.json.Json; +import javax.json.JsonObject; +import javax.sql.DataSource; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class StorageAliasDaoTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + StorageAliasDao dao; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { ds.close(); } + } + + @BeforeEach + void init() throws Exception { + this.dao = new StorageAliasDao(ds); + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM storage_aliases"); + conn.createStatement().execute("DELETE FROM repositories"); + } + } + + @Test + void putsAndListsGlobalAlias() { + final JsonObject config = Json.createObjectBuilder() + .add("type", "fs").add("path", "/var/artipie/data").build(); + this.dao.put("default", null, config); + final List globals = this.dao.listGlobal(); + assertEquals(1, globals.size()); + assertEquals("default", globals.get(0).getString("name")); + assertEquals("fs", globals.get(0).getJsonObject("config").getString("type")); + } + + @Test + void putsAndListsRepoAlias() { + final JsonObject config = Json.createObjectBuilder() + .add("type", "s3").add("bucket", "my-bucket").build(); + this.dao.put("s3-store", "maven-central", config); + final List repoAliases = this.dao.listForRepo("maven-central"); + assertEquals(1, repoAliases.size()); + assertEquals("s3-store", repoAliases.get(0).getString("name")); + } + + @Test + void updatesExistingAlias() { + final JsonObject orig = Json.createObjectBuilder() + .add("type", "fs").add("path", "/old").build(); + this.dao.put("default", null, orig); + final JsonObject updated = Json.createObjectBuilder() + .add("type", "fs").add("path", "/new").build(); + this.dao.put("default", null, updated); + final List globals = this.dao.listGlobal(); + assertEquals(1, globals.size()); + assertEquals("/new", globals.get(0).getJsonObject("config").getString("path")); + } + + @Test + void deletesGlobalAlias() { + this.dao.put("temp", null, Json.createObjectBuilder().add("type", "fs").build()); + assertEquals(1, this.dao.listGlobal().size()); + this.dao.delete("temp", null); + assertEquals(0, this.dao.listGlobal().size()); + } + + @Test + void deletesRepoAlias() { + this.dao.put("store", "my-repo", Json.createObjectBuilder().add("type", "fs").build()); + assertEquals(1, this.dao.listForRepo("my-repo").size()); + this.dao.delete("store", "my-repo"); + assertEquals(0, this.dao.listForRepo("my-repo").size()); + } + + @Test + void findReposUsingAlias() throws Exception { + // Insert a repo whose config references alias "default" + try (var conn = ds.getConnection()) { + conn.createStatement().execute( + "INSERT INTO repositories (name, type, config) VALUES " + + "('maven-central', 'maven-proxy', " + + "'{\"repo\":{\"type\":\"maven-proxy\",\"storage\":\"default\"}}'::jsonb)" + ); + } + final List repos = this.dao.findReposUsing("default"); + assertEquals(1, repos.size()); + assertEquals("maven-central", repos.get(0)); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/dao/UserDaoTest.java b/artipie-main/src/test/java/com/artipie/db/dao/UserDaoTest.java new file mode 100644 index 000000000..b7a881007 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/dao/UserDaoTest.java @@ -0,0 +1,166 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.dao; + +import java.util.Optional; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonObject; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +class UserDaoTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + static HikariDataSource ds; + UserDao dao; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { + ds.close(); + } + } + + @BeforeEach + void init() throws Exception { + this.dao = new UserDao(ds); + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM user_roles"); + conn.createStatement().execute("DELETE FROM users"); + conn.createStatement().execute("DELETE FROM roles"); + } + } + + @Test + void addsAndGetsUser() { + final JsonObject info = Json.createObjectBuilder() + .add("pass", "secret123") + .add("type", "plain") + .add("email", "john@example.com") + .build(); + this.dao.addOrUpdate(info, "john"); + final Optional result = this.dao.get("john"); + assertTrue(result.isPresent()); + assertEquals("john@example.com", result.get().getString("email")); + // Password should NOT be in get() result + assertFalse(result.get().containsKey("pass")); + } + + @Test + void listsUsers() { + addTestUser("alice"); + addTestUser("bob"); + final JsonArray list = this.dao.list(); + assertEquals(2, list.size()); + } + + @Test + void updatesExistingUser() { + addTestUser("charlie"); + final JsonObject updated = Json.createObjectBuilder() + .add("email", "new@example.com") + .add("pass", "newpass") + .add("type", "plain") + .build(); + this.dao.addOrUpdate(updated, "charlie"); + assertEquals("new@example.com", this.dao.get("charlie").get().getString("email")); + } + + @Test + void enablesAndDisablesUser() { + addTestUser("dave"); + this.dao.disable("dave"); + assertFalse(this.dao.get("dave").get().getBoolean("enabled")); + this.dao.enable("dave"); + assertTrue(this.dao.get("dave").get().getBoolean("enabled")); + } + + @Test + void removesUser() { + addTestUser("eve"); + assertTrue(this.dao.get("eve").isPresent()); + this.dao.remove("eve"); + assertTrue(this.dao.get("eve").isEmpty()); + } + + @Test + void altersPassword() { + addTestUser("frank"); + final JsonObject passInfo = Json.createObjectBuilder() + .add("new_pass", "updated_hash") + .add("new_type", "sha256") + .build(); + this.dao.alterPassword("frank", passInfo); + // Verify internally that password was changed + try (var conn = ds.getConnection(); + var ps = conn.prepareStatement( + "SELECT password_hash FROM users WHERE username = ?")) { + ps.setString(1, "frank"); + var rs = ps.executeQuery(); + assertTrue(rs.next()); + assertEquals("updated_hash", rs.getString("password_hash")); + } catch (final Exception ex) { + fail(ex); + } + } + + @Test + void returnsEmptyForMissingUser() { + assertTrue(this.dao.get("nobody").isEmpty()); + } + + @Test + void addsUserWithRoles() throws Exception { + // Seed a role first + try (var conn = ds.getConnection()) { + conn.createStatement().execute( + "INSERT INTO roles (name, permissions) VALUES ('readers', '{}'::jsonb)" + ); + } + final JsonObject info = Json.createObjectBuilder() + .add("pass", "secret") + .add("type", "plain") + .add("roles", Json.createArrayBuilder().add("readers")) + .build(); + this.dao.addOrUpdate(info, "grace"); + final JsonObject user = this.dao.get("grace").get(); + assertTrue(user.getJsonArray("roles").getString(0).equals("readers")); + } + + private void addTestUser(final String name) { + this.dao.addOrUpdate( + Json.createObjectBuilder() + .add("pass", "pass123") + .add("type", "plain") + .add("email", name + "@example.com") + .build(), + name + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/db/migration/YamlToDbMigratorTest.java b/artipie-main/src/test/java/com/artipie/db/migration/YamlToDbMigratorTest.java new file mode 100644 index 000000000..9f705d2c8 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/db/migration/YamlToDbMigratorTest.java @@ -0,0 +1,242 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.db.migration; + +import com.artipie.api.RepositoryName; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.artipie.db.dao.AuthProviderDao; +import com.artipie.db.dao.RoleDao; +import com.artipie.db.dao.RepositoryDao; +import com.artipie.db.dao.SettingsDao; +import com.artipie.db.dao.UserDao; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.json.Json; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link YamlToDbMigrator}. + * @since 1.0 + */ +@Testcontainers +class YamlToDbMigratorTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + + static HikariDataSource ds; + + @TempDir + Path configDir; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(2); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { + ds.close(); + } + } + + @BeforeEach + void clean() throws Exception { + try (var conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM user_roles"); + conn.createStatement().execute("DELETE FROM repositories"); + conn.createStatement().execute("DELETE FROM users"); + conn.createStatement().execute("DELETE FROM roles"); + conn.createStatement().execute("DELETE FROM storage_aliases"); + conn.createStatement().execute("DELETE FROM settings"); + conn.createStatement().execute("DELETE FROM auth_providers"); + } + } + + @Test + void migratesRepoConfigs() throws Exception { + final Path repos = this.configDir.resolve("repo"); + Files.createDirectories(repos); + Files.writeString( + repos.resolve("maven-central.yaml"), + String.join( + "\n", + "repo:", + " type: maven-proxy", + " remotes:", + " - url: https://repo1.maven.org/maven2", + " cache:", + " storage: default", + " storage: default" + ) + ); + final YamlToDbMigrator migrator = new YamlToDbMigrator( + ds, this.configDir.resolve("security"), repos + ); + migrator.migrate(); + final RepositoryDao dao = new RepositoryDao(ds); + assertTrue(dao.exists(new RepositoryName.Simple("maven-central"))); + // Verify YAML sequences (remotes array) were preserved + final var config = dao.value( + new RepositoryName.Simple("maven-central") + ).asJsonObject(); + assertTrue(config.getJsonObject("repo").containsKey("remotes")); + } + + @Test + void migratesUsersWithRoles() throws Exception { + // Seed a role first + final Path rolesDir = this.configDir.resolve("security").resolve("roles"); + Files.createDirectories(rolesDir); + Files.writeString( + rolesDir.resolve("readers.yaml"), + String.join( + "\n", + "adapter_basic_permissions:", + " \"*\":", + " - read" + ) + ); + final Path usersDir = this.configDir.resolve("security").resolve("users"); + Files.createDirectories(usersDir); + Files.writeString( + usersDir.resolve("john.yaml"), + String.join( + "\n", + "type: plain", + "pass: secret123", + "email: john@example.com", + "enabled: true", + "roles:", + " - readers" + ) + ); + final Path repos = this.configDir.resolve("repo"); + Files.createDirectories(repos); + final YamlToDbMigrator migrator = new YamlToDbMigrator( + ds, this.configDir.resolve("security"), repos + ); + migrator.migrate(); + final UserDao userDao = new UserDao(ds); + assertTrue(userDao.get("john").isPresent()); + // Verify password was NOT stored as plaintext + try (var conn = ds.getConnection(); + var ps = conn.prepareStatement( + "SELECT password_hash FROM users WHERE username = 'john'" + )) { + final var rs = ps.executeQuery(); + assertTrue(rs.next()); + // Password should be bcrypt-hashed (starts with $2) + assertTrue(rs.getString("password_hash").startsWith("$2")); + } + // Verify role assignment + final var user = userDao.get("john").get(); + assertEquals(1, user.getJsonArray("roles").size()); + assertEquals("readers", user.getJsonArray("roles").getString(0)); + } + + @Test + void migratesRoles() throws Exception { + final Path rolesDir = this.configDir.resolve("security").resolve("roles"); + Files.createDirectories(rolesDir); + Files.writeString( + rolesDir.resolve("devs.yaml"), + String.join( + "\n", + "adapter_basic_permissions:", + " maven-repo:", + " - read", + " - write" + ) + ); + final Path repos = this.configDir.resolve("repo"); + Files.createDirectories(repos); + final YamlToDbMigrator migrator = new YamlToDbMigrator( + ds, this.configDir.resolve("security"), repos + ); + migrator.migrate(); + final RoleDao roleDao = new RoleDao(ds); + assertTrue(roleDao.get("devs").isPresent()); + } + + @Test + void migratesSettingsFromArtipieYml() throws Exception { + Files.writeString( + this.configDir.resolve("artipie.yml"), + String.join( + "\n", + "meta:", + " layout: flat", + " credentials:", + " - type: artipie", + " - type: keycloak", + " url: http://keycloak:8080", + " realm: artipie" + ) + ); + final Path repos = this.configDir.resolve("repo"); + Files.createDirectories(repos); + final YamlToDbMigrator migrator = new YamlToDbMigrator( + ds, this.configDir.resolve("security"), repos, + this.configDir.resolve("artipie.yml") + ); + migrator.migrate(); + // Verify settings + final SettingsDao settings = new SettingsDao(ds); + assertTrue(settings.get("layout").isPresent()); + // Verify auth providers + final AuthProviderDao authDao = new AuthProviderDao(ds); + assertEquals(2, authDao.list().size()); + } + + @Test + void skipsIfAlreadyMigrated() throws Exception { + final SettingsDao settings = new SettingsDao(ds); + settings.put( + "migration_completed", + Json.createObjectBuilder() + .add("completed", true) + .add("version", 3) + .build(), + "system" + ); + final YamlToDbMigrator migrator = new YamlToDbMigrator( + ds, this.configDir.resolve("security"), this.configDir.resolve("repo") + ); + assertFalse(migrator.migrate()); + } + + @Test + void setsMigrationFlag() throws Exception { + final Path repos = this.configDir.resolve("repo"); + Files.createDirectories(repos); + final YamlToDbMigrator migrator = new YamlToDbMigrator( + ds, this.configDir.resolve("security"), repos + ); + migrator.migrate(); + final SettingsDao settings = new SettingsDao(ds); + assertTrue(settings.get("migration_completed").isPresent()); + } +} diff --git a/artipie-main/src/test/java/com/artipie/group/ArtifactNameParserTest.java b/artipie-main/src/test/java/com/artipie/group/ArtifactNameParserTest.java new file mode 100644 index 000000000..e2b887ab5 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/group/ArtifactNameParserTest.java @@ -0,0 +1,683 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.hamcrest.core.IsEqual; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Optional; + +/** + * Extensive tests for {@link ArtifactNameParser}. + * Validates URL-to-name extraction for all supported adapter types. + * The hit rate (successful parse) must be >= 95% across realistic URL patterns. + * + * @since 1.21.0 + */ +@SuppressWarnings("PMD.TooManyMethods") +final class ArtifactNameParserTest { + + // ---- Maven: artifact downloads ---- + + @ParameterizedTest + @CsvSource({ + // Standard artifact JAR + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar, com.google.guava.guava", + // Artifact POM + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.pom, com.google.guava.guava", + // Sources JAR + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre-sources.jar, com.google.guava.guava", + // Javadoc JAR + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre-javadoc.jar, com.google.guava.guava", + // WAR file + "/com/example/webapp/1.0/webapp-1.0.war, com.example.webapp", + // AAR file (Android) + "/com/android/support/appcompat-v7/28.0.0/appcompat-v7-28.0.0.aar, com.android.support.appcompat-v7", + // Gradle module metadata + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.module, com.google.guava.guava", + // Single-segment groupId + "/junit/junit/4.13.2/junit-4.13.2.jar, junit.junit", + // Deep groupId + "/org/apache/maven/plugins/maven-compiler-plugin/3.11.0/maven-compiler-plugin-3.11.0.jar, org.apache.maven.plugins.maven-compiler-plugin", + // SNAPSHOT version + "/org/example/mylib/1.0-SNAPSHOT/mylib-1.0-20230101.120000-1.jar, org.example.mylib", + // Without leading slash + "com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar, com.google.guava.guava", + }) + void mavenArtifactFiles(final String url, final String expected) { + MatcherAssert.assertThat( + "Maven artifact: " + url, + ArtifactNameParser.parse("maven-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + // ---- Maven: checksums and signatures ---- + + @ParameterizedTest + @CsvSource({ + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.sha1, com.google.guava.guava", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.sha256, com.google.guava.guava", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.md5, com.google.guava.guava", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.pom.asc, com.google.guava.guava", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.sha512, com.google.guava.guava", + }) + void mavenChecksums(final String url, final String expected) { + MatcherAssert.assertThat( + "Maven checksum: " + url, + ArtifactNameParser.parse("maven-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + // ---- Maven: metadata requests ---- + + @ParameterizedTest + @CsvSource({ + // Metadata at artifact level (no version directory) + "/com/google/guava/guava/maven-metadata.xml, com.google.guava.guava", + // Metadata checksum + "/com/google/guava/guava/maven-metadata.xml.sha1, com.google.guava.guava", + // Metadata at version level + "/com/google/guava/guava/31.1.3-jre/maven-metadata.xml, com.google.guava.guava", + // Plugin metadata + "/org/apache/maven/plugins/maven-compiler-plugin/maven-metadata.xml, org.apache.maven.plugins.maven-compiler-plugin", + }) + void mavenMetadata(final String url, final String expected) { + MatcherAssert.assertThat( + "Maven metadata: " + url, + ArtifactNameParser.parse("maven-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + // ---- Maven: also works with maven-proxy repo type ---- + + @Test + void mavenProxyRepoType() { + MatcherAssert.assertThat( + ArtifactNameParser.parse( + "maven-proxy", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar" + ), + new IsEqual<>(Optional.of("com.google.guava.guava")) + ); + } + + @Test + void mavenLocalRepoType() { + MatcherAssert.assertThat( + ArtifactNameParser.parse( + "maven", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar" + ), + new IsEqual<>(Optional.of("com.google.guava.guava")) + ); + } + + // ---- Maven: edge cases ---- + + @Test + void mavenRootPath() { + MatcherAssert.assertThat( + ArtifactNameParser.parse("maven-group", "/"), + new IsEqual<>(Optional.empty()) + ); + } + + @Test + void mavenEmptyPath() { + MatcherAssert.assertThat( + ArtifactNameParser.parse("maven-group", ""), + new IsEqual<>(Optional.empty()) + ); + } + + // ---- npm: tarball downloads ---- + + @ParameterizedTest + @CsvSource({ + "/lodash/-/lodash-4.17.21.tgz, lodash", + "/@babel/core/-/@babel/core-7.23.0.tgz, @babel/core", + "/@types/node/-/@types/node-20.10.0.tgz, @types/node", + "/@angular/core/-/@angular/core-17.0.0.tgz, @angular/core", + "/express/-/express-4.18.2.tgz, express", + // Without leading slash + "lodash/-/lodash-4.17.21.tgz, lodash", + }) + void npmTarballs(final String url, final String expected) { + MatcherAssert.assertThat( + "npm tarball: " + url, + ArtifactNameParser.parse("npm-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + // ---- npm: metadata requests ---- + + @ParameterizedTest + @CsvSource({ + "/lodash, lodash", + "/@babel/core, @babel/core", + "/@types/node, @types/node", + "/express, express", + }) + void npmMetadata(final String url, final String expected) { + MatcherAssert.assertThat( + "npm metadata: " + url, + ArtifactNameParser.parse("npm-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + @Test + void npmRootPath() { + // Root path "/" strips to empty string, which is useless for lookup + final Optional result = ArtifactNameParser.parse("npm-group", "/"); + // Either empty or an empty string — both are acceptable + MatcherAssert.assertThat( + "Root path should not produce a useful name", + result.filter(s -> !s.isEmpty()), + new IsEqual<>(Optional.empty()) + ); + } + + // ---- Docker: manifest and blob requests ---- + + @ParameterizedTest + @CsvSource({ + "/v2/library/nginx/manifests/latest, library/nginx", + "/v2/library/nginx/manifests/sha256:abc123, library/nginx", + "/v2/library/nginx/blobs/sha256:abc123, library/nginx", + "/v2/library/nginx/tags/list, library/nginx", + "/v2/myimage/manifests/1.0, myimage", + "/v2/myorg/myimage/manifests/latest, myorg/myimage", + "/v2/registry.example.com/myorg/myimage/manifests/v1, registry.example.com/myorg/myimage", + }) + void dockerPaths(final String url, final String expected) { + MatcherAssert.assertThat( + "Docker: " + url, + ArtifactNameParser.parse("docker-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"/v2/", "/v2", "/"}) + void dockerBasePaths(final String url) { + MatcherAssert.assertThat( + "Docker base path should not match: " + url, + ArtifactNameParser.parse("docker-group", url), + new IsEqual<>(Optional.empty()) + ); + } + + // ---- PyPI: simple index and packages ---- + + @ParameterizedTest + @CsvSource({ + "/simple/numpy/, numpy", + "/simple/requests/, requests", + "/simple/my-package/, my-package", + "/simple/My_Package/, my-package", + "/simple/my.package/, my-package", + }) + void pypiSimpleIndex(final String url, final String expected) { + MatcherAssert.assertThat( + "PyPI simple: " + url, + ArtifactNameParser.parse("pypi-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + @ParameterizedTest + @CsvSource({ + "/packages/numpy-1.24.0.whl, numpy", + "/packages/numpy-1.24.0-cp310-cp310-manylinux_2_17_x86_64.whl, numpy", + "/packages/requests-2.31.0.tar.gz, requests", + "/packages/my_package-1.0.0.zip, my-package", + }) + void pypiPackages(final String url, final String expected) { + MatcherAssert.assertThat( + "PyPI package: " + url, + ArtifactNameParser.parse("pypi-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + @Test + void pypiSimpleRoot() { + MatcherAssert.assertThat( + ArtifactNameParser.parse("pypi-group", "/simple/"), + new IsEqual<>(Optional.empty()) + ); + } + + // ---- Go: module paths ---- + + @ParameterizedTest + @CsvSource({ + "/github.com/gin-gonic/gin/@v/v1.9.1.info, github.com/gin-gonic/gin", + "/github.com/gin-gonic/gin/@v/v1.9.1.mod, github.com/gin-gonic/gin", + "/github.com/gin-gonic/gin/@v/v1.9.1.zip, github.com/gin-gonic/gin", + "/github.com/gin-gonic/gin/@v/list, github.com/gin-gonic/gin", + "/github.com/gin-gonic/gin/@latest, github.com/gin-gonic/gin", + "/golang.org/x/text/@v/v0.14.0.info, golang.org/x/text", + }) + void goPaths(final String url, final String expected) { + MatcherAssert.assertThat( + "Go: " + url, + ArtifactNameParser.parse("go-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + // ---- Gradle: same as Maven ---- + + @ParameterizedTest + @CsvSource({ + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar, com.google.guava.guava", + "/org/gradle/gradle-tooling-api/8.5/gradle-tooling-api-8.5.jar, org.gradle.gradle-tooling-api", + "/com/android/tools/build/gradle/8.2.0/gradle-8.2.0.pom, com.android.tools.build.gradle", + "/com/google/guava/guava/maven-metadata.xml, com.google.guava.guava", + }) + void gradleUsessameparserAsMaven(final String url, final String expected) { + MatcherAssert.assertThat( + "Gradle: " + url, + ArtifactNameParser.parse("gradle-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + // ---- Gem: gem downloads and API ---- + + @ParameterizedTest + @CsvSource({ + "/gems/rails-7.1.2.gem, rails", + "/gems/nokogiri-1.15.4.gem, nokogiri", + "/gems/aws-sdk-core-3.190.0.gem, aws-sdk-core", + "/api/v1/gems/rails.json, rails", + "/quick/Marshal.4.8/rails-7.1.2.gemspec.rz, rails", + "/quick/Marshal.4.8/nokogiri-1.15.4.gemspec.rz, nokogiri", + }) + void gemPaths(final String url, final String expected) { + MatcherAssert.assertThat( + "Gem: " + url, + ArtifactNameParser.parse("gem-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + @Test + void gemDependenciesQuery() { + MatcherAssert.assertThat( + ArtifactNameParser.parse("gem-group", + "/api/v1/dependencies?gems=rails"), + new IsEqual<>(Optional.of("rails")) + ); + } + + // ---- PHP/Composer: package metadata ---- + + @ParameterizedTest + @CsvSource({ + "/p2/monolog/monolog.json, monolog/monolog", + "/p2/symfony/console.json, symfony/console", + "/p/vendor/package.json, vendor/package", + }) + void composerPaths(final String url, final String expected) { + MatcherAssert.assertThat( + "Composer: " + url, + ArtifactNameParser.parse("php-group", url), + new IsEqual<>(Optional.of(expected)) + ); + } + + @Test + void composerSatisCacheBusting() { + // Satis format: /p2/vendor/package$hash.json + MatcherAssert.assertThat( + ArtifactNameParser.parse("php-group", + "/p2/monolog/monolog$abc123def.json"), + new IsEqual<>(Optional.of("monolog/monolog")) + ); + } + + @Test + void composerPackagesJsonReturnsEmpty() { + MatcherAssert.assertThat( + ArtifactNameParser.parse("php-group", "/packages.json"), + new IsEqual<>(Optional.empty()) + ); + } + + // ---- Unknown/unsupported repo types ---- + + @ParameterizedTest + @ValueSource(strings = {"file-group", "helm-group", "unknown", ""}) + void unsupportedTypesReturnEmpty(final String repoType) { + MatcherAssert.assertThat( + "Unsupported type '" + repoType + "' should return empty", + ArtifactNameParser.parse(repoType, "/some/path/file.tar.gz"), + new IsEqual<>(Optional.empty()) + ); + } + + @Test + void nullRepoType() { + MatcherAssert.assertThat( + ArtifactNameParser.parse(null, "/some/path"), + new IsEqual<>(Optional.empty()) + ); + } + + @Test + void nullPath() { + MatcherAssert.assertThat( + ArtifactNameParser.parse("maven-group", null), + new IsEqual<>(Optional.empty()) + ); + } + + // ---- normalizeType ---- + + @ParameterizedTest + @CsvSource({ + "maven-group, maven", + "maven-proxy, maven", + "maven-local, maven", + "maven, maven", + "npm-group, npm", + "npm-proxy, npm", + "docker-group, docker", + "docker-remote, docker", + "pypi-group, pypi", + "go-group, go", + "gradle-group, gradle", + "gem-group, gem", + "php-group, php", + "file-group, file", + }) + void normalizeType(final String input, final String expected) { + MatcherAssert.assertThat( + ArtifactNameParser.normalizeType(input), + new IsEqual<>(expected) + ); + } + + // ---- Hit rate test: Maven ---- + + @Test + void mavenHitRateAbove95Percent() { + final String[] urls = { + // Standard artifacts + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.pom", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre-sources.jar", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre-javadoc.jar", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.module", + // Checksums + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.sha1", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.md5", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.pom.sha1", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.pom.sha256", + // Metadata + "/com/google/guava/guava/maven-metadata.xml", + "/com/google/guava/guava/maven-metadata.xml.sha1", + "/com/google/guava/guava/31.1.3-jre/maven-metadata.xml", + // Different libraries + "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar", + "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.pom", + "/org/apache/commons/commons-lang3/maven-metadata.xml", + "/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar", + "/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.pom", + "/org/slf4j/slf4j-api/maven-metadata.xml", + "/junit/junit/4.13.2/junit-4.13.2.jar", + "/junit/junit/4.13.2/junit-4.13.2.pom", + "/io/netty/netty-all/4.1.100.Final/netty-all-4.1.100.Final.jar", + "/io/netty/netty-all/4.1.100.Final/netty-all-4.1.100.Final.pom", + "/org/springframework/spring-core/6.1.0/spring-core-6.1.0.jar", + "/com/fasterxml/jackson/core/jackson-databind/2.16.0/jackson-databind-2.16.0.jar", + "/org/projectlombok/lombok/1.18.30/lombok-1.18.30.jar", + // SNAPSHOT + "/org/example/mylib/1.0-SNAPSHOT/mylib-1.0-20230101.120000-1.jar", + "/org/example/mylib/1.0-SNAPSHOT/maven-metadata.xml", + // Plugins + "/org/apache/maven/plugins/maven-compiler-plugin/3.11.0/maven-compiler-plugin-3.11.0.jar", + "/org/apache/maven/plugins/maven-surefire-plugin/3.2.3/maven-surefire-plugin-3.2.3.jar", + // Gradle wrapper + "/org/gradle/gradle-tooling-api/8.5/gradle-tooling-api-8.5.jar", + }; + int hits = 0; + for (final String url : urls) { + final Optional result = ArtifactNameParser.parse("maven-group", url); + if (result.isPresent() && !result.get().isEmpty()) { + hits++; + } + } + final double hitRate = (double) hits / urls.length * 100; + MatcherAssert.assertThat( + String.format("Maven hit rate %.1f%% must be >= 95%% (%d/%d)", hitRate, hits, urls.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + // ---- Hit rate test: npm ---- + + @Test + void npmHitRateAbove95Percent() { + final String[] urls = { + // Metadata + "/lodash", + "/@babel/core", + "/@types/node", + "/express", + "/react", + "/react-dom", + "/@angular/core", + "/@angular/common", + "/typescript", + "/webpack", + // Tarballs + "/lodash/-/lodash-4.17.21.tgz", + "/@babel/core/-/@babel/core-7.23.5.tgz", + "/@types/node/-/@types/node-20.10.4.tgz", + "/express/-/express-4.18.2.tgz", + "/react/-/react-18.2.0.tgz", + "/@angular/core/-/@angular/core-17.0.8.tgz", + "/typescript/-/typescript-5.3.3.tgz", + "/webpack/-/webpack-5.89.0.tgz", + "/@verdaccio/auth/-/@verdaccio/auth-7.0.0.tgz", + "/@nestjs/core/-/@nestjs/core-10.3.0.tgz", + }; + int hits = 0; + for (final String url : urls) { + final Optional result = ArtifactNameParser.parse("npm-group", url); + if (result.isPresent() && !result.get().isEmpty()) { + hits++; + } + } + final double hitRate = (double) hits / urls.length * 100; + MatcherAssert.assertThat( + String.format("npm hit rate %.1f%% must be >= 95%% (%d/%d)", hitRate, hits, urls.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + // ---- Hit rate test: Docker ---- + + @Test + void dockerHitRateAbove95Percent() { + final String[] urls = { + "/v2/library/nginx/manifests/latest", + "/v2/library/nginx/manifests/1.25", + "/v2/library/nginx/manifests/sha256:abc123def", + "/v2/library/nginx/blobs/sha256:abc123def", + "/v2/library/nginx/tags/list", + "/v2/library/ubuntu/manifests/22.04", + "/v2/library/ubuntu/blobs/sha256:xyz789", + "/v2/myorg/myapp/manifests/v1.0.0", + "/v2/myorg/myapp/blobs/sha256:abc", + "/v2/myorg/myapp/tags/list", + "/v2/registry.example.com/project/service/manifests/latest", + "/v2/registry.example.com/project/service/blobs/sha256:deadbeef", + "/v2/alpine/manifests/3.18", + "/v2/alpine/blobs/sha256:abc", + "/v2/python/manifests/3.12-slim", + }; + int hits = 0; + for (final String url : urls) { + final Optional result = ArtifactNameParser.parse("docker-group", url); + if (result.isPresent() && !result.get().isEmpty()) { + hits++; + } + } + final double hitRate = (double) hits / urls.length * 100; + MatcherAssert.assertThat( + String.format("Docker hit rate %.1f%% must be >= 95%% (%d/%d)", hitRate, hits, urls.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + // ---- Hit rate test: PyPI ---- + + @Test + void pypiHitRateAbove95Percent() { + final String[] urls = { + "/simple/numpy/", + "/simple/requests/", + "/simple/flask/", + "/simple/django/", + "/simple/scipy/", + "/simple/pandas/", + "/simple/tensorflow/", + "/simple/my-package/", + "/simple/My_Package/", + "/packages/numpy-1.24.0.whl", + "/packages/requests-2.31.0.tar.gz", + "/packages/flask-3.0.0.whl", + "/packages/django-5.0.tar.gz", + "/packages/scipy-1.12.0-cp39-cp39-linux_x86_64.whl", + "/packages/my_package-1.0.0.zip", + }; + int hits = 0; + for (final String url : urls) { + final Optional result = ArtifactNameParser.parse("pypi-group", url); + if (result.isPresent() && !result.get().isEmpty()) { + hits++; + } + } + final double hitRate = (double) hits / urls.length * 100; + MatcherAssert.assertThat( + String.format("PyPI hit rate %.1f%% must be >= 95%% (%d/%d)", hitRate, hits, urls.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + // ---- Cross-adapter hit rate: overall ---- + + @Test + void overallHitRateAbove95Percent() { + final String[][] cases = { + {"maven-group", "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar"}, + {"maven-group", "/com/google/guava/guava/maven-metadata.xml"}, + {"maven-group", "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar"}, + {"maven-group", "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.pom"}, + {"maven-group", "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar.sha1"}, + {"maven-group", "/org/apache/commons/commons-lang3/maven-metadata.xml"}, + {"maven-group", "/io/netty/netty-all/4.1.100.Final/netty-all-4.1.100.Final.jar"}, + {"maven-group", "/junit/junit/4.13.2/junit-4.13.2.jar"}, + {"npm-group", "/lodash"}, + {"npm-group", "/lodash/-/lodash-4.17.21.tgz"}, + {"npm-group", "/@babel/core"}, + {"npm-group", "/@babel/core/-/@babel/core-7.23.5.tgz"}, + {"npm-group", "/@types/node/-/@types/node-20.10.4.tgz"}, + {"docker-group", "/v2/library/nginx/manifests/latest"}, + {"docker-group", "/v2/library/nginx/blobs/sha256:abc123"}, + {"docker-group", "/v2/myorg/myapp/manifests/v1.0.0"}, + {"docker-group", "/v2/myorg/myapp/tags/list"}, + {"pypi-group", "/simple/numpy/"}, + {"pypi-group", "/simple/requests/"}, + {"pypi-group", "/packages/numpy-1.24.0.whl"}, + {"go-group", "/github.com/gin-gonic/gin/@v/v1.9.1.info"}, + {"go-group", "/github.com/gin-gonic/gin/@latest"}, + // Gradle (uses Maven format) + {"gradle-group", "/org/gradle/gradle-tooling-api/8.5/gradle-tooling-api-8.5.jar"}, + {"gradle-group", "/com/android/tools/build/gradle/8.2.0/gradle-8.2.0.pom"}, + // Gem + {"gem-group", "/gems/rails-7.1.2.gem"}, + {"gem-group", "/gems/nokogiri-1.15.4.gem"}, + {"gem-group", "/api/v1/gems/rails.json"}, + {"gem-group", "/quick/Marshal.4.8/rails-7.1.2.gemspec.rz"}, + // PHP/Composer + {"php-group", "/p2/monolog/monolog.json"}, + {"php-group", "/p2/symfony/console.json"}, + }; + int hits = 0; + for (final String[] tc : cases) { + final Optional result = ArtifactNameParser.parse(tc[0], tc[1]); + if (result.isPresent() && !result.get().isEmpty()) { + hits++; + } + } + final double hitRate = (double) hits / cases.length * 100; + MatcherAssert.assertThat( + String.format("Overall hit rate %.1f%% must be >= 95%% (%d/%d)", hitRate, hits, cases.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + // ---- Correctness: parsed names match what adapters store in DB ---- + + @Test + void mavenParsedNameMatchesDbFormat() { + // formatArtifactName replaces / with . on the groupId/artifactId path + MatcherAssert.assertThat( + ArtifactNameParser.parse("maven-group", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar"), + new IsEqual<>(Optional.of("com.google.guava.guava")) + ); + } + + @Test + void npmParsedNameMatchesDbFormat() { + // npm stores the exact package name (with scope if any) + MatcherAssert.assertThat( + ArtifactNameParser.parse("npm-group", + "/@babel/core/-/@babel/core-7.23.5.tgz"), + new IsEqual<>(Optional.of("@babel/core")) + ); + } + + @Test + void dockerParsedNameMatchesDbFormat() { + // Docker stores the image name including namespace + MatcherAssert.assertThat( + ArtifactNameParser.parse("docker-group", + "/v2/library/nginx/manifests/latest"), + new IsEqual<>(Optional.of("library/nginx")) + ); + } + + @Test + void pypiParsedNameMatchesDbFormat() { + // PyPI normalizes names (underscores/dots/hyphens → hyphens, lowercase) + MatcherAssert.assertThat( + ArtifactNameParser.parse("pypi-group", + "/simple/My_Package/"), + new IsEqual<>(Optional.of("my-package")) + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/group/GroupMetadataCacheTest.java b/artipie-main/src/test/java/com/artipie/group/GroupMetadataCacheTest.java new file mode 100644 index 000000000..f2ad8aaa1 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/group/GroupMetadataCacheTest.java @@ -0,0 +1,97 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** + * Tests for {@link GroupMetadataCache} stale fallback. + */ +class GroupMetadataCacheTest { + + @Test + void getStaleReturnsEmptyWhenNeverCached() throws Exception { + final GroupMetadataCache cache = new GroupMetadataCache("test-group"); + final Optional result = cache.getStale("/com/example/maven-metadata.xml") + .get(5, TimeUnit.SECONDS); + MatcherAssert.assertThat( + "Stale returns empty when never cached", + result.isPresent(), + Matchers.is(false) + ); + } + + @Test + void getStaleReturnsDataAfterPut() throws Exception { + final GroupMetadataCache cache = new GroupMetadataCache("test-group"); + final byte[] data = "test" + .getBytes(StandardCharsets.UTF_8); + cache.put("/com/example/stale1/maven-metadata.xml", data); + final Optional result = cache.getStale( + "/com/example/stale1/maven-metadata.xml" + ).get(5, TimeUnit.SECONDS); + MatcherAssert.assertThat( + "Stale returns data that was previously put", + result.isPresent(), + Matchers.is(true) + ); + MatcherAssert.assertThat( + "Stale data matches what was put", + new String(result.get(), StandardCharsets.UTF_8), + Matchers.equalTo("test") + ); + } + + @Test + void getStaleReturnsPreviousDataAfterInvalidate() throws Exception { + final GroupMetadataCache cache = new GroupMetadataCache("test-group"); + final byte[] data = "stale-data" + .getBytes(StandardCharsets.UTF_8); + cache.put("/com/example/stale2/maven-metadata.xml", data); + // Invalidate removes from L1/L2 but NOT from last-known-good + cache.invalidate("/com/example/stale2/maven-metadata.xml"); + // Primary get should return empty (invalidated) + final Optional primary = cache.get( + "/com/example/stale2/maven-metadata.xml" + ).get(5, TimeUnit.SECONDS); + MatcherAssert.assertThat( + "Primary cache returns empty after invalidate", + primary.isPresent(), + Matchers.is(false) + ); + // Stale should still return the data + final Optional stale = cache.getStale( + "/com/example/stale2/maven-metadata.xml" + ).get(5, TimeUnit.SECONDS); + MatcherAssert.assertThat( + "Stale still returns data after invalidate", + stale.isPresent(), + Matchers.is(true) + ); + } + + @Test + void getStaleUpdatesWithLatestPut() throws Exception { + final GroupMetadataCache cache = new GroupMetadataCache("test-group"); + cache.put("/com/example/stale3/maven-metadata.xml", + "v1".getBytes(StandardCharsets.UTF_8)); + cache.put("/com/example/stale3/maven-metadata.xml", + "v2".getBytes(StandardCharsets.UTF_8)); + final Optional result = cache.getStale( + "/com/example/stale3/maven-metadata.xml" + ).get(5, TimeUnit.SECONDS); + MatcherAssert.assertThat( + "Stale returns most recently put data", + new String(result.get(), StandardCharsets.UTF_8), + Matchers.equalTo("v2") + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/group/LocateHitRateTest.java b/artipie-main/src/test/java/com/artipie/group/LocateHitRateTest.java new file mode 100644 index 000000000..1d9e7a719 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/group/LocateHitRateTest.java @@ -0,0 +1,402 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import com.amihaiemil.eoyaml.Yaml; +import com.artipie.db.ArtifactDbFactory; +import com.artipie.db.PostgreSQLTestConfig; +import com.artipie.index.ArtifactDocument; +import com.artipie.index.DbArtifactIndex; +import org.hamcrest.MatcherAssert; +import org.hamcrest.core.IsEqual; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.Statement; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +/** + * End-to-end hit rate test for the full locate flow: + * HTTP URL path -> ArtifactNameParser -> locateByName() -> DB lookup -> repo found. + * + * This test populates the DB with artifacts exactly as each adapter stores them, + * then verifies that URL paths clients actually send result in successful lookups. + * Hit rate must be >= 95%. + * + * @since 1.21.0 + */ +@SuppressWarnings({"PMD.AvoidDuplicateLiterals", "PMD.TooManyMethods"}) +@Testcontainers +final class LocateHitRateTest { + + @Container + static final PostgreSQLContainer POSTGRES = PostgreSQLTestConfig.createContainer(); + + private DataSource dataSource; + private DbArtifactIndex index; + + @BeforeEach + void setUp() throws Exception { + this.dataSource = new ArtifactDbFactory( + Yaml.createYamlMappingBuilder().add( + "artifacts_database", + Yaml.createYamlMappingBuilder() + .add(ArtifactDbFactory.YAML_HOST, POSTGRES.getHost()) + .add(ArtifactDbFactory.YAML_PORT, String.valueOf(POSTGRES.getFirstMappedPort())) + .add(ArtifactDbFactory.YAML_DATABASE, POSTGRES.getDatabaseName()) + .add(ArtifactDbFactory.YAML_USER, POSTGRES.getUsername()) + .add(ArtifactDbFactory.YAML_PASSWORD, POSTGRES.getPassword()) + .build() + ).build(), + "artifacts" + ).initialize(); + try (Connection conn = this.dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DELETE FROM artifacts"); + } + this.index = new DbArtifactIndex(this.dataSource); + } + + @AfterEach + void tearDown() { + if (this.index != null) { + this.index.close(); + } + } + + @Test + void mavenEndToEndHitRate() throws Exception { + // Populate DB with Maven artifacts in the format UploadSlice/MavenProxyPackageProcessor + // stores them: name = groupId.artifactId (dots), version = version string + final String[][] artifacts = { + {"com.google.guava.guava", "31.1.3-jre"}, + {"com.google.guava.guava", "32.1.2-jre"}, + {"org.apache.commons.commons-lang3", "3.14.0"}, + {"org.slf4j.slf4j-api", "2.0.9"}, + {"junit.junit", "4.13.2"}, + {"io.netty.netty-all", "4.1.100.Final"}, + {"com.fasterxml.jackson.core.jackson-databind", "2.16.0"}, + {"org.projectlombok.lombok", "1.18.30"}, + {"org.springframework.spring-core", "6.1.0"}, + {"org.apache.maven.plugins.maven-compiler-plugin", "3.11.0"}, + {"org.apache.maven.plugins.maven-surefire-plugin", "3.2.3"}, + {"org.example.mylib", "1.0-SNAPSHOT"}, + }; + for (final String[] art : artifacts) { + this.index.index(new ArtifactDocument( + "maven", "maven-central", art[0], + art[0].substring(art[0].lastIndexOf('.') + 1), + art[1], 100_000L, Instant.now(), "proxy" + )).join(); + } + + // URLs that Maven clients actually send + final String[] urls = { + // Artifact JARs + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar", + "/com/google/guava/guava/32.1.2-jre/guava-32.1.2-jre.jar", + "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar", + "/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar", + "/junit/junit/4.13.2/junit-4.13.2.jar", + "/io/netty/netty-all/4.1.100.Final/netty-all-4.1.100.Final.jar", + "/com/fasterxml/jackson/core/jackson-databind/2.16.0/jackson-databind-2.16.0.jar", + "/org/projectlombok/lombok/1.18.30/lombok-1.18.30.jar", + "/org/springframework/spring-core/6.1.0/spring-core-6.1.0.jar", + // POMs + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.pom", + "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.pom", + "/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.pom", + // Checksums + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.sha1", + "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar.md5", + "/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.pom.sha256", + // Sources/javadoc + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre-sources.jar", + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre-javadoc.jar", + // Metadata + "/com/google/guava/guava/maven-metadata.xml", + "/org/apache/commons/commons-lang3/maven-metadata.xml", + "/org/slf4j/slf4j-api/maven-metadata.xml", + "/org/slf4j/slf4j-api/2.0.9/maven-metadata.xml", + "/com/google/guava/guava/maven-metadata.xml.sha1", + // Gradle module + "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.module", + // Plugins + "/org/apache/maven/plugins/maven-compiler-plugin/3.11.0/maven-compiler-plugin-3.11.0.jar", + "/org/apache/maven/plugins/maven-compiler-plugin/3.11.0/maven-compiler-plugin-3.11.0.pom", + "/org/apache/maven/plugins/maven-surefire-plugin/3.2.3/maven-surefire-plugin-3.2.3.jar", + // SNAPSHOT + "/org/example/mylib/1.0-SNAPSHOT/mylib-1.0-20230101.120000-1.jar", + "/org/example/mylib/1.0-SNAPSHOT/maven-metadata.xml", + }; + + int hits = 0; + int total = urls.length; + for (final String url : urls) { + final Optional parsed = ArtifactNameParser.parse("maven-group", url); + if (parsed.isPresent()) { + final List repos = this.index.locateByName(parsed.get()).join(); + if (!repos.isEmpty()) { + hits++; + } + } + } + final double hitRate = (double) hits / total * 100; + MatcherAssert.assertThat( + String.format( + "Maven E2E hit rate %.1f%% must be >= 95%% (%d/%d)", + hitRate, hits, total + ), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + @Test + void npmEndToEndHitRate() throws Exception { + // npm stores: name = package name (with scope), version = version string + final String[][] artifacts = { + {"lodash", "4.17.21"}, + {"@babel/core", "7.23.5"}, + {"@types/node", "20.10.4"}, + {"express", "4.18.2"}, + {"react", "18.2.0"}, + {"typescript", "5.3.3"}, + {"webpack", "5.89.0"}, + {"@angular/core", "17.0.8"}, + }; + for (final String[] art : artifacts) { + this.index.index(new ArtifactDocument( + "npm", "npm-proxy", art[0], art[0], + art[1], 50_000L, Instant.now(), "proxy" + )).join(); + } + + final String[] urls = { + // Metadata + "/lodash", + "/@babel/core", + "/@types/node", + "/express", + "/react", + "/typescript", + "/webpack", + "/@angular/core", + // Tarballs + "/lodash/-/lodash-4.17.21.tgz", + "/@babel/core/-/@babel/core-7.23.5.tgz", + "/@types/node/-/@types/node-20.10.4.tgz", + "/express/-/express-4.18.2.tgz", + "/react/-/react-18.2.0.tgz", + "/typescript/-/typescript-5.3.3.tgz", + "/webpack/-/webpack-5.89.0.tgz", + "/@angular/core/-/@angular/core-17.0.8.tgz", + }; + + int hits = 0; + for (final String url : urls) { + final Optional parsed = ArtifactNameParser.parse("npm-group", url); + if (parsed.isPresent()) { + final List repos = this.index.locateByName(parsed.get()).join(); + if (!repos.isEmpty()) { + hits++; + } + } + } + final double hitRate = (double) hits / urls.length * 100; + MatcherAssert.assertThat( + String.format("npm E2E hit rate %.1f%% must be >= 95%% (%d/%d)", + hitRate, hits, urls.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + @Test + void dockerEndToEndHitRate() throws Exception { + // Docker stores: name = image name, version = manifest digest + final String[][] artifacts = { + {"library/nginx", "sha256:abc123"}, + {"library/ubuntu", "sha256:def456"}, + {"myorg/myapp", "sha256:789xyz"}, + {"alpine", "sha256:aaa111"}, + {"python", "sha256:bbb222"}, + }; + for (final String[] art : artifacts) { + this.index.index(new ArtifactDocument( + "docker", "docker-proxy", art[0], art[0], + art[1], 200_000_000L, Instant.now(), "proxy" + )).join(); + } + + final String[] urls = { + "/v2/library/nginx/manifests/latest", + "/v2/library/nginx/manifests/1.25", + "/v2/library/nginx/manifests/sha256:abc123", + "/v2/library/nginx/blobs/sha256:abc123", + "/v2/library/nginx/tags/list", + "/v2/library/ubuntu/manifests/22.04", + "/v2/library/ubuntu/blobs/sha256:def456", + "/v2/myorg/myapp/manifests/v1.0.0", + "/v2/myorg/myapp/blobs/sha256:789xyz", + "/v2/myorg/myapp/tags/list", + "/v2/alpine/manifests/3.18", + "/v2/alpine/blobs/sha256:aaa111", + "/v2/python/manifests/3.12-slim", + "/v2/python/blobs/sha256:bbb222", + }; + + int hits = 0; + for (final String url : urls) { + final Optional parsed = ArtifactNameParser.parse("docker-group", url); + if (parsed.isPresent()) { + final List repos = this.index.locateByName(parsed.get()).join(); + if (!repos.isEmpty()) { + hits++; + } + } + } + final double hitRate = (double) hits / urls.length * 100; + MatcherAssert.assertThat( + String.format("Docker E2E hit rate %.1f%% must be >= 95%% (%d/%d)", + hitRate, hits, urls.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + @Test + void pypiEndToEndHitRate() throws Exception { + // PyPI stores: name = normalized project name (lowercase, hyphens) + final String[][] artifacts = { + {"numpy", "1.24.0"}, + {"requests", "2.31.0"}, + {"flask", "3.0.0"}, + {"django", "5.0"}, + {"scipy", "1.12.0"}, + {"my-package", "1.0.0"}, + }; + for (final String[] art : artifacts) { + this.index.index(new ArtifactDocument( + "pypi", "pypi-proxy", art[0], art[0], + art[1], 10_000_000L, Instant.now(), "proxy" + )).join(); + } + + final String[] urls = { + "/simple/numpy/", + "/simple/requests/", + "/simple/flask/", + "/simple/django/", + "/simple/scipy/", + "/simple/my-package/", + "/simple/My_Package/", + "/packages/numpy-1.24.0.whl", + "/packages/requests-2.31.0.tar.gz", + "/packages/flask-3.0.0.whl", + "/packages/django-5.0.tar.gz", + "/packages/scipy-1.12.0-cp39-cp39-linux_x86_64.whl", + "/packages/my_package-1.0.0.zip", + }; + + int hits = 0; + for (final String url : urls) { + final Optional parsed = ArtifactNameParser.parse("pypi-group", url); + if (parsed.isPresent()) { + final List repos = this.index.locateByName(parsed.get()).join(); + if (!repos.isEmpty()) { + hits++; + } + } + } + final double hitRate = (double) hits / urls.length * 100; + MatcherAssert.assertThat( + String.format("PyPI E2E hit rate %.1f%% must be >= 95%% (%d/%d)", + hitRate, hits, urls.length), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } + + @Test + void combinedEndToEndHitRate() throws Exception { + // Populate a realistic mixed-adapter database + // Maven artifacts + for (final String name : List.of( + "com.google.guava.guava", "org.apache.commons.commons-lang3", + "org.slf4j.slf4j-api", "junit.junit", "io.netty.netty-all" + )) { + this.index.index(new ArtifactDocument( + "maven", "maven-central", name, + name.substring(name.lastIndexOf('.') + 1), + "1.0.0", 100_000L, Instant.now(), "proxy" + )).join(); + } + // npm artifacts + for (final String name : List.of("lodash", "@babel/core", "express")) { + this.index.index(new ArtifactDocument( + "npm", "npm-proxy", name, name, + "1.0.0", 50_000L, Instant.now(), "proxy" + )).join(); + } + // Docker artifacts + for (final String name : List.of("library/nginx", "alpine")) { + this.index.index(new ArtifactDocument( + "docker", "docker-proxy", name, name, + "sha256:abc", 200_000_000L, Instant.now(), "proxy" + )).join(); + } + + final String[][] cases = { + {"maven-group", "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar"}, + {"maven-group", "/com/google/guava/guava/maven-metadata.xml"}, + {"maven-group", "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.pom"}, + {"maven-group", "/com/google/guava/guava/31.1.3-jre/guava-31.1.3-jre.jar.sha1"}, + {"maven-group", "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar"}, + {"maven-group", "/org/apache/commons/commons-lang3/maven-metadata.xml"}, + {"maven-group", "/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar"}, + {"maven-group", "/junit/junit/4.13.2/junit-4.13.2.jar"}, + {"maven-group", "/io/netty/netty-all/4.1.100.Final/netty-all-4.1.100.Final.jar"}, + {"npm-group", "/lodash"}, + {"npm-group", "/lodash/-/lodash-4.17.21.tgz"}, + {"npm-group", "/@babel/core"}, + {"npm-group", "/@babel/core/-/@babel/core-7.23.5.tgz"}, + {"npm-group", "/express"}, + {"npm-group", "/express/-/express-4.18.2.tgz"}, + {"docker-group", "/v2/library/nginx/manifests/latest"}, + {"docker-group", "/v2/library/nginx/blobs/sha256:abc"}, + {"docker-group", "/v2/alpine/manifests/3.18"}, + {"docker-group", "/v2/alpine/tags/list"}, + }; + + int hits = 0; + for (final String[] tc : cases) { + final Optional parsed = ArtifactNameParser.parse(tc[0], tc[1]); + if (parsed.isPresent()) { + final List repos = this.index.locateByName(parsed.get()).join(); + if (!repos.isEmpty()) { + hits++; + } + } + } + final double hitRate = (double) hits / cases.length * 100; + MatcherAssert.assertThat( + String.format( + "Combined E2E hit rate %.1f%% must be >= 95%% (%d/%d)", + hitRate, hits, cases.length + ), + hitRate >= 95.0, + new IsEqual<>(true) + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/group/MavenGroupSliceTest.java b/artipie-main/src/test/java/com/artipie/group/MavenGroupSliceTest.java index d3f0ee874..05043e6e9 100644 --- a/artipie-main/src/test/java/com/artipie/group/MavenGroupSliceTest.java +++ b/artipie-main/src/test/java/com/artipie/group/MavenGroupSliceTest.java @@ -381,6 +381,58 @@ void supportsMavenPomRelocation() throws Exception { ); } + @Test + void returnsStaleMetadataWhenAllMembersFailAndCacheExpired() throws Exception { + // Pre-populate a GroupMetadataCache with stale data + final GroupMetadataCache cache = new GroupMetadataCache("stale-test-group"); + final String staleMetadata = "" + + "com.stale" + + "fallback" + + "" + + "1.0-stale" + + ""; + final String stalePath = "/com/stale/fallback/maven-metadata.xml"; + // Directly put into cache (populates both L1 and last-known-good) + cache.put(stalePath, staleMetadata.getBytes(StandardCharsets.UTF_8)); + // Invalidate L1/L2 to simulate cache expiry + cache.invalidate(stalePath); + + // All members return 503 (upstream down) + final Map members = new HashMap<>(); + members.put("repo1", new StaticSlice(RsStatus.SERVICE_UNAVAILABLE)); + + final MavenGroupSlice slice = new MavenGroupSlice( + new FakeGroupSlice(), + "stale-test-group", + List.of("repo1"), + new MapResolver(members), + 8080, + 0, + cache + ); + + final Response response = slice.response( + new RequestLine("GET", stalePath), + Headers.EMPTY, + Content.EMPTY + ).get(10, TimeUnit.SECONDS); + + MatcherAssert.assertThat( + "Stale metadata is returned when all members fail", + response.status(), + Matchers.equalTo(RsStatus.OK) + ); + + final String body = new String( + response.body().asBytes(), StandardCharsets.UTF_8 + ); + MatcherAssert.assertThat( + "Response contains stale version", + body, + Matchers.containsString("1.0-stale") + ); + } + // Helper classes private static final class MapResolver implements SliceResolver { diff --git a/artipie-main/src/test/java/com/artipie/group/MemberSliceTest.java b/artipie-main/src/test/java/com/artipie/group/MemberSliceTest.java new file mode 100644 index 000000000..c54a1c909 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/group/MemberSliceTest.java @@ -0,0 +1,62 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import com.artipie.http.timeout.AutoBlockRegistry; +import com.artipie.http.timeout.AutoBlockSettings; +import org.junit.jupiter.api.Test; +import java.time.Duration; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +final class MemberSliceTest { + + @Test + void reportsOpenCircuitFromRegistry() { + final AutoBlockRegistry registry = new AutoBlockRegistry(new AutoBlockSettings( + 1, Duration.ofMinutes(5), Duration.ofMinutes(60) + )); + final MemberSlice member = new MemberSlice("test-member", null, registry); + assertThat(member.isCircuitOpen(), is(false)); + registry.recordFailure("test-member"); + assertThat(member.isCircuitOpen(), is(true)); + } + + @Test + void recordsSuccessViaRegistry() { + final AutoBlockRegistry registry = new AutoBlockRegistry(new AutoBlockSettings( + 1, Duration.ofMinutes(5), Duration.ofMinutes(60) + )); + final MemberSlice member = new MemberSlice("test-member", null, registry); + registry.recordFailure("test-member"); + assertThat(member.isCircuitOpen(), is(true)); + member.recordSuccess(); + assertThat(member.isCircuitOpen(), is(false)); + } + + @Test + void recordsFailureViaRegistry() { + final AutoBlockRegistry registry = new AutoBlockRegistry(new AutoBlockSettings( + 2, Duration.ofMinutes(5), Duration.ofMinutes(60) + )); + final MemberSlice member = new MemberSlice("test-member", null, registry); + member.recordFailure(); + assertThat(member.isCircuitOpen(), is(false)); + member.recordFailure(); + assertThat(member.isCircuitOpen(), is(true)); + } + + @Test + void reportsCircuitState() { + final AutoBlockRegistry registry = new AutoBlockRegistry(new AutoBlockSettings( + 1, Duration.ofMinutes(5), Duration.ofMinutes(60) + )); + final MemberSlice member = new MemberSlice("test-member", null, registry); + assertThat(member.circuitState(), equalTo("ONLINE")); + registry.recordFailure("test-member"); + assertThat(member.circuitState(), equalTo("BLOCKED")); + } +} diff --git a/artipie-main/src/test/java/com/artipie/group/RoutingRuleTest.java b/artipie-main/src/test/java/com/artipie/group/RoutingRuleTest.java new file mode 100644 index 000000000..a95ebf173 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/group/RoutingRuleTest.java @@ -0,0 +1,96 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link RoutingRule}. + */ +class RoutingRuleTest { + + @Test + void pathPrefixMatchesExactPrefix() { + final RoutingRule rule = new RoutingRule.PathPrefix("repo1", "com/mycompany/"); + assertThat(rule.matches("/com/mycompany/foo/1.0/foo-1.0.jar"), is(true)); + assertThat(rule.matches("com/mycompany/bar"), is(true)); + } + + @Test + void pathPrefixDoesNotMatchDifferentPrefix() { + final RoutingRule rule = new RoutingRule.PathPrefix("repo1", "com/mycompany/"); + assertThat(rule.matches("/org/apache/foo"), is(false)); + assertThat(rule.matches("org/other/bar"), is(false)); + } + + @Test + void pathPrefixNormalizesLeadingSlash() { + final RoutingRule rule = new RoutingRule.PathPrefix("repo1", "com/example/"); + assertThat(rule.matches("/com/example/test"), is(true)); + assertThat(rule.matches("com/example/test"), is(true)); + } + + @Test + void pathPatternMatchesRegex() { + final RoutingRule rule = new RoutingRule.PathPattern("repo1", "org/apache/.*"); + assertThat(rule.matches("/org/apache/commons/1.0/commons-1.0.jar"), is(true)); + assertThat(rule.matches("org/apache/maven/settings.xml"), is(true)); + } + + @Test + void pathPatternDoesNotMatchDifferentPath() { + final RoutingRule rule = new RoutingRule.PathPattern("repo1", "org/apache/.*"); + assertThat(rule.matches("/com/example/foo"), is(false)); + } + + @Test + void pathPatternNormalizesLeadingSlash() { + final RoutingRule rule = new RoutingRule.PathPattern("repo1", "com/.*\\.jar"); + assertThat(rule.matches("/com/example/foo-1.0.jar"), is(true)); + assertThat(rule.matches("com/example/foo-1.0.jar"), is(true)); + assertThat(rule.matches("/com/example/foo-1.0.pom"), is(false)); + } + + @Test + void memberReturnsMemberName() { + assertThat( + new RoutingRule.PathPrefix("test-member", "com/").member(), + equalTo("test-member") + ); + assertThat( + new RoutingRule.PathPattern("test-member", ".*").member(), + equalTo("test-member") + ); + } + + @Test + void pathPrefixRejectsNullMember() { + assertThrows( + NullPointerException.class, + () -> new RoutingRule.PathPrefix(null, "com/") + ); + } + + @Test + void pathPrefixRejectsNullPrefix() { + assertThrows( + NullPointerException.class, + () -> new RoutingRule.PathPrefix("member", null) + ); + } + + @Test + void pathPatternRejectsInvalidRegex() { + assertThrows( + java.util.regex.PatternSyntaxException.class, + () -> new RoutingRule.PathPattern("member", "[invalid") + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/group/WritableGroupSliceTest.java b/artipie-main/src/test/java/com/artipie/group/WritableGroupSliceTest.java new file mode 100644 index 000000000..3cd3b4ce0 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/group/WritableGroupSliceTest.java @@ -0,0 +1,89 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.group; + +import com.artipie.asto.Content; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.Slice; +import com.artipie.http.rq.RequestLine; +import com.artipie.http.rq.RqMethod; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for {@link WritableGroupSlice}. + */ +class WritableGroupSliceTest { + + @Test + void routesGetToReadDelegate() throws Exception { + final AtomicBoolean readCalled = new AtomicBoolean(false); + final Slice readSlice = (line, headers, body) -> { + readCalled.set(true); + return CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + }; + final Slice writeSlice = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + final Response resp = new WritableGroupSlice(readSlice, writeSlice) + .response(new RequestLine(RqMethod.GET, "/test"), Headers.EMPTY, Content.EMPTY) + .get(); + assertThat(readCalled.get(), is(true)); + assertThat(resp.status().code(), equalTo(200)); + } + + @Test + void routesPutToWriteTarget() throws Exception { + final AtomicBoolean writeCalled = new AtomicBoolean(false); + final Slice readSlice = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + final Slice writeSlice = (line, headers, body) -> { + writeCalled.set(true); + return CompletableFuture.completedFuture(ResponseBuilder.created().build()); + }; + final Response resp = new WritableGroupSlice(readSlice, writeSlice) + .response(new RequestLine(RqMethod.PUT, "/test"), Headers.EMPTY, Content.EMPTY) + .get(); + assertThat(writeCalled.get(), is(true)); + assertThat(resp.status().code(), equalTo(201)); + } + + @Test + void routesDeleteToWriteTarget() throws Exception { + final AtomicBoolean writeCalled = new AtomicBoolean(false); + final Slice writeSlice = (line, headers, body) -> { + writeCalled.set(true); + return CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + }; + final Slice readSlice = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + new WritableGroupSlice(readSlice, writeSlice) + .response(new RequestLine(RqMethod.DELETE, "/test"), Headers.EMPTY, Content.EMPTY) + .get(); + assertThat(writeCalled.get(), is(true)); + } + + @Test + void routesHeadToReadDelegate() throws Exception { + final AtomicBoolean readCalled = new AtomicBoolean(false); + final Slice readSlice = (line, headers, body) -> { + readCalled.set(true); + return CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + }; + final Slice writeSlice = (line, headers, body) -> + CompletableFuture.completedFuture(ResponseBuilder.ok().build()); + new WritableGroupSlice(readSlice, writeSlice) + .response(new RequestLine(RqMethod.HEAD, "/test"), Headers.EMPTY, Content.EMPTY) + .get(); + assertThat(readCalled.get(), is(true)); + } +} diff --git a/artipie-main/src/test/java/com/artipie/http/ApiRoutingSliceTest.java b/artipie-main/src/test/java/com/artipie/http/ApiRoutingSliceTest.java index aa9670119..781988ad0 100644 --- a/artipie-main/src/test/java/com/artipie/http/ApiRoutingSliceTest.java +++ b/artipie-main/src/test/java/com/artipie/http/ApiRoutingSliceTest.java @@ -14,6 +14,7 @@ import org.junit.jupiter.params.provider.CsvSource; import java.net.URI; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; /** @@ -54,7 +55,7 @@ void shouldRewriteApiPaths(final String input, final String expected) { Headers.EMPTY, Content.EMPTY ).join(); - + MatcherAssert.assertThat( "Path should be rewritten correctly", captured.get(), @@ -75,7 +76,7 @@ void shouldPreserveQueryParameters() { Headers.EMPTY, Content.EMPTY ).join(); - + MatcherAssert.assertThat( "Path should be rewritten", captured.get().getPath(), @@ -101,7 +102,7 @@ void shouldHandleRootApiPath() { Headers.EMPTY, Content.EMPTY ).join(); - + MatcherAssert.assertThat( "Root API path should be rewritten", captured.get(), @@ -122,11 +123,53 @@ void shouldPassThroughNonApiPaths() { Headers.EMPTY, Content.EMPTY ).join(); - + MatcherAssert.assertThat( "Non-API paths should pass through unchanged", captured.get(), Matchers.equalTo("/direct/repo/path") ); } + + /** + * When repo registry is available and segments[1] is NOT a known repo, + * the first segment should be treated as repo_name, not repo_type. + * This handles: /api/npm/@scope%2fpkg -> /npm/@scope/pkg + * and: /api/npm/some-package -> /npm/some-package + */ + @ParameterizedTest + @CsvSource({ + // Scoped npm package: npm is repo name, @ayd%2fnpm-proxy-test is path + "/api/npm/@ayd%2fnpm-proxy-test,/npm/@ayd/npm-proxy-test", + // Non-scoped package: npm is repo name, some-package is path + "/api/npm/some-package,/npm/some-package", + // With prefix + "/test_prefix/api/npm/@ayd%2fnpm-proxy-test,/test_prefix/npm/@ayd/npm-proxy-test", + "/test_prefix/api/npm/some-package,/test_prefix/npm/some-package", + // repo_type + repo_name still works when repo name exists in registry + "/api/npm/npm/some-package,/npm/some-package", + "/test_prefix/api/npm/npm/@ayd%2fnpm-proxy-test,/test_prefix/npm/@ayd/npm-proxy-test" + }) + void shouldDisambiguateWithRepoRegistry(final String input, final String expected) { + // Registry knows about repo "npm" but NOT "some-package" or "@ayd..." + final Set knownRepos = Set.of("npm"); + final AtomicReference captured = new AtomicReference<>(); + new ApiRoutingSlice( + (line, headers, body) -> { + captured.set(line.uri().getPath()); + return ResponseBuilder.ok().completedFuture(); + }, + knownRepos::contains + ).response( + new RequestLine(RqMethod.PUT, input), + Headers.EMPTY, + Content.EMPTY + ).join(); + + MatcherAssert.assertThat( + "Path should be rewritten correctly with repo registry", + captured.get(), + Matchers.equalTo(expected) + ); + } } diff --git a/artipie-main/src/test/java/com/artipie/http/HealthSliceTest.java b/artipie-main/src/test/java/com/artipie/http/HealthSliceTest.java index 07a7360df..124f1807b 100644 --- a/artipie-main/src/test/java/com/artipie/http/HealthSliceTest.java +++ b/artipie-main/src/test/java/com/artipie/http/HealthSliceTest.java @@ -5,19 +5,17 @@ package com.artipie.http; import com.artipie.asto.Content; -import com.artipie.asto.Key; -import com.artipie.asto.Meta; -import com.artipie.asto.Storage; -import com.artipie.http.hm.ResponseAssert; import com.artipie.http.rq.RequestLine; import com.artipie.http.rq.RqMethod; -import com.artipie.test.TestSettings; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; import org.junit.jupiter.api.Test; -import java.util.Collection; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.function.Function; +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonReader; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; /** * Test case for {@link HealthSlice}. @@ -25,78 +23,25 @@ * @since 0.10 */ final class HealthSliceTest { - /** - * Request line for health endpoint. - */ - private static final RequestLine REQ_LINE = new RequestLine(RqMethod.GET, "/.health"); - @Test - void returnsOkForValidStorage() { - ResponseAssert.check( - new HealthSlice(new TestSettings()).response( - REQ_LINE, Headers.EMPTY, Content.EMPTY - ).join(), - RsStatus.OK, "[{\"storage\":\"ok\"}]".getBytes() - ); - } + private static final RequestLine REQ_LINE = new RequestLine(RqMethod.GET, "/.health"); @Test - void returnsBadRequestForBrokenStorage() { - ResponseAssert.check( - new HealthSlice(new TestSettings(new FakeStorage())).response( - REQ_LINE, Headers.EMPTY, Content.EMPTY - ).join(), - RsStatus.SERVICE_UNAVAILABLE, "[{\"storage\":\"failure\"}]".getBytes() + void returnsOkImmediately() { + final Response response = new HealthSlice().response( + REQ_LINE, Headers.EMPTY, Content.EMPTY + ).join(); + MatcherAssert.assertThat( + "status should be OK", + response.status(), Matchers.is(RsStatus.OK) ); - } - - /** - * Implementation of broken storage. - * All methods throw exception. - * - * @since 0.10 - */ - private static class FakeStorage implements Storage { - @Override - public CompletableFuture exists(final Key key) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture> list(final Key prefix) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture save(final Key key, final Content content) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture move(final Key source, final Key destination) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture metadata(final Key key) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture value(final Key key) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture delete(final Key key) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletionStage exclusively( - final Key key, - final Function> function) { - throw new UnsupportedOperationException(); + final String body = new String(response.body().asBytes(), StandardCharsets.UTF_8); + try (JsonReader reader = Json.createReader(new StringReader(body))) { + final JsonObject json = reader.readObject(); + MatcherAssert.assertThat( + "status field should be ok", + json.getString("status"), Matchers.is("ok") + ); } } } diff --git a/artipie-main/src/test/java/com/artipie/http/SliceByPathPrefixTest.java b/artipie-main/src/test/java/com/artipie/http/SliceByPathPrefixTest.java index 46384bbe0..0a911d79f 100644 --- a/artipie-main/src/test/java/com/artipie/http/SliceByPathPrefixTest.java +++ b/artipie-main/src/test/java/com/artipie/http/SliceByPathPrefixTest.java @@ -10,22 +10,15 @@ import com.artipie.http.rq.RequestLine; import com.artipie.http.rq.RqMethod; import com.artipie.settings.PrefixesConfig; -import com.artipie.settings.repo.Repositories; import com.artipie.test.TestSettings; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; /** * Tests for {@link SliceByPath} with prefix support. @@ -35,177 +28,143 @@ class SliceByPathPrefixTest { @Test void routesUnprefixedPath() { final PrefixesConfig prefixes = new PrefixesConfig(Arrays.asList("p1", "p2")); - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - slice.response( + final RecordingSlices slices = new RecordingSlices(); + new SliceByPath(slices, prefixes).response( new RequestLine(RqMethod.GET, "/test/artifact.jar"), Headers.EMPTY, Content.EMPTY ).join(); - - final ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices).slice(keyCaptor.capture(), anyInt()); - assertEquals("test", keyCaptor.getValue().string()); + assertEquals("test", slices.lastKey()); } @Test void stripsPrefixFromPath() { final PrefixesConfig prefixes = new PrefixesConfig(Arrays.asList("p1", "p2")); - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - slice.response( + final RecordingSlices slices = new RecordingSlices(); + new SliceByPath(slices, prefixes).response( new RequestLine(RqMethod.GET, "/p1/test/artifact.jar"), Headers.EMPTY, Content.EMPTY ).join(); - - final ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices).slice(keyCaptor.capture(), anyInt()); - assertEquals("test", keyCaptor.getValue().string()); + assertEquals("test", slices.lastKey()); } @Test void stripsMultiplePrefixes() { - final PrefixesConfig prefixes = new PrefixesConfig(Arrays.asList("p1", "p2", "migration")); - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - // Test p1 prefix - slice.response( + final PrefixesConfig prefixes = new PrefixesConfig( + Arrays.asList("p1", "p2", "migration") + ); + final RecordingSlices s1 = new RecordingSlices(); + new SliceByPath(s1, prefixes).response( new RequestLine(RqMethod.GET, "/p1/maven/artifact.jar"), Headers.EMPTY, Content.EMPTY ).join(); - - ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices).slice(keyCaptor.capture(), anyInt()); - assertEquals("maven", keyCaptor.getValue().string()); - - // Test p2 prefix - final RepositorySlices slices2 = mockSlices(); - final SliceByPath slice2 = new SliceByPath(slices2, prefixes); - slice2.response( + assertEquals("maven", s1.lastKey()); + final RecordingSlices s2 = new RecordingSlices(); + new SliceByPath(s2, prefixes).response( new RequestLine(RqMethod.GET, "/p2/npm/package.tgz"), Headers.EMPTY, Content.EMPTY ).join(); - - keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices2).slice(keyCaptor.capture(), anyInt()); - assertEquals("npm", keyCaptor.getValue().string()); - - // Test migration prefix - final RepositorySlices slices3 = mockSlices(); - final SliceByPath slice3 = new SliceByPath(slices3, prefixes); - slice3.response( + assertEquals("npm", s2.lastKey()); + final RecordingSlices s3 = new RecordingSlices(); + new SliceByPath(s3, prefixes).response( new RequestLine(RqMethod.GET, "/migration/docker/image"), Headers.EMPTY, Content.EMPTY ).join(); - - keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices3).slice(keyCaptor.capture(), anyInt()); - assertEquals("docker", keyCaptor.getValue().string()); + assertEquals("docker", s3.lastKey()); } @Test void doesNotStripUnknownPrefix() { final PrefixesConfig prefixes = new PrefixesConfig(Arrays.asList("p1", "p2")); - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - slice.response( + final RecordingSlices slices = new RecordingSlices(); + new SliceByPath(slices, prefixes).response( new RequestLine(RqMethod.GET, "/unknown/test/artifact.jar"), Headers.EMPTY, Content.EMPTY ).join(); - - final ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices).slice(keyCaptor.capture(), anyInt()); - assertEquals("unknown", keyCaptor.getValue().string()); + assertEquals("unknown", slices.lastKey()); } @Test void handlesEmptyPrefixList() { final PrefixesConfig prefixes = new PrefixesConfig(); - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - slice.response( + final RecordingSlices slices = new RecordingSlices(); + new SliceByPath(slices, prefixes).response( new RequestLine(RqMethod.GET, "/test/artifact.jar"), Headers.EMPTY, Content.EMPTY ).join(); - - final ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices).slice(keyCaptor.capture(), anyInt()); - assertEquals("test", keyCaptor.getValue().string()); + assertEquals("test", slices.lastKey()); } @Test void supportsAllHttpMethods() { final PrefixesConfig prefixes = new PrefixesConfig(Arrays.asList("p1")); - for (final RqMethod method : Arrays.asList( RqMethod.GET, RqMethod.HEAD, RqMethod.PUT, RqMethod.POST, RqMethod.DELETE )) { - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - slice.response( + final RecordingSlices slices = new RecordingSlices(); + new SliceByPath(slices, prefixes).response( new RequestLine(method, "/p1/test/artifact.jar"), Headers.EMPTY, Content.EMPTY ).join(); - - final ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(Key.class); - verify(slices).slice(keyCaptor.capture(), anyInt()); - assertEquals("test", keyCaptor.getValue().string()); + assertEquals("test", slices.lastKey()); } } @Test void handlesRootPath() { final PrefixesConfig prefixes = new PrefixesConfig(Arrays.asList("p1")); - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - final Response response = slice.response( + final RecordingSlices slices = new RecordingSlices(); + final Response response = new SliceByPath(slices, prefixes).response( new RequestLine(RqMethod.GET, "/"), Headers.EMPTY, Content.EMPTY ).join(); - assertEquals(404, response.status().code()); } @Test void handlesPrefixOnlyPath() { final PrefixesConfig prefixes = new PrefixesConfig(Arrays.asList("p1")); - final RepositorySlices slices = mockSlices(); - final SliceByPath slice = new SliceByPath(slices, prefixes); - - final Response response = slice.response( + final RecordingSlices slices = new RecordingSlices(); + final Response response = new SliceByPath(slices, prefixes).response( new RequestLine(RqMethod.GET, "/p1"), Headers.EMPTY, Content.EMPTY ).join(); - - // Should result in empty repo name after stripping assertEquals(404, response.status().code()); } - private RepositorySlices mockSlices() { - final RepositorySlices slices = mock(RepositorySlices.class); - final Slice repoSlice = mock(Slice.class); - when(repoSlice.response(any(), any(), any())).thenReturn( - CompletableFuture.completedFuture( + /** + * Subclass of RepositorySlices that records which Key was passed to slice(). + */ + private static final class RecordingSlices extends RepositorySlices { + /** + * Keys that were requested. + */ + private final List keys; + + RecordingSlices() { + super(new TestSettings(), null, null); + this.keys = Collections.synchronizedList(new ArrayList<>(4)); + } + + @Override + public Slice slice(final Key name, final int port) { + this.keys.add(name.string()); + return (line, headers, body) -> CompletableFuture.completedFuture( ResponseBuilder.ok().build() - ) - ); - when(slices.slice(any(Key.class), anyInt())).thenReturn(repoSlice); - return slices; + ); + } + + String lastKey() { + return this.keys.get(this.keys.size() - 1); + } } } diff --git a/artipie-main/src/test/java/com/artipie/index/DbArtifactIndexTest.java b/artipie-main/src/test/java/com/artipie/index/DbArtifactIndexTest.java new file mode 100644 index 000000000..3ed908450 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/index/DbArtifactIndexTest.java @@ -0,0 +1,438 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.index; + +import com.amihaiemil.eoyaml.Yaml; +import com.artipie.db.ArtifactDbFactory; +import com.artipie.db.PostgreSQLTestConfig; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.hamcrest.core.IsEqual; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Tests for {@link DbArtifactIndex}. + * Uses Testcontainers PostgreSQL for integration testing. + * + * @since 1.20.13 + */ +@SuppressWarnings({"PMD.AvoidDuplicateLiterals", "PMD.TooManyMethods"}) +@Testcontainers +class DbArtifactIndexTest { + + /** + * PostgreSQL test container. + */ + @Container + static final PostgreSQLContainer POSTGRES = PostgreSQLTestConfig.createContainer(); + + /** + * DataSource for tests. + */ + private DataSource dataSource; + + /** + * Index under test. + */ + private DbArtifactIndex index; + + @BeforeEach + void setUp() throws Exception { + this.dataSource = new ArtifactDbFactory( + Yaml.createYamlMappingBuilder().add( + "artifacts_database", + Yaml.createYamlMappingBuilder() + .add(ArtifactDbFactory.YAML_HOST, POSTGRES.getHost()) + .add(ArtifactDbFactory.YAML_PORT, String.valueOf(POSTGRES.getFirstMappedPort())) + .add(ArtifactDbFactory.YAML_DATABASE, POSTGRES.getDatabaseName()) + .add(ArtifactDbFactory.YAML_USER, POSTGRES.getUsername()) + .add(ArtifactDbFactory.YAML_PASSWORD, POSTGRES.getPassword()) + .build() + ).build(), + "artifacts" + ).initialize(); + // Clean up artifacts table before each test + try (Connection conn = this.dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DELETE FROM artifacts"); + } + this.index = new DbArtifactIndex(this.dataSource); + } + + @AfterEach + void tearDown() { + if (this.index != null) { + this.index.close(); + } + } + + @Test + void indexAndLocate() throws Exception { + final ArtifactDocument doc = new ArtifactDocument( + "maven", "my-repo", "com/example/lib", "lib", + "1.0.0", 1024L, Instant.now(), "admin" + ); + this.index.index(doc).join(); + final List repos = this.index.locate("com/example/lib").join(); + MatcherAssert.assertThat(repos, Matchers.contains("my-repo")); + } + + @Test + void indexAndSearch() throws Exception { + this.index.index(new ArtifactDocument( + "maven", "repo1", "com/example/alpha-lib", "alpha-lib", + "1.0", 100L, Instant.now(), "user1" + )).join(); + this.index.index(new ArtifactDocument( + "maven", "repo2", "com/example/beta-lib", "beta-lib", + "2.0", 200L, Instant.now(), "user2" + )).join(); + final SearchResult result = this.index.search("lib", 10, 0).join(); + MatcherAssert.assertThat( + "Should find both artifacts containing 'lib'", + result.documents().size(), + new IsEqual<>(2) + ); + MatcherAssert.assertThat( + "Total hits should be 2", + result.totalHits(), + new IsEqual<>(2L) + ); + } + + @Test + void indexUpsert() throws Exception { + final Instant now = Instant.now(); + this.index.index(new ArtifactDocument( + "maven", "repo1", "com/example/lib", "lib", + "1.0", 100L, now, "user1" + )).join(); + // Upsert same doc with different size + this.index.index(new ArtifactDocument( + "maven", "repo1", "com/example/lib", "lib", + "1.0", 999L, now, "user2" + )).join(); + final SearchResult result = this.index.search("com/example/lib", 10, 0).join(); + MatcherAssert.assertThat( + "Should have exactly 1 document after upsert", + result.documents().size(), + new IsEqual<>(1) + ); + MatcherAssert.assertThat( + "Size should be updated to 999", + result.documents().get(0).size(), + new IsEqual<>(999L) + ); + } + + @Test + void removeByRepoAndName() throws Exception { + this.index.index(new ArtifactDocument( + "maven", "repo1", "com/example/lib", "lib", + "1.0", 100L, Instant.now(), "user1" + )).join(); + this.index.remove("repo1", "com/example/lib").join(); + final List repos = this.index.locate("com/example/lib").join(); + MatcherAssert.assertThat( + "Locate should return empty after removal", + repos, + Matchers.empty() + ); + } + + @Test + void locateReturnsMultipleRepos() throws Exception { + this.index.index(new ArtifactDocument( + "maven", "repo-a", "shared/artifact", "artifact", + "1.0", 100L, Instant.now(), "user1" + )).join(); + this.index.index(new ArtifactDocument( + "maven", "repo-b", "shared/artifact", "artifact", + "1.0", 200L, Instant.now(), "user2" + )).join(); + final List repos = this.index.locate("shared/artifact").join(); + MatcherAssert.assertThat( + "Should find artifact in both repos", + repos, + Matchers.containsInAnyOrder("repo-a", "repo-b") + ); + } + + @Test + void searchWithPagination() throws Exception { + for (int idx = 0; idx < 10; idx++) { + this.index.index(new ArtifactDocument( + "maven", "repo1", "com/example/item-" + idx, "item-" + idx, + "1.0", idx * 10L, Instant.now(), "user1" + )).join(); + } + final SearchResult page1 = this.index.search("item", 3, 0).join(); + MatcherAssert.assertThat( + "First page should have 3 results", + page1.documents().size(), + new IsEqual<>(3) + ); + MatcherAssert.assertThat( + "Total hits should be 10", + page1.totalHits(), + new IsEqual<>(10L) + ); + final SearchResult page2 = this.index.search("item", 3, 3).join(); + MatcherAssert.assertThat( + "Second page should have 3 results", + page2.documents().size(), + new IsEqual<>(3) + ); + } + + @Test + void getStatsReturnsCount() throws Exception { + for (int idx = 0; idx < 5; idx++) { + this.index.index(new ArtifactDocument( + "maven", "repo1", "artifact-" + idx, "artifact-" + idx, + "1.0", 100L, Instant.now(), "user1" + )).join(); + } + final Map stats = this.index.getStats().join(); + MatcherAssert.assertThat( + "Document count should be 5", + stats.get("documents"), + new IsEqual<>(5L) + ); + MatcherAssert.assertThat( + "Should be warmed up", + stats.get("warmedUp"), + new IsEqual<>(true) + ); + MatcherAssert.assertThat( + "Type should be postgresql", + stats.get("type"), + new IsEqual<>("postgresql") + ); + } + + @Test + void isAlwaysWarmedUp() { + MatcherAssert.assertThat( + "DbArtifactIndex should always be warmed up", + this.index.isWarmedUp(), + new IsEqual<>(true) + ); + } + + @Test + void locateByPathPrefix() throws Exception { + // Insert rows with path_prefix directly (DbConsumer sets this, not DbArtifactIndex.index) + try (Connection conn = this.dataSource.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "INSERT INTO artifacts (repo_type, repo_name, name, version, size, created_date, owner, path_prefix) " + + "VALUES (?,?,?,?,?,?,?,?)" + )) { + stmt.setString(1, "maven"); + stmt.setString(2, "maven-central"); + stmt.setString(3, "com.google.guava:guava"); + stmt.setString(4, "31.1"); + stmt.setLong(5, 1000L); + stmt.setLong(6, System.currentTimeMillis()); + stmt.setString(7, "proxy"); + stmt.setString(8, "com/google/guava/guava/31.1"); + stmt.executeUpdate(); + } + // Locate with a full artifact path — should match via path_prefix + final List repos = this.index.locate( + "com/google/guava/guava/31.1/guava-31.1.jar" + ).join(); + MatcherAssert.assertThat( + "Should find repo via path_prefix match", + repos, + Matchers.contains("maven-central") + ); + // Locate with a path that doesn't match any prefix + final List empty = this.index.locate("org/apache/commons/commons-lang3/3.12/commons-lang3-3.12.jar").join(); + MatcherAssert.assertThat( + "Should return empty for non-matching path", + empty, + Matchers.empty() + ); + } + + @Test + void pathPrefixesDecomposition() { + MatcherAssert.assertThat( + "Multi-segment path", + DbArtifactIndex.pathPrefixes("com/google/guava/guava/31.1/guava-31.1.jar"), + Matchers.contains( + "com", "com/google", "com/google/guava", + "com/google/guava/guava", "com/google/guava/guava/31.1" + ) + ); + MatcherAssert.assertThat( + "Single-segment path", + DbArtifactIndex.pathPrefixes("artifact.jar"), + Matchers.contains("artifact.jar") + ); + MatcherAssert.assertThat( + "Leading slash stripped", + DbArtifactIndex.pathPrefixes("/com/example/lib"), + Matchers.contains("com", "com/example") + ); + } + + @Test + void locateByNameFindsRepos() throws Exception { + // Index artifacts with known names (as adapters store them) + this.index.index(new ArtifactDocument( + "maven", "maven-central", "com.google.guava.guava", "guava", + "31.1.3-jre", 2_000_000L, Instant.now(), "proxy" + )).join(); + this.index.index(new ArtifactDocument( + "maven", "maven-releases", "com.google.guava.guava", "guava", + "31.1.3-jre", 2_000_000L, Instant.now(), "admin" + )).join(); + this.index.index(new ArtifactDocument( + "maven", "maven-central", "org.slf4j.slf4j-api", "slf4j-api", + "2.0.9", 50_000L, Instant.now(), "proxy" + )).join(); + // locateByName should find both repos for guava + final List guavaRepos = this.index.locateByName("com.google.guava.guava").join(); + MatcherAssert.assertThat( + "Should find guava in both repos", + guavaRepos, + Matchers.containsInAnyOrder("maven-central", "maven-releases") + ); + // locateByName should find only maven-central for slf4j + final List slf4jRepos = this.index.locateByName("org.slf4j.slf4j-api").join(); + MatcherAssert.assertThat( + "Should find slf4j in maven-central only", + slf4jRepos, + Matchers.contains("maven-central") + ); + // locateByName with non-existent name + final List missing = this.index.locateByName("com.nonexistent.lib").join(); + MatcherAssert.assertThat( + "Should return empty for missing artifact", + missing, + Matchers.empty() + ); + } + + @Test + void locateByNameUsesExistingIndex() throws Exception { + // This test verifies the name-based locate works for all adapter name formats + // Maven: dotted notation + this.index.index(new ArtifactDocument( + "maven", "repo1", "com.google.guava.guava", "guava", + "31.1", 100L, Instant.now(), "user" + )).join(); + // npm: package name with scope + this.index.index(new ArtifactDocument( + "npm", "repo2", "@babel/core", "core", + "7.23.0", 200L, Instant.now(), "user" + )).join(); + // Docker: image name + this.index.index(new ArtifactDocument( + "docker", "repo3", "library/nginx", "nginx", + "sha256:abc", 300L, Instant.now(), "user" + )).join(); + // PyPI: normalized name + this.index.index(new ArtifactDocument( + "pypi", "repo4", "numpy", "numpy", + "1.24.0", 400L, Instant.now(), "user" + )).join(); + MatcherAssert.assertThat( + "Maven name lookup", + this.index.locateByName("com.google.guava.guava").join(), + Matchers.contains("repo1") + ); + MatcherAssert.assertThat( + "npm scoped name lookup", + this.index.locateByName("@babel/core").join(), + Matchers.contains("repo2") + ); + MatcherAssert.assertThat( + "Docker image name lookup", + this.index.locateByName("library/nginx").join(), + Matchers.contains("repo3") + ); + MatcherAssert.assertThat( + "PyPI name lookup", + this.index.locateByName("numpy").join(), + Matchers.contains("repo4") + ); + } + + @Test + void locateByNameHitRateWithMixedData() throws Exception { + // Simulate realistic data: index many artifacts, then verify + // that locateByName finds them all (100% hit rate for indexed data) + final String[] mavenNames = { + "com.google.guava.guava", + "org.apache.commons.commons-lang3", + "org.slf4j.slf4j-api", + "junit.junit", + "io.netty.netty-all", + "com.fasterxml.jackson.core.jackson-databind", + "org.projectlombok.lombok", + "org.springframework.spring-core", + "org.apache.maven.plugins.maven-compiler-plugin", + "org.apache.maven.plugins.maven-surefire-plugin", + }; + for (final String name : mavenNames) { + this.index.index(new ArtifactDocument( + "maven", "maven-central", name, name.substring(name.lastIndexOf('.') + 1), + "1.0.0", 100L, Instant.now(), "proxy" + )).join(); + } + int hits = 0; + for (final String name : mavenNames) { + final List repos = this.index.locateByName(name).join(); + if (!repos.isEmpty()) { + hits++; + } + } + MatcherAssert.assertThat( + String.format("locateByName hit rate: %d/%d", hits, mavenNames.length), + hits, + new IsEqual<>(mavenNames.length) + ); + } + + @Test + void indexBatchMultipleDocs() throws Exception { + final List docs = new ArrayList<>(); + for (int idx = 0; idx < 5; idx++) { + docs.add(new ArtifactDocument( + "npm", "npm-repo", "pkg-" + idx, "pkg-" + idx, + "2.0." + idx, 50L * idx, Instant.now(), "dev" + )); + } + this.index.indexBatch(docs).join(); + final SearchResult result = this.index.search("pkg", 10, 0).join(); + MatcherAssert.assertThat( + "All batch-indexed docs should be searchable", + result.documents().size(), + new IsEqual<>(5) + ); + MatcherAssert.assertThat( + "Total hits from batch should be 5", + result.totalHits(), + new IsEqual<>(5L) + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/proxy/OfflineAwareSliceTest.java b/artipie-main/src/test/java/com/artipie/proxy/OfflineAwareSliceTest.java new file mode 100644 index 000000000..e09ee8237 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/proxy/OfflineAwareSliceTest.java @@ -0,0 +1,66 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.proxy; + +import com.artipie.asto.Content; +import com.artipie.http.Headers; +import com.artipie.http.Response; +import com.artipie.http.ResponseBuilder; +import com.artipie.http.rq.RequestLine; +import com.artipie.http.rq.RqMethod; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for {@link OfflineAwareSlice}. + */ +class OfflineAwareSliceTest { + + @Test + void delegatesWhenOnline() throws Exception { + final OfflineAwareSlice slice = new OfflineAwareSlice( + (line, headers, body) -> CompletableFuture.completedFuture( + ResponseBuilder.ok().textBody("hello").build() + ) + ); + final Response resp = slice.response( + new RequestLine(RqMethod.GET, "/test"), Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(resp.status().code(), equalTo(200)); + } + + @Test + void returns503WhenOffline() throws Exception { + final OfflineAwareSlice slice = new OfflineAwareSlice( + (line, headers, body) -> CompletableFuture.completedFuture( + ResponseBuilder.ok().build() + ) + ); + slice.goOffline(); + final Response resp = slice.response( + new RequestLine(RqMethod.GET, "/test"), Headers.EMPTY, Content.EMPTY + ).get(); + assertThat(resp.status().code(), equalTo(503)); + } + + @Test + void togglesOfflineMode() { + final OfflineAwareSlice slice = new OfflineAwareSlice( + (line, headers, body) -> CompletableFuture.completedFuture( + ResponseBuilder.ok().build() + ) + ); + assertThat(slice.isOffline(), is(false)); + slice.goOffline(); + assertThat(slice.isOffline(), is(true)); + slice.goOnline(); + assertThat(slice.isOffline(), is(false)); + } +} diff --git a/artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceJdbcTest.java b/artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceJdbcTest.java new file mode 100644 index 000000000..ae8afc9c9 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceJdbcTest.java @@ -0,0 +1,182 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.scheduling; + +import com.amihaiemil.eoyaml.Yaml; +import com.artipie.db.ArtifactDbFactory; +import com.artipie.db.PostgreSQLTestConfig; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Queue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import javax.sql.DataSource; +import org.awaitility.Awaitility; +import org.cactoos.list.ListOf; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Tests for {@link QuartzService} in JDBC clustering mode. + * Uses Testcontainers PostgreSQL. + * + * @since 1.20.13 + */ +@Testcontainers +@SuppressWarnings("PMD.AvoidDuplicateLiterals") +final class QuartzServiceJdbcTest { + + /** + * PostgreSQL test container. + */ + @Container + static final PostgreSQLContainer POSTGRES = PostgreSQLTestConfig.createContainer(); + + /** + * Shared DataSource. + */ + private DataSource source; + + /** + * Service under test. + */ + private QuartzService service; + + @BeforeEach + void setUp() { + this.source = new ArtifactDbFactory( + Yaml.createYamlMappingBuilder().add( + "artifacts_database", + Yaml.createYamlMappingBuilder() + .add(ArtifactDbFactory.YAML_HOST, POSTGRES.getHost()) + .add( + ArtifactDbFactory.YAML_PORT, + String.valueOf(POSTGRES.getFirstMappedPort()) + ) + .add(ArtifactDbFactory.YAML_DATABASE, POSTGRES.getDatabaseName()) + .add(ArtifactDbFactory.YAML_USER, POSTGRES.getUsername()) + .add(ArtifactDbFactory.YAML_PASSWORD, POSTGRES.getPassword()) + .build() + ).build(), + "artifacts" + ).initialize(); + this.service = new QuartzService(this.source); + } + + @AfterEach + void tearDown() { + if (this.service != null) { + this.service.stop(); + } + } + + @Test + void createsQuartzSchemaTablesOnStartup() throws Exception { + try (Connection conn = this.source.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rset = stmt.executeQuery( + "SELECT tablename FROM pg_tables WHERE tablename LIKE 'qrtz_%' ORDER BY tablename" + )) { + final java.util.List tables = new java.util.ArrayList<>(); + while (rset.next()) { + tables.add(rset.getString(1)); + } + MatcherAssert.assertThat( + "QRTZ tables should be created", + tables, + Matchers.hasItems( + "qrtz_job_details", + "qrtz_triggers", + "qrtz_simple_triggers", + "qrtz_cron_triggers", + "qrtz_fired_triggers", + "qrtz_locks", + "qrtz_scheduler_state", + "qrtz_calendars", + "qrtz_paused_trigger_grps" + ) + ); + } + } + + @Test + void isClusteredModeEnabled() { + MatcherAssert.assertThat( + "JDBC constructor should enable clustered mode", + this.service.isClustered(), + Matchers.is(true) + ); + } + + @Test + void ramModeIsNotClustered() { + final QuartzService ram = new QuartzService(); + try { + MatcherAssert.assertThat( + "RAM constructor should not enable clustered mode", + ram.isClustered(), + Matchers.is(false) + ); + } finally { + ram.stop(); + } + } + + @Test + void schedulesAndExecutesPeriodicJob() throws Exception { + final AtomicInteger count = new AtomicInteger(); + final Queue queue = this.service.addPeriodicEventsProcessor( + 1, + new ListOf>(item -> count.incrementAndGet()) + ); + this.service.start(); + queue.add("one"); + queue.add("two"); + queue.add("three"); + Awaitility.await().atMost(15, TimeUnit.SECONDS) + .until(() -> count.get() >= 3); + MatcherAssert.assertThat( + "All 3 items should be processed by JDBC-backed scheduler", + count.get(), + Matchers.greaterThanOrEqualTo(3) + ); + } + + @Test + void registersSchedulerStateInDatabase() throws Exception { + this.service.start(); + // Allow scheduler to register with the DB + Thread.sleep(2000); + try (Connection conn = this.source.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rset = stmt.executeQuery( + "SELECT COUNT(*) FROM QRTZ_SCHEDULER_STATE" + )) { + rset.next(); + MatcherAssert.assertThat( + "Scheduler should register its state in the database", + rset.getInt(1), + Matchers.greaterThanOrEqualTo(1) + ); + } + } + + @Test + void doubleStopDoesNotThrowInJdbcMode() { + this.service.start(); + this.service.stop(); + this.service.stop(); + // If we get here without exception, the test passes + this.service = null; + } +} diff --git a/artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceTest.java b/artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceTest.java index ec56ff644..231d66b90 100644 --- a/artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceTest.java +++ b/artipie-main/src/test/java/com/artipie/scheduling/QuartzServiceTest.java @@ -11,6 +11,7 @@ import org.awaitility.Awaitility; import org.cactoos.list.ListOf; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.quartz.Job; @@ -69,6 +70,19 @@ void runsGivenJobs() throws SchedulerException { Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> count.get() > 12); } + @Test + void doubleStopDoesNotThrow() { + final QuartzService svc = new QuartzService(); + svc.start(); + Assertions.assertDoesNotThrow( + () -> { + svc.stop(); + svc.stop(); + }, + "Calling stop() twice must not throw an exception" + ); + } + /** * Test consumer. * @since 1.3 diff --git a/artipie-main/src/test/java/com/artipie/scheduling/TempFileCleanupJobTest.java b/artipie-main/src/test/java/com/artipie/scheduling/TempFileCleanupJobTest.java new file mode 100644 index 000000000..44abd7eb5 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/scheduling/TempFileCleanupJobTest.java @@ -0,0 +1,225 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.scheduling; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link TempFileCleanupJob}. + * + * @since 1.20.13 + */ +public final class TempFileCleanupJobTest { + + @Test + void deletesOldTmpFiles(@TempDir final Path dir) throws Exception { + final Path old = dir.resolve("artipie-stc-abc123.tmp"); + Files.writeString(old, "old data"); + setOldTimestamp(old, 120); + final Path recent = dir.resolve("artipie-stc-def456.tmp"); + Files.writeString(recent, "recent data"); + TempFileCleanupJob.cleanup(dir, 60L); + Assertions.assertFalse( + Files.exists(old), + "Old .tmp file should have been deleted" + ); + Assertions.assertTrue( + Files.exists(recent), + "Recent .tmp file should be kept" + ); + } + + @Test + void deletesOldArtipieCacheFiles(@TempDir final Path dir) throws Exception { + final Path old = dir.resolve("artipie-cache-data"); + Files.writeString(old, "cache data"); + setOldTimestamp(old, 90); + final Path recent = dir.resolve("artipie-cache-fresh"); + Files.writeString(recent, "fresh cache"); + TempFileCleanupJob.cleanup(dir, 60L); + Assertions.assertFalse( + Files.exists(old), + "Old artipie-cache- file should have been deleted" + ); + Assertions.assertTrue( + Files.exists(recent), + "Recent artipie-cache- file should be kept" + ); + } + + @Test + void deletesOldPartFiles(@TempDir final Path dir) throws Exception { + final Path old = dir.resolve("data.part-abc123"); + Files.writeString(old, "partial data"); + setOldTimestamp(old, 90); + final Path recent = dir.resolve("data.part-def456"); + Files.writeString(recent, "recent partial"); + TempFileCleanupJob.cleanup(dir, 60L); + Assertions.assertFalse( + Files.exists(old), + "Old .part- file should have been deleted" + ); + Assertions.assertTrue( + Files.exists(recent), + "Recent .part- file should be kept" + ); + } + + @Test + void deletesFilesInDotTmpSubdir(@TempDir final Path dir) throws Exception { + final Path tmpDir = dir.resolve(".tmp"); + Files.createDirectories(tmpDir); + final Path old = tmpDir.resolve("some-uuid-file"); + Files.writeString(old, "uuid data"); + setOldTimestamp(old, 120); + final Path recent = tmpDir.resolve("another-uuid-file"); + Files.writeString(recent, "recent uuid data"); + TempFileCleanupJob.cleanup(dir, 60L); + Assertions.assertFalse( + Files.exists(old), + "Old file in .tmp/ directory should have been deleted" + ); + Assertions.assertTrue( + Files.exists(recent), + "Recent file in .tmp/ directory should be kept" + ); + } + + @Test + void keepsNonTempFiles(@TempDir final Path dir) throws Exception { + final Path normal = dir.resolve("important-data.jar"); + Files.writeString(normal, "keep me"); + setOldTimestamp(normal, 120); + final Path readme = dir.resolve("README.md"); + Files.writeString(readme, "keep me too"); + setOldTimestamp(readme, 120); + TempFileCleanupJob.cleanup(dir, 60L); + Assertions.assertTrue( + Files.exists(normal), + "Non-temp .jar file should not be deleted" + ); + Assertions.assertTrue( + Files.exists(readme), + "Non-temp .md file should not be deleted" + ); + } + + @Test + void recursesIntoSubdirectories(@TempDir final Path dir) throws Exception { + final Path sub = dir.resolve("subdir"); + Files.createDirectories(sub); + final Path deep = sub.resolve("deep.tmp"); + Files.writeString(deep, "deep data"); + setOldTimestamp(deep, 120); + TempFileCleanupJob.cleanup(dir, 60L); + Assertions.assertFalse( + Files.exists(deep), + "Old .tmp file in subdirectory should have been deleted" + ); + } + + @Test + void handlesNonExistentDirectory() { + final Path missing = Path.of("/nonexistent/dir/that/does/not/exist"); + Assertions.assertDoesNotThrow( + () -> TempFileCleanupJob.cleanup(missing, 60L), + "Job should handle non-existent directory gracefully" + ); + } + + @Test + void handlesNullDirectory() { + Assertions.assertDoesNotThrow( + () -> TempFileCleanupJob.cleanup(null, 60L), + "Job should handle null directory gracefully" + ); + } + + @Test + void usesDefaultMaxAge(@TempDir final Path dir) throws Exception { + final Path old = dir.resolve("artipie-stc-test.tmp"); + Files.writeString(old, "data"); + setOldTimestamp(old, 120); + final Path recent = dir.resolve("artipie-stc-new.tmp"); + Files.writeString(recent, "new data"); + TempFileCleanupJob.cleanup(dir, TempFileCleanupJob.DEFAULT_MAX_AGE_MINUTES); + Assertions.assertFalse( + Files.exists(old), + "File older than default 60 min should be deleted" + ); + Assertions.assertTrue( + Files.exists(recent), + "Recent file should be kept with default max age" + ); + } + + @Test + void isTempFileMatchesCorrectPatterns() { + Assertions.assertTrue( + TempFileCleanupJob.isTempFile(Path.of("/tmp/artipie-stc-abc.tmp")), + "Should match artipie-stc-*.tmp" + ); + Assertions.assertTrue( + TempFileCleanupJob.isTempFile(Path.of("/tmp/something.tmp")), + "Should match *.tmp" + ); + Assertions.assertTrue( + TempFileCleanupJob.isTempFile(Path.of("/tmp/artipie-cache-data")), + "Should match artipie-cache-*" + ); + Assertions.assertTrue( + TempFileCleanupJob.isTempFile(Path.of("/cache/.tmp/uuid-file")), + "Should match files in .tmp/ directory" + ); + Assertions.assertTrue( + TempFileCleanupJob.isTempFile(Path.of("/tmp/data.part-abc123")), + "Should match .part- files" + ); + Assertions.assertFalse( + TempFileCleanupJob.isTempFile(Path.of("/tmp/important.jar")), + "Should not match .jar files" + ); + Assertions.assertFalse( + TempFileCleanupJob.isTempFile(Path.of("/data/artifact.pom")), + "Should not match .pom files" + ); + } + + @Test + void deletesOldArtipieStcFilesWithoutTmpExtension(@TempDir final Path dir) + throws Exception { + final Path old = dir.resolve("artipie-stc-nosuffix"); + Files.writeString(old, "stc data"); + setOldTimestamp(old, 120); + TempFileCleanupJob.cleanup(dir, 60L); + Assertions.assertFalse( + Files.exists(old), + "Old artipie-stc- file without .tmp extension should have been deleted" + ); + } + + /** + * Sets the last modified time of a file to the given number of minutes in the past. + * + * @param file File to modify + * @param minutesAgo How many minutes in the past + * @throws IOException On error + */ + private static void setOldTimestamp(final Path file, final int minutesAgo) + throws IOException { + Files.setLastModifiedTime( + file, + FileTime.from(Instant.now().minus(minutesAgo, ChronoUnit.MINUTES)) + ); + } +} diff --git a/artipie-main/src/test/java/com/artipie/security/policy/CachedDbPolicyTest.java b/artipie-main/src/test/java/com/artipie/security/policy/CachedDbPolicyTest.java new file mode 100644 index 000000000..f68bf7dfc --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/security/policy/CachedDbPolicyTest.java @@ -0,0 +1,294 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.security.policy; + +import com.artipie.api.perms.ApiRepositoryPermission; +import com.artipie.api.perms.ApiSearchPermission; +import com.artipie.db.DbManager; +import com.artipie.db.PostgreSQLTestConfig; +import com.artipie.http.auth.AuthUser; +import com.artipie.security.perms.AdapterBasicPermission; +import com.artipie.security.perms.UserPermissions; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import javax.json.Json; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link CachedDbPolicy}. + * @since 1.21 + */ +@Testcontainers +class CachedDbPolicyTest { + + @Container + static final PostgreSQLContainer PG = PostgreSQLTestConfig.createContainer(); + + static HikariDataSource ds; + CachedDbPolicy policy; + + @BeforeAll + static void setup() { + final HikariConfig cfg = new HikariConfig(); + cfg.setJdbcUrl(PG.getJdbcUrl()); + cfg.setUsername(PG.getUsername()); + cfg.setPassword(PG.getPassword()); + cfg.setMaximumPoolSize(3); + ds = new HikariDataSource(cfg); + DbManager.migrate(ds); + } + + @AfterAll + static void teardown() { + if (ds != null) { + ds.close(); + } + } + + @BeforeEach + void init() throws Exception { + this.policy = new CachedDbPolicy(ds); + try (Connection conn = ds.getConnection()) { + conn.createStatement().execute("DELETE FROM user_roles"); + conn.createStatement().execute("DELETE FROM users"); + conn.createStatement().execute("DELETE FROM roles"); + } + } + + @Test + void grantsSearchPermissionFromRole() throws Exception { + createRole("reader", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("api_search_permissions", Json.createArrayBuilder().add("read")) + .add("api_repository_permissions", Json.createArrayBuilder().add("read")) + ).build().toString() + ); + createUser("alice", true); + assignRole("alice", "reader"); + final UserPermissions perms = this.policy.getPermissions( + new AuthUser("alice", "artipie") + ); + assertTrue( + perms.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)), + "User with reader role should have search read permission" + ); + assertTrue( + perms.implies(new ApiRepositoryPermission( + ApiRepositoryPermission.RepositoryAction.READ + )), + "User with reader role should have repository read permission" + ); + } + + @Test + void deniesUnassignedPermission() throws Exception { + createRole("reader", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("api_search_permissions", Json.createArrayBuilder().add("read")) + ).build().toString() + ); + createUser("bob", true); + assignRole("bob", "reader"); + final UserPermissions perms = this.policy.getPermissions( + new AuthUser("bob", "artipie") + ); + assertFalse( + perms.implies(new ApiRepositoryPermission( + ApiRepositoryPermission.RepositoryAction.DELETE + )), + "User should not have repository delete permission" + ); + } + + @Test + void deniesDisabledUser() throws Exception { + createRole("admin", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("api_search_permissions", Json.createArrayBuilder().add("read")) + ).build().toString() + ); + createUser("disabled_user", false); + assignRole("disabled_user", "admin"); + final UserPermissions perms = this.policy.getPermissions( + new AuthUser("disabled_user", "artipie") + ); + assertFalse( + perms.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)), + "Disabled user should not have any permissions" + ); + } + + @Test + void deniesDisabledRole() throws Exception { + createRole("suspended", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("api_search_permissions", Json.createArrayBuilder().add("read")) + ).build().toString() + ); + disableRole("suspended"); + createUser("charlie", true); + assignRole("charlie", "suspended"); + final UserPermissions perms = this.policy.getPermissions( + new AuthUser("charlie", "artipie") + ); + assertFalse( + perms.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)), + "User with disabled role should not have permissions from that role" + ); + } + + @Test + void grantsAdapterPermissionFromRole() throws Exception { + createRole("go_reader", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("adapter_basic_permissions", Json.createObjectBuilder() + .add("go", Json.createArrayBuilder().add("read"))) + ).build().toString() + ); + createUser("dave", true); + assignRole("dave", "go_reader"); + final UserPermissions perms = this.policy.getPermissions( + new AuthUser("dave", "artipie") + ); + assertTrue( + perms.implies(new AdapterBasicPermission("go", "read")), + "User should have read permission on go repo" + ); + assertFalse( + perms.implies(new AdapterBasicPermission("go", "write")), + "User should not have write permission on go repo" + ); + } + + @Test + void invalidationClearsCache() throws Exception { + createRole("mutable", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("api_search_permissions", Json.createArrayBuilder().add("read")) + ).build().toString() + ); + createUser("eve", true); + assignRole("eve", "mutable"); + // Load into cache + assertTrue( + this.policy.getPermissions(new AuthUser("eve", "artipie")) + .implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)) + ); + // Remove role permissions in DB + try (Connection conn = ds.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE roles SET permissions = '{}'::jsonb WHERE name = ?" + )) { + ps.setString(1, "mutable"); + ps.executeUpdate(); + } + // Invalidate + this.policy.invalidate("mutable"); + this.policy.invalidate("eve"); + // Should reflect new state + assertFalse( + this.policy.getPermissions(new AuthUser("eve", "artipie")) + .implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)), + "After invalidation, removed permissions should no longer be granted" + ); + } + + @Test + void handlesUserNotInDb() { + final UserPermissions perms = this.policy.getPermissions( + new AuthUser("unknown_user", "artipie") + ); + assertFalse( + perms.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)), + "Unknown user should have no permissions" + ); + } + + @Test + void handlesMultipleRoles() throws Exception { + createRole("searcher", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("api_search_permissions", Json.createArrayBuilder().add("read")) + ).build().toString() + ); + createRole("go_dev", Json.createObjectBuilder() + .add("permissions", Json.createObjectBuilder() + .add("adapter_basic_permissions", Json.createObjectBuilder() + .add("go-repo", Json.createArrayBuilder().add("read").add("write"))) + ).build().toString() + ); + createUser("frank", true); + assignRole("frank", "searcher"); + assignRole("frank", "go_dev"); + final UserPermissions perms = this.policy.getPermissions( + new AuthUser("frank", "artipie") + ); + assertTrue( + perms.implies(new ApiSearchPermission(ApiSearchPermission.SearchAction.READ)), + "User should have search permission from searcher role" + ); + assertTrue( + perms.implies(new AdapterBasicPermission("go-repo", "write")), + "User should have write permission from go_dev role" + ); + } + + private void createRole(final String name, final String permsJson) throws Exception { + try (Connection conn = ds.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO roles (name, permissions) VALUES (?, ?::jsonb)" + )) { + ps.setString(1, name); + ps.setString(2, permsJson); + ps.executeUpdate(); + } + } + + private void disableRole(final String name) throws Exception { + try (Connection conn = ds.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE roles SET enabled = false WHERE name = ?" + )) { + ps.setString(1, name); + ps.executeUpdate(); + } + } + + private void createUser(final String name, final boolean enabled) throws Exception { + try (Connection conn = ds.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO users (username, password_hash, enabled) VALUES (?, 'test', ?)" + )) { + ps.setString(1, name); + ps.setBoolean(2, enabled); + ps.executeUpdate(); + } + } + + private void assignRole(final String username, final String roleName) throws Exception { + try (Connection conn = ds.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO user_roles (user_id, role_id) " + + "SELECT u.id, r.id FROM users u, roles r " + + "WHERE u.username = ? AND r.name = ?" + )) { + ps.setString(1, username); + ps.setString(2, roleName); + ps.executeUpdate(); + } + } +} diff --git a/artipie-main/src/test/java/com/artipie/settings/YamlSettingsTest.java b/artipie-main/src/test/java/com/artipie/settings/YamlSettingsTest.java index d08f2476c..67bfe518b 100644 --- a/artipie-main/src/test/java/com/artipie/settings/YamlSettingsTest.java +++ b/artipie-main/src/test/java/com/artipie/settings/YamlSettingsTest.java @@ -339,6 +339,29 @@ private String artipieGithubKeycloakEnvCredsAndPolicy() { ); } + @Test + void closeIsIdempotent() throws Exception { + final YamlSettings settings = new YamlSettings( + this.config("some/path"), this.temp, new QuartzService() + ); + settings.close(); + Assertions.assertDoesNotThrow( + settings::close, + "Calling close() a second time should not throw" + ); + } + + @Test + void closeWithNoDatabaseOrValkey() throws Exception { + final YamlSettings settings = new YamlSettings( + this.config("some/path"), this.temp, new QuartzService() + ); + Assertions.assertDoesNotThrow( + settings::close, + "close() should complete without error when no database or Valkey is configured" + ); + } + @SuppressWarnings("PMD.UnusedPrivateMethod") private static Stream badYamls() { return Stream.of( diff --git a/artipie-main/src/test/java/com/artipie/settings/cache/PublishingFiltersCacheTest.java b/artipie-main/src/test/java/com/artipie/settings/cache/PublishingFiltersCacheTest.java new file mode 100644 index 000000000..e6f8f60a3 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/settings/cache/PublishingFiltersCacheTest.java @@ -0,0 +1,222 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.settings.cache; + +import com.amihaiemil.eoyaml.YamlMapping; +import com.artipie.cache.CacheInvalidationPubSub; +import com.artipie.cache.ValkeyConnection; +import com.artipie.http.filter.Filters; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.awaitility.Awaitility; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Tests for {@link PublishingFiltersCache}. + * + * @since 1.20.13 + */ +@Testcontainers +@SuppressWarnings("PMD.AvoidDuplicateLiterals") +final class PublishingFiltersCacheTest { + + /** + * Valkey container. + */ + @Container + @SuppressWarnings("rawtypes") + private static final GenericContainer VALKEY = + new GenericContainer<>("redis:7-alpine") + .withExposedPorts(6379); + + /** + * Instance A connection (publisher side). + */ + private ValkeyConnection connA; + + /** + * Instance B connection (subscriber side). + */ + private ValkeyConnection connB; + + /** + * Pub/sub for instance A. + */ + private CacheInvalidationPubSub pubsubA; + + /** + * Pub/sub for instance B. + */ + private CacheInvalidationPubSub pubsubB; + + @BeforeEach + void setUp() { + final String host = VALKEY.getHost(); + final int port = VALKEY.getMappedPort(6379); + this.connA = new ValkeyConnection(host, port, Duration.ofSeconds(5)); + this.connB = new ValkeyConnection(host, port, Duration.ofSeconds(5)); + this.pubsubA = new CacheInvalidationPubSub(this.connA); + this.pubsubB = new CacheInvalidationPubSub(this.connB); + } + + @AfterEach + void tearDown() { + if (this.pubsubA != null) { + this.pubsubA.close(); + } + if (this.pubsubB != null) { + this.pubsubB.close(); + } + if (this.connA != null) { + this.connA.close(); + } + if (this.connB != null) { + this.connB.close(); + } + } + + @Test + void delegatesFiltersToInnerCache() { + final RecordingFiltersCache inner = new RecordingFiltersCache(); + final PublishingFiltersCache cache = + new PublishingFiltersCache(inner, this.pubsubA); + cache.filters("my-repo", null); + MatcherAssert.assertThat( + "Should delegate filters() to inner cache", + inner.queriedRepos(), + Matchers.contains("my-repo") + ); + } + + @Test + void delegatesSizeToInnerCache() { + final RecordingFiltersCache inner = new RecordingFiltersCache(); + final PublishingFiltersCache cache = + new PublishingFiltersCache(inner, this.pubsubA); + MatcherAssert.assertThat( + "Should delegate size() to inner cache", + cache.size(), + Matchers.is(42L) + ); + } + + @Test + void invalidateDelegatesAndPublishes() { + final RecordingFiltersCache innerA = new RecordingFiltersCache(); + final RecordingFiltersCache innerB = new RecordingFiltersCache(); + this.pubsubB.register("filters", innerB); + final PublishingFiltersCache cache = + new PublishingFiltersCache(innerA, this.pubsubA); + cache.invalidate("docker-repo"); + MatcherAssert.assertThat( + "Should invalidate inner cache directly", + innerA.invalidatedKeys(), + Matchers.contains("docker-repo") + ); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> MatcherAssert.assertThat( + "Remote instance should receive invalidation", + innerB.invalidatedKeys(), + Matchers.contains("docker-repo") + ) + ); + } + + @Test + void invalidateAllDelegatesAndPublishes() { + final RecordingFiltersCache innerA = new RecordingFiltersCache(); + final RecordingFiltersCache innerB = new RecordingFiltersCache(); + this.pubsubB.register("filters", innerB); + final PublishingFiltersCache cache = + new PublishingFiltersCache(innerA, this.pubsubA); + cache.invalidateAll(); + MatcherAssert.assertThat( + "Should invalidateAll on inner cache directly", + innerA.allInvalidations(), + Matchers.is(1) + ); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> MatcherAssert.assertThat( + "Remote instance should receive invalidateAll", + innerB.allInvalidations(), + Matchers.is(1) + ) + ); + } + + /** + * Recording implementation of {@link FiltersCache} for test verification. + */ + private static final class RecordingFiltersCache implements FiltersCache { + /** + * Repo names queried via filters(). + */ + private final List repos; + + /** + * Keys invalidated. + */ + private final List keys; + + /** + * Count of invalidateAll calls. + */ + private int allCount; + + RecordingFiltersCache() { + this.repos = Collections.synchronizedList(new ArrayList<>(4)); + this.keys = Collections.synchronizedList(new ArrayList<>(4)); + } + + @Override + public Optional filters(final String reponame, + final YamlMapping repoyaml) { + this.repos.add(reponame); + return Optional.empty(); + } + + @Override + public long size() { + return 42L; + } + + @Override + public void invalidate(final String reponame) { + this.keys.add(reponame); + } + + @Override + public void invalidateAll() { + this.allCount += 1; + } + + List queriedRepos() { + return this.repos; + } + + List invalidatedKeys() { + return this.keys; + } + + int allInvalidations() { + return this.allCount; + } + } +} diff --git a/artipie-main/src/test/java/com/artipie/webhook/WebhookConfigTest.java b/artipie-main/src/test/java/com/artipie/webhook/WebhookConfigTest.java new file mode 100644 index 000000000..be65e3d7b --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/webhook/WebhookConfigTest.java @@ -0,0 +1,72 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.webhook; + +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for {@link WebhookConfig}. + */ +class WebhookConfigTest { + + @Test + void matchesAllEventsWhenEmpty() { + final WebhookConfig cfg = new WebhookConfig( + "https://example.com/hook", null, List.of(), null + ); + assertThat(cfg.matchesEvent("artifact.published"), is(true)); + assertThat(cfg.matchesEvent("artifact.deleted"), is(true)); + } + + @Test + void matchesSpecificEvent() { + final WebhookConfig cfg = new WebhookConfig( + "https://example.com/hook", null, List.of("artifact.published"), null + ); + assertThat(cfg.matchesEvent("artifact.published"), is(true)); + assertThat(cfg.matchesEvent("artifact.deleted"), is(false)); + } + + @Test + void matchesAllReposWhenEmpty() { + final WebhookConfig cfg = new WebhookConfig( + "https://example.com/hook", null, null, List.of() + ); + assertThat(cfg.matchesRepo("central"), is(true)); + assertThat(cfg.matchesRepo("any-repo"), is(true)); + } + + @Test + void matchesSpecificRepo() { + final WebhookConfig cfg = new WebhookConfig( + "https://example.com/hook", null, null, List.of("central") + ); + assertThat(cfg.matchesRepo("central"), is(true)); + assertThat(cfg.matchesRepo("snapshots"), is(false)); + } + + @Test + void returnsSigningSecret() { + final WebhookConfig cfg = new WebhookConfig( + "https://example.com/hook", "my-secret", null, null + ); + assertThat(cfg.signingSecret(), equalTo(Optional.of("my-secret"))); + } + + @Test + void returnsEmptySecretWhenNull() { + final WebhookConfig cfg = new WebhookConfig( + "https://example.com/hook", null, null, null + ); + assertThat(cfg.signingSecret(), equalTo(Optional.empty())); + } +} diff --git a/artipie-main/src/test/java/com/artipie/webhook/WebhookDispatcherTest.java b/artipie-main/src/test/java/com/artipie/webhook/WebhookDispatcherTest.java new file mode 100644 index 000000000..c15767394 --- /dev/null +++ b/artipie-main/src/test/java/com/artipie/webhook/WebhookDispatcherTest.java @@ -0,0 +1,45 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2023 artipie.com + * https://github.com/artipie/artipie/blob/master/LICENSE.txt + */ +package com.artipie.webhook; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.emptyString; + +/** + * Tests for {@link WebhookDispatcher} HMAC computation. + */ +class WebhookDispatcherTest { + + @Test + void computesHmacSha256() { + final String hmac = WebhookDispatcher.computeHmac( + "{\"event\":\"test\"}", "secret-key" + ); + assertThat(hmac, not(emptyString())); + // HMAC should be deterministic + assertThat( + WebhookDispatcher.computeHmac("{\"event\":\"test\"}", "secret-key"), + equalTo(hmac) + ); + } + + @Test + void differentPayloadsProduceDifferentHmac() { + final String hmac1 = WebhookDispatcher.computeHmac("payload1", "key"); + final String hmac2 = WebhookDispatcher.computeHmac("payload2", "key"); + assertThat(hmac1, not(equalTo(hmac2))); + } + + @Test + void differentSecretsProduceDifferentHmac() { + final String hmac1 = WebhookDispatcher.computeHmac("payload", "key1"); + final String hmac2 = WebhookDispatcher.computeHmac("payload", "key2"); + assertThat(hmac1, not(equalTo(hmac2))); + } +} diff --git a/artipie-ui/.gitignore b/artipie-ui/.gitignore new file mode 100644 index 000000000..39e41485e --- /dev/null +++ b/artipie-ui/.gitignore @@ -0,0 +1,4 @@ +node_modules +dist +*.local +.env*.local diff --git a/artipie-ui/Dockerfile b/artipie-ui/Dockerfile new file mode 100644 index 000000000..9e1f88cef --- /dev/null +++ b/artipie-ui/Dockerfile @@ -0,0 +1,17 @@ +# Build stage +FROM node:22-alpine AS build +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY . . +RUN npm run build + +# Runtime stage +FROM nginx:alpine +COPY --from=build /app/dist /usr/share/nginx/html +COPY nginx/default.conf /etc/nginx/conf.d/default.conf +COPY docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh +EXPOSE 80 +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/artipie-ui/README.md b/artipie-ui/README.md new file mode 100644 index 000000000..0e2219e44 --- /dev/null +++ b/artipie-ui/README.md @@ -0,0 +1,145 @@ +# Artipie UI + +Web management console for [Artipie](https://github.com/artipie/artipie) — a binary artifact repository manager. + +Built with Vue 3, TypeScript, Vite, PrimeVue 4, and Tailwind CSS 4. + +## Prerequisites + +- Node.js 22+ +- npm 10+ +- A running Artipie instance with the `/api/v1/*` endpoints (AsyncApiVerticle on port 8086) + +## Quick Start (Development) + +```bash +# Install dependencies +npm install + +# Start dev server (http://localhost:3000) +npm run dev +``` + +The dev server proxies `/api` requests to `http://localhost:8086` (configurable in `vite.config.ts`). + +## Available Scripts + +| Command | Description | +|---------|-------------| +| `npm run dev` | Start Vite dev server with hot reload | +| `npm run build` | Type-check and build for production | +| `npm run preview` | Preview the production build locally | +| `npm test` | Run unit tests (Vitest) | +| `npm run test:watch` | Run tests in watch mode | +| `npm run type-check` | Run TypeScript type checking | + +## Production Build + +```bash +npm run build +``` + +Output is written to `dist/`. The build runs `vue-tsc --noEmit` for type checking before `vite build`. + +## Docker + +### Standalone + +```bash +# Build the image +docker build -t artipie-ui . + +# Run (assumes Artipie is reachable at http://artipie:8086 from within Docker network) +docker run -d -p 8090:80 --name artipie-ui artipie-ui +``` + +### With Docker Compose + +The UI is included in the main Artipie `docker-compose.yaml`. It builds from this directory and is available at **http://localhost:8090**. + +```bash +cd ../artipie-main/docker-compose +docker compose up -d +``` + +The compose nginx service also exposes the UI at **http://localhost:8081/ui/**. + +## Runtime Configuration + +The UI reads `config.json` at startup (before Vue mounts). In Docker, configuration is driven by **environment variables** — no rebuild or volume mount required. + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `API_BASE_URL` | Base URL for API requests (relative or absolute) | `/api/v1` | +| `GRAFANA_URL` | Link to Grafana dashboard (shown on Dashboard page) | _(empty)_ | +| `APP_TITLE` | Application title in header and login page | `Artipie` | +| `DEFAULT_PAGE_SIZE` | Default page size for paginated lists | `20` | + +```bash +docker run -d -p 8090:80 \ + -e API_BASE_URL=https://registry.example.com/api/v1 \ + -e GRAFANA_URL=https://grafana.example.com \ + -e APP_TITLE="My Registry" \ + -e DEFAULT_PAGE_SIZE=50 \ + artipie-ui +``` + +The Docker entrypoint generates `config.json` from these variables at container startup using `envsubst`. When no variables are set, the defaults above apply. + +### Docker Compose + +In `docker-compose.yaml`, pass variables with the `UI_` prefix to avoid collisions: + +```yaml +artipie-ui: + environment: + - API_BASE_URL=${UI_API_BASE_URL:-/api/v1} + - GRAFANA_URL=${UI_GRAFANA_URL:-http://localhost:3000} + - APP_TITLE=${UI_APP_TITLE:-Artipie} + - DEFAULT_PAGE_SIZE=${UI_DEFAULT_PAGE_SIZE:-20} +``` + +### Local Development + +For local development without Docker, edit `public/config.json` directly. This file is loaded by `fetch('/config.json')` in `main.ts` and is not processed by Vite's build pipeline. + +## Architecture + +``` +src/ + api/ # Axios HTTP client and per-domain API modules + assets/ # Global CSS (Tailwind + PrimeIcons) + components/ # Reusable components (layout shell, health indicator) + composables/ # Vue composables (pagination, search, permissions) + router/ # Vue Router with auth guards + stores/ # Pinia stores (auth, config, theme, notifications) + types/ # TypeScript interfaces matching API responses + views/ # Page components organized by feature + admin/ # Admin-only pages (repo mgmt, users, roles, settings) + auth/ # Login, OAuth callback + dashboard/ # Dashboard with stats + profile/ # User profile + repos/ # Repository list and detail + search/ # Global artifact search +``` + +## Nginx Proxy + +The built-in nginx config (`nginx/default.conf`) handles: + +- **`/api/`** -- Proxied to `http://artipie:8086/api/` (Artipie REST API) +- **`/assets/`** -- Immutable hashed assets with 1-year cache +- **`/`** -- SPA fallback (`try_files` to `index.html`) + +## Tech Stack + +- **Vue 3.5** -- Composition API with ` + + diff --git a/artipie-ui/nginx/default.conf b/artipie-ui/nginx/default.conf new file mode 100644 index 000000000..d2fee3392 --- /dev/null +++ b/artipie-ui/nginx/default.conf @@ -0,0 +1,33 @@ +server { + listen 80; + server_name _; + root /usr/share/nginx/html; + index index.html; + + # Gzip + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml; + gzip_min_length 256; + + # API proxy to Artipie AsyncApiVerticle + location /api/ { + proxy_pass http://artipie:8086/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 10s; + } + + # Hashed assets — long cache + location /assets/ { + expires 1y; + add_header Cache-Control "public, immutable"; + } + + # SPA fallback — no cache for index.html + location / { + try_files $uri $uri/ /index.html; + add_header Cache-Control "no-cache"; + } +} diff --git a/artipie-ui/package-lock.json b/artipie-ui/package-lock.json new file mode 100644 index 000000000..010d94d88 --- /dev/null +++ b/artipie-ui/package-lock.json @@ -0,0 +1,4758 @@ +{ + "name": "artipie-ui", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "artipie-ui", + "version": "0.1.0", + "dependencies": { + "@primevue/themes": "^4.3.0", + "axios": "^1.7.0", + "chart.js": "^4.4.0", + "pinia": "^2.3.0", + "primeicons": "^7.0.0", + "primevue": "^4.3.0", + "vue": "^3.5.0", + "vue-chartjs": "^5.3.0", + "vue-router": "^4.5.0" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.0", + "@tsconfig/node22": "^22.0.0", + "@types/node": "^22.0.0", + "@vitejs/plugin-vue": "^5.2.0", + "@vue/test-utils": "^2.4.0", + "eslint": "^9.0.0", + "happy-dom": "^16.0.0", + "tailwindcss": "^4.1.0", + "typescript": "^5.7.0", + "vite": "^6.2.0", + "vitest": "^3.0.0", + "vue-tsc": "^2.2.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", + "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.5" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", + "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.14.0", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.1", + "minimatch": "^3.1.5", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", + "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@kurkle/color": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz", + "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", + "license": "MIT" + }, + "node_modules/@one-ini/wasm": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@one-ini/wasm/-/wasm-0.1.1.tgz", + "integrity": "sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@primeuix/styled": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/@primeuix/styled/-/styled-0.7.4.tgz", + "integrity": "sha512-QSO/NpOQg8e9BONWRBx9y8VGMCMYz0J/uKfNJEya/RGEu7ARx0oYW0ugI1N3/KB1AAvyGxzKBzGImbwg0KUiOQ==", + "license": "MIT", + "dependencies": { + "@primeuix/utils": "^0.6.1" + }, + "engines": { + "node": ">=12.11.0" + } + }, + "node_modules/@primeuix/styles": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@primeuix/styles/-/styles-2.0.3.tgz", + "integrity": "sha512-2ykAB6BaHzR/6TwF8ShpJTsZrid6cVIEBVlookSdvOdmlWuevGu5vWOScgIwqWwlZcvkFYAGR/SUV3OHCTBMdw==", + "license": "MIT", + "dependencies": { + "@primeuix/styled": "^0.7.4" + } + }, + "node_modules/@primeuix/themes": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@primeuix/themes/-/themes-2.0.3.tgz", + "integrity": "sha512-3fS1883mtCWhgUgNf/feiaaDSOND4EBIOu9tZnzJlJ8QtYyL6eFLcA6V3ymCWqLVXQ1+lTVEZv1gl47FIdXReg==", + "license": "MIT", + "dependencies": { + "@primeuix/styled": "^0.7.4" + } + }, + "node_modules/@primeuix/utils": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/@primeuix/utils/-/utils-0.6.4.tgz", + "integrity": "sha512-pZ5f+vj7wSzRhC7KoEQRU5fvYAe+RP9+m39CTscZ3UywCD1Y2o6Fe1rRgklMPSkzUcty2jzkA0zMYkiJBD1hgg==", + "license": "MIT", + "engines": { + "node": ">=12.11.0" + } + }, + "node_modules/@primevue/core": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@primevue/core/-/core-4.5.4.tgz", + "integrity": "sha512-lYJJB3wTrDJ8MkLctzHfrPZAqXVxoatjIsswSJzupatf6ZogJHVYADUKcn1JAkLLk8dtV1FA2AxDek663fHO5Q==", + "license": "MIT", + "dependencies": { + "@primeuix/styled": "^0.7.4", + "@primeuix/utils": "^0.6.2" + }, + "engines": { + "node": ">=12.11.0" + }, + "peerDependencies": { + "vue": "^3.5.0" + } + }, + "node_modules/@primevue/icons": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@primevue/icons/-/icons-4.5.4.tgz", + "integrity": "sha512-DxgryEc7ZmUqcEhYMcxGBRyFzdtLIoy3jLtlH1zsVSRZaG+iSAcjQ88nvfkZxGUZtZBFL7sRjF6KLq3bJZJwUw==", + "license": "MIT", + "dependencies": { + "@primeuix/utils": "^0.6.2", + "@primevue/core": "4.5.4" + }, + "engines": { + "node": ">=12.11.0" + } + }, + "node_modules/@primevue/themes": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@primevue/themes/-/themes-4.5.4.tgz", + "integrity": "sha512-rUFZxMHLanTZdvZq4zgZPk+KRBZ3s7fE3bBK32OrZBkHQhEJmkJ7Ftd4w4QFlXyz1B7c+k5invZiOOCjwHXg9Q==", + "deprecated": "Deprecated. This package is no longer maintained. Please migrate to @primeuix/themes: https://www.npmjs.com/package/@primeuix/themes", + "license": "MIT", + "dependencies": { + "@primeuix/styled": "^0.7.4", + "@primeuix/themes": "^2.0.2" + }, + "engines": { + "node": ">=12.11.0" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", + "integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz", + "integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz", + "integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz", + "integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz", + "integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz", + "integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz", + "integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz", + "integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz", + "integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz", + "integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz", + "integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz", + "integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz", + "integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz", + "integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz", + "integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz", + "integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz", + "integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz", + "integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz", + "integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz", + "integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz", + "integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz", + "integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz", + "integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz", + "integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz", + "integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tailwindcss/node": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.2.1.tgz", + "integrity": "sha512-jlx6sLk4EOwO6hHe1oCGm1Q4AN/s0rSrTTPBGPM0/RQ6Uylwq17FuU8IeJJKEjtc6K6O07zsvP+gDO6MMWo7pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "^5.19.0", + "jiti": "^2.6.1", + "lightningcss": "1.31.1", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.2.1" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.2.1.tgz", + "integrity": "sha512-yv9jeEFWnjKCI6/T3Oq50yQEOqmpmpfzG1hcZsAOaXFQPfzWprWrlHSdGPEF3WQTi8zu8ohC9Mh9J470nT5pUw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.2.1", + "@tailwindcss/oxide-darwin-arm64": "4.2.1", + "@tailwindcss/oxide-darwin-x64": "4.2.1", + "@tailwindcss/oxide-freebsd-x64": "4.2.1", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.2.1", + "@tailwindcss/oxide-linux-arm64-gnu": "4.2.1", + "@tailwindcss/oxide-linux-arm64-musl": "4.2.1", + "@tailwindcss/oxide-linux-x64-gnu": "4.2.1", + "@tailwindcss/oxide-linux-x64-musl": "4.2.1", + "@tailwindcss/oxide-wasm32-wasi": "4.2.1", + "@tailwindcss/oxide-win32-arm64-msvc": "4.2.1", + "@tailwindcss/oxide-win32-x64-msvc": "4.2.1" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.2.1.tgz", + "integrity": "sha512-eZ7G1Zm5EC8OOKaesIKuw77jw++QJ2lL9N+dDpdQiAB/c/B2wDh0QPFHbkBVrXnwNugvrbJFk1gK2SsVjwWReg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.2.1.tgz", + "integrity": "sha512-q/LHkOstoJ7pI1J0q6djesLzRvQSIfEto148ppAd+BVQK0JYjQIFSK3JgYZJa+Yzi0DDa52ZsQx2rqytBnf8Hw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.2.1.tgz", + "integrity": "sha512-/f/ozlaXGY6QLbpvd/kFTro2l18f7dHKpB+ieXz+Cijl4Mt9AI2rTrpq7V+t04nK+j9XBQHnSMdeQRhbGyt6fw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.2.1.tgz", + "integrity": "sha512-5e/AkgYJT/cpbkys/OU2Ei2jdETCLlifwm7ogMC7/hksI2fC3iiq6OcXwjibcIjPung0kRtR3TxEITkqgn0TcA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.2.1.tgz", + "integrity": "sha512-Uny1EcVTTmerCKt/1ZuKTkb0x8ZaiuYucg2/kImO5A5Y/kBz41/+j0gxUZl+hTF3xkWpDmHX+TaWhOtba2Fyuw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.2.1.tgz", + "integrity": "sha512-CTrwomI+c7n6aSSQlsPL0roRiNMDQ/YzMD9EjcR+H4f0I1SQ8QqIuPnsVp7QgMkC1Qi8rtkekLkOFjo7OlEFRQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.2.1.tgz", + "integrity": "sha512-WZA0CHRL/SP1TRbA5mp9htsppSEkWuQ4KsSUumYQnyl8ZdT39ntwqmz4IUHGN6p4XdSlYfJwM4rRzZLShHsGAQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.2.1.tgz", + "integrity": "sha512-qMFzxI2YlBOLW5PhblzuSWlWfwLHaneBE0xHzLrBgNtqN6mWfs+qYbhryGSXQjFYB1Dzf5w+LN5qbUTPhW7Y5g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.2.1.tgz", + "integrity": "sha512-5r1X2FKnCMUPlXTWRYpHdPYUY6a1Ar/t7P24OuiEdEOmms5lyqjDRvVY1yy9Rmioh+AunQ0rWiOTPE8F9A3v5g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.2.1.tgz", + "integrity": "sha512-MGFB5cVPvshR85MTJkEvqDUnuNoysrsRxd6vnk1Lf2tbiqNlXpHYZqkqOQalydienEWOHHFyyuTSYRsLfxFJ2Q==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.8.1", + "@emnapi/runtime": "^1.8.1", + "@emnapi/wasi-threads": "^1.1.0", + "@napi-rs/wasm-runtime": "^1.1.1", + "@tybys/wasm-util": "^0.10.1", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.2.1.tgz", + "integrity": "sha512-YlUEHRHBGnCMh4Nj4GnqQyBtsshUPdiNroZj8VPkvTZSoHsilRCwXcVKnG9kyi0ZFAS/3u+qKHBdDc81SADTRA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.2.1.tgz", + "integrity": "sha512-rbO34G5sMWWyrN/idLeVxAZgAKWrn5LiR3/I90Q9MkA67s6T1oB0xtTe+0heoBvHSpbU9Mk7i6uwJnpo4u21XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.2.1.tgz", + "integrity": "sha512-TBf2sJjYeb28jD2U/OhwdW0bbOsxkWPwQ7SrqGf9sVcoYwZj7rkXljroBO9wKBut9XnmQLXanuDUeqQK0lGg/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.2.1", + "@tailwindcss/oxide": "4.2.1", + "tailwindcss": "4.2.1" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7" + } + }, + "node_modules/@tsconfig/node22": { + "version": "22.0.5", + "resolved": "https://registry.npmjs.org/@tsconfig/node22/-/node22-22.0.5.tgz", + "integrity": "sha512-hLf2ld+sYN/BtOJjHUWOk568dvjFQkHnLNa6zce25GIH+vxKfvTgm3qpaH6ToF5tu/NN0IH66s+Bb5wElHrLcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.15", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.15.tgz", + "integrity": "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vitejs/plugin-vue": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.2.4.tgz", + "integrity": "sha512-7Yx/SXSOcQq5HiiV3orevHUFn+pmMB4cgbEkDYgnkUWb0WfeQ/wa2yFv6D5ICiCQOVpjA7vYDXrC7AGO8yjDHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "peerDependencies": { + "vite": "^5.0.0 || ^6.0.0", + "vue": "^3.2.25" + } + }, + "node_modules/@vitest/expect": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", + "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", + "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", + "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.4", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", + "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", + "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@volar/language-core": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@volar/language-core/-/language-core-2.4.15.tgz", + "integrity": "sha512-3VHw+QZU0ZG9IuQmzT68IyN4hZNd9GchGPhbD9+pa8CVv7rnoOZwo7T8weIbrRmihqy3ATpdfXFnqRrfPVK6CA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/source-map": "2.4.15" + } + }, + "node_modules/@volar/source-map": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@volar/source-map/-/source-map-2.4.15.tgz", + "integrity": "sha512-CPbMWlUN6hVZJYGcU/GSoHu4EnCHiLaXI9n8c9la6RaI9W5JHX+NqG+GSQcB0JdC2FIBLdZJwGsfKyBB71VlTg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@volar/typescript": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@volar/typescript/-/typescript-2.4.15.tgz", + "integrity": "sha512-2aZ8i0cqPGjXb4BhkMsPYDkkuc2ZQ6yOpqwAuNwUoncELqoy5fRgOQtLR9gB0g902iS0NAkvpIzs27geVyVdPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/language-core": "2.4.15", + "path-browserify": "^1.0.1", + "vscode-uri": "^3.0.8" + } + }, + "node_modules/@vue/compiler-core": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.30.tgz", + "integrity": "sha512-s3DfdZkcu/qExZ+td75015ljzHc6vE+30cFMGRPROYjqkroYI5NV2X1yAMX9UeyBNWB9MxCfPcsjpLS11nzkkw==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@vue/shared": "3.5.30", + "entities": "^7.0.1", + "estree-walker": "^2.0.2", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-core/node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "license": "MIT" + }, + "node_modules/@vue/compiler-dom": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.30.tgz", + "integrity": "sha512-eCFYESUEVYHhiMuK4SQTldO3RYxyMR/UQL4KdGD1Yrkfdx4m/HYuZ9jSfPdA+nWJY34VWndiYdW/wZXyiPEB9g==", + "license": "MIT", + "dependencies": { + "@vue/compiler-core": "3.5.30", + "@vue/shared": "3.5.30" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.30.tgz", + "integrity": "sha512-LqmFPDn89dtU9vI3wHJnwaV6GfTRD87AjWpTWpyrdVOObVtjIuSeZr181z5C4PmVx/V3j2p+0f7edFKGRMpQ5A==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@vue/compiler-core": "3.5.30", + "@vue/compiler-dom": "3.5.30", + "@vue/compiler-ssr": "3.5.30", + "@vue/shared": "3.5.30", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.21", + "postcss": "^8.5.8", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-sfc/node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "license": "MIT" + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.30.tgz", + "integrity": "sha512-NsYK6OMTnx109PSL2IAyf62JP6EUdk4Dmj6AkWcJGBvN0dQoMYtVekAmdqgTtWQgEJo+Okstbf/1p7qZr5H+bA==", + "license": "MIT", + "dependencies": { + "@vue/compiler-dom": "3.5.30", + "@vue/shared": "3.5.30" + } + }, + "node_modules/@vue/compiler-vue2": { + "version": "2.7.16", + "resolved": "https://registry.npmjs.org/@vue/compiler-vue2/-/compiler-vue2-2.7.16.tgz", + "integrity": "sha512-qYC3Psj9S/mfu9uVi5WvNZIzq+xnXMhOwbTFKKDD7b1lhpnn71jXSFdTQ+WsIEk0ONCd7VV2IMm7ONl6tbQ86A==", + "dev": true, + "license": "MIT", + "dependencies": { + "de-indent": "^1.0.2", + "he": "^1.2.0" + } + }, + "node_modules/@vue/devtools-api": { + "version": "6.6.4", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.6.4.tgz", + "integrity": "sha512-sGhTPMuXqZ1rVOk32RylztWkfXTRhuS7vgAKv0zjqk8gbsHkJ7xfFf+jbySxt7tWObEJwyKaHMikV/WGDiQm8g==", + "license": "MIT" + }, + "node_modules/@vue/language-core": { + "version": "2.2.12", + "resolved": "https://registry.npmjs.org/@vue/language-core/-/language-core-2.2.12.tgz", + "integrity": "sha512-IsGljWbKGU1MZpBPN+BvPAdr55YPkj2nB/TBNGNC32Vy2qLG25DYu/NBN2vNtZqdRbTRjaoYrahLrToim2NanA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/language-core": "2.4.15", + "@vue/compiler-dom": "^3.5.0", + "@vue/compiler-vue2": "^2.7.16", + "@vue/shared": "^3.5.0", + "alien-signals": "^1.0.3", + "minimatch": "^9.0.3", + "muggle-string": "^0.4.1", + "path-browserify": "^1.0.1" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@vue/language-core/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@vue/language-core/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@vue/reactivity": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.30.tgz", + "integrity": "sha512-179YNgKATuwj9gB+66snskRDOitDiuOZqkYia7mHKJaidOMo/WJxHKF8DuGc4V4XbYTJANlfEKb0yxTQotnx4Q==", + "license": "MIT", + "dependencies": { + "@vue/shared": "3.5.30" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.30.tgz", + "integrity": "sha512-e0Z+8PQsUTdwV8TtEsLzUM7SzC7lQwYKePydb7K2ZnmS6jjND+WJXkmmfh/swYzRyfP1EY3fpdesyYoymCzYfg==", + "license": "MIT", + "dependencies": { + "@vue/reactivity": "3.5.30", + "@vue/shared": "3.5.30" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.30.tgz", + "integrity": "sha512-2UIGakjU4WSQ0T4iwDEW0W7vQj6n7AFn7taqZ9Cvm0Q/RA2FFOziLESrDL4GmtI1wV3jXg5nMoJSYO66egDUBw==", + "license": "MIT", + "dependencies": { + "@vue/reactivity": "3.5.30", + "@vue/runtime-core": "3.5.30", + "@vue/shared": "3.5.30", + "csstype": "^3.2.3" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.30.tgz", + "integrity": "sha512-v+R34icapydRwbZRD0sXwtHqrQJv38JuMB4JxbOxd8NEpGLny7cncMp53W9UH/zo4j8eDHjQ1dEJXwzFQknjtQ==", + "license": "MIT", + "dependencies": { + "@vue/compiler-ssr": "3.5.30", + "@vue/shared": "3.5.30" + }, + "peerDependencies": { + "vue": "3.5.30" + } + }, + "node_modules/@vue/shared": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.30.tgz", + "integrity": "sha512-YXgQ7JjaO18NeK2K9VTbDHaFy62WrObMa6XERNfNOkAhD1F1oDSf3ZJ7K6GqabZ0BvSDHajp8qfS5Sa2I9n8uQ==", + "license": "MIT" + }, + "node_modules/@vue/test-utils": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/@vue/test-utils/-/test-utils-2.4.6.tgz", + "integrity": "sha512-FMxEjOpYNYiFe0GkaHsnJPXFHxQ6m4t8vI/ElPGpMWxZKpmRvQ33OIrvRXemy6yha03RxhOlQuy+gZMC3CQSow==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-beautify": "^1.14.9", + "vue-component-type-helpers": "^2.0.0" + } + }, + "node_modules/abbrev": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-2.0.0.tgz", + "integrity": "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", + "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/alien-signals": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/alien-signals/-/alien-signals-1.0.13.tgz", + "integrity": "sha512-OGj9yyTnJEttvzhTUWuscOvtqxq5vrhF7vL9oS0xJ2mK0ItPYP1/y+vCFebfxoEyAz0++1AIwJ5CMr+Fk3nDmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.6.tgz", + "integrity": "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chart.js": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", + "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", + "license": "MIT", + "dependencies": { + "@kurkle/color": "^0.3.0" + }, + "engines": { + "pnpm": ">=8" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "license": "MIT" + }, + "node_modules/de-indent": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz", + "integrity": "sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/editorconfig": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz", + "integrity": "sha512-e0GOtq/aTQhVdNyDU9e02+wz9oDDM+SIOQxWME2QRjzRX5yyLAuHDE+0aE8vHb9XRC8XD37eO2u57+F09JqFhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@one-ini/wasm": "0.1.1", + "commander": "^10.0.0", + "minimatch": "^9.0.1", + "semver": "^7.5.3" + }, + "bin": { + "editorconfig": "bin/editorconfig" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/editorconfig/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/editorconfig/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/enhanced-resolve": { + "version": "5.20.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.20.1.tgz", + "integrity": "sha512-Qohcme7V1inbAfvjItgw0EaxVX5q2rdVEZHRBrEQdRZTssLDGsL8Lwrznl8oQ/6kuTJONLaDcGjkNP247XEhcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.4", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", + "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.2", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.5", + "@eslint/js": "9.39.4", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.1.tgz", + "integrity": "sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/happy-dom": { + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-16.8.1.tgz", + "integrity": "sha512-n0QrmT9lD81rbpKsyhnlz3DgnMZlaOkJPpgi746doA+HvaMC79bdWkwjrNnGJRvDrWTI8iOcJiVTJ5CdT/AZRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "webidl-conversions": "^7.0.0", + "whatwg-mimetype": "^3.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/jiti": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", + "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/js-beautify": { + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.15.4.tgz", + "integrity": "sha512-9/KXeZUKKJwqCXUdBxFJ3vPh467OCckSBmYDwSK/EtV090K+iMJ7zx2S3HLVDIWFQdqMIsZWbnaGiba18aWhaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "config-chain": "^1.1.13", + "editorconfig": "^1.0.4", + "glob": "^10.4.2", + "js-cookie": "^3.0.5", + "nopt": "^7.2.1" + }, + "bin": { + "css-beautify": "js/bin/css-beautify.js", + "html-beautify": "js/bin/html-beautify.js", + "js-beautify": "js/bin/js-beautify.js" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/js-cookie": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz", + "integrity": "sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lightningcss": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.31.1.tgz", + "integrity": "sha512-l51N2r93WmGUye3WuFoN5k10zyvrVs0qfKBhyC5ogUQ6Ew6JUSswh78mbSO+IU3nTWsyOArqPCcShdQSadghBQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.31.1", + "lightningcss-darwin-arm64": "1.31.1", + "lightningcss-darwin-x64": "1.31.1", + "lightningcss-freebsd-x64": "1.31.1", + "lightningcss-linux-arm-gnueabihf": "1.31.1", + "lightningcss-linux-arm64-gnu": "1.31.1", + "lightningcss-linux-arm64-musl": "1.31.1", + "lightningcss-linux-x64-gnu": "1.31.1", + "lightningcss-linux-x64-musl": "1.31.1", + "lightningcss-win32-arm64-msvc": "1.31.1", + "lightningcss-win32-x64-msvc": "1.31.1" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.31.1.tgz", + "integrity": "sha512-HXJF3x8w9nQ4jbXRiNppBCqeZPIAfUo8zE/kOEGbW5NZvGc/K7nMxbhIr+YlFlHW5mpbg/YFPdbnCh1wAXCKFg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.31.1.tgz", + "integrity": "sha512-02uTEqf3vIfNMq3h/z2cJfcOXnQ0GRwQrkmPafhueLb2h7mqEidiCzkE4gBMEH65abHRiQvhdcQ+aP0D0g67sg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.31.1.tgz", + "integrity": "sha512-1ObhyoCY+tGxtsz1lSx5NXCj3nirk0Y0kB/g8B8DT+sSx4G9djitg9ejFnjb3gJNWo7qXH4DIy2SUHvpoFwfTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.31.1.tgz", + "integrity": "sha512-1RINmQKAItO6ISxYgPwszQE1BrsVU5aB45ho6O42mu96UiZBxEXsuQ7cJW4zs4CEodPUioj/QrXW1r9pLUM74A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.31.1.tgz", + "integrity": "sha512-OOCm2//MZJ87CdDK62rZIu+aw9gBv4azMJuA8/KB74wmfS3lnC4yoPHm0uXZ/dvNNHmnZnB8XLAZzObeG0nS1g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.31.1.tgz", + "integrity": "sha512-WKyLWztD71rTnou4xAD5kQT+982wvca7E6QoLpoawZ1gP9JM0GJj4Tp5jMUh9B3AitHbRZ2/H3W5xQmdEOUlLg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.31.1.tgz", + "integrity": "sha512-mVZ7Pg2zIbe3XlNbZJdjs86YViQFoJSpc41CbVmKBPiGmC4YrfeOyz65ms2qpAobVd7WQsbW4PdsSJEMymyIMg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.31.1.tgz", + "integrity": "sha512-xGlFWRMl+0KvUhgySdIaReQdB4FNudfUTARn7q0hh/V67PVGCs3ADFjw+6++kG1RNd0zdGRlEKa+T13/tQjPMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.31.1.tgz", + "integrity": "sha512-eowF8PrKHw9LpoZii5tdZwnBcYDxRw2rRCyvAXLi34iyeYfqCQNA9rmUM0ce62NlPhCvof1+9ivRaTY6pSKDaA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.31.1.tgz", + "integrity": "sha512-aJReEbSEQzx1uBlQizAOBSjcmr9dCdL3XuC/6HLXAxmtErsj2ICo5yYggg1qOODQMtnjNQv2UHb9NpOuFtYe4w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.31.1.tgz", + "integrity": "sha512-I9aiFrbd7oYHwlnQDqr1Roz+fTz61oDDJX7n9tYF9FJymH1cIN1DtKw3iYt6b8WZgEjoNwVSncwF4wx/ZedMhw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/muggle-string": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/muggle-string/-/muggle-string-0.4.1.tgz", + "integrity": "sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/nopt": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.1.tgz", + "integrity": "sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==", + "dev": true, + "license": "ISC", + "dependencies": { + "abbrev": "^2.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pinia": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/pinia/-/pinia-2.3.1.tgz", + "integrity": "sha512-khUlZSwt9xXCaTbbxFYBKDc/bWAGWJjOgvxETwkTN7KRm66EeT1ZdZj6i2ceh9sP2Pzqsbc704r2yngBrxBVug==", + "license": "MIT", + "dependencies": { + "@vue/devtools-api": "^6.6.3", + "vue-demi": "^0.14.10" + }, + "funding": { + "url": "https://github.com/sponsors/posva" + }, + "peerDependencies": { + "typescript": ">=4.4.4", + "vue": "^2.7.0 || ^3.5.11" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/postcss": { + "version": "8.5.8", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", + "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/primeicons": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/primeicons/-/primeicons-7.0.0.tgz", + "integrity": "sha512-jK3Et9UzwzTsd6tzl2RmwrVY/b8raJ3QZLzoDACj+oTJ0oX7L9Hy+XnVwgo4QVKlKpnP/Ur13SXV/pVh4LzaDw==", + "license": "MIT" + }, + "node_modules/primevue": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/primevue/-/primevue-4.5.4.tgz", + "integrity": "sha512-nTyEohZABFJhVIpeUxgP0EJ8vKcJAhD+Z7DYj95e7ie/MNUCjRNcGjqmE1cXtXi4z54qDfTSI9h2uJ51qz2DIw==", + "license": "MIT", + "dependencies": { + "@primeuix/styled": "^0.7.4", + "@primeuix/styles": "^2.0.2", + "@primeuix/utils": "^0.6.2", + "@primevue/core": "4.5.4", + "@primevue/icons": "4.5.4" + }, + "engines": { + "node": ">=12.11.0" + } + }, + "node_modules/proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "dev": true, + "license": "ISC" + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rollup": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", + "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.59.0", + "@rollup/rollup-android-arm64": "4.59.0", + "@rollup/rollup-darwin-arm64": "4.59.0", + "@rollup/rollup-darwin-x64": "4.59.0", + "@rollup/rollup-freebsd-arm64": "4.59.0", + "@rollup/rollup-freebsd-x64": "4.59.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", + "@rollup/rollup-linux-arm-musleabihf": "4.59.0", + "@rollup/rollup-linux-arm64-gnu": "4.59.0", + "@rollup/rollup-linux-arm64-musl": "4.59.0", + "@rollup/rollup-linux-loong64-gnu": "4.59.0", + "@rollup/rollup-linux-loong64-musl": "4.59.0", + "@rollup/rollup-linux-ppc64-gnu": "4.59.0", + "@rollup/rollup-linux-ppc64-musl": "4.59.0", + "@rollup/rollup-linux-riscv64-gnu": "4.59.0", + "@rollup/rollup-linux-riscv64-musl": "4.59.0", + "@rollup/rollup-linux-s390x-gnu": "4.59.0", + "@rollup/rollup-linux-x64-gnu": "4.59.0", + "@rollup/rollup-linux-x64-musl": "4.59.0", + "@rollup/rollup-openbsd-x64": "4.59.0", + "@rollup/rollup-openharmony-arm64": "4.59.0", + "@rollup/rollup-win32-arm64-msvc": "4.59.0", + "@rollup/rollup-win32-ia32-msvc": "4.59.0", + "@rollup/rollup-win32-x64-gnu": "4.59.0", + "@rollup/rollup-win32-x64-msvc": "4.59.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tailwindcss": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.1.tgz", + "integrity": "sha512-/tBrSQ36vCleJkAOsy9kbNTgaxvGbyOamC30PRePTQe/o1MFwEKHQk4Cn7BNGaPtjp+PuUrByJehM1hgxfq4sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", + "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", + "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", + "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.4", + "@vitest/mocker": "3.2.4", + "@vitest/pretty-format": "^3.2.4", + "@vitest/runner": "3.2.4", + "@vitest/snapshot": "3.2.4", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.4", + "@vitest/ui": "3.2.4", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vscode-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.1.0.tgz", + "integrity": "sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vue": { + "version": "3.5.30", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.30.tgz", + "integrity": "sha512-hTHLc6VNZyzzEH/l7PFGjpcTvUgiaPK5mdLkbjrTeWSRcEfxFrv56g/XckIYlE9ckuobsdwqd5mk2g1sBkMewg==", + "license": "MIT", + "dependencies": { + "@vue/compiler-dom": "3.5.30", + "@vue/compiler-sfc": "3.5.30", + "@vue/runtime-dom": "3.5.30", + "@vue/server-renderer": "3.5.30", + "@vue/shared": "3.5.30" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/vue-chartjs": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/vue-chartjs/-/vue-chartjs-5.3.3.tgz", + "integrity": "sha512-jqxtL8KZ6YJ5NTv6XzrzLS7osyegOi28UGNZW0h9OkDL7Sh1396ht4Dorh04aKrl2LiSalQ84WtqiG0RIJb0tA==", + "license": "MIT", + "peerDependencies": { + "chart.js": "^4.1.1", + "vue": "^3.0.0-0 || ^2.7.0" + } + }, + "node_modules/vue-component-type-helpers": { + "version": "2.2.12", + "resolved": "https://registry.npmjs.org/vue-component-type-helpers/-/vue-component-type-helpers-2.2.12.tgz", + "integrity": "sha512-YbGqHZ5/eW4SnkPNR44mKVc6ZKQoRs/Rux1sxC6rdwXb4qpbOSYfDr9DsTHolOTGmIKgM9j141mZbBeg05R1pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/vue-demi": { + "version": "0.14.10", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.10.tgz", + "integrity": "sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/vue-router": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-4.6.4.tgz", + "integrity": "sha512-Hz9q5sa33Yhduglwz6g9skT8OBPii+4bFn88w6J+J4MfEo4KRRpmiNG/hHHkdbRFlLBOqxN8y8gf2Fb0MTUgVg==", + "license": "MIT", + "dependencies": { + "@vue/devtools-api": "^6.6.4" + }, + "funding": { + "url": "https://github.com/sponsors/posva" + }, + "peerDependencies": { + "vue": "^3.5.0" + } + }, + "node_modules/vue-tsc": { + "version": "2.2.12", + "resolved": "https://registry.npmjs.org/vue-tsc/-/vue-tsc-2.2.12.tgz", + "integrity": "sha512-P7OP77b2h/Pmk+lZdJ0YWs+5tJ6J2+uOQPo7tlBnY44QqQSPYvS0qVT4wqDJgwrZaLe47etJLLQRFia71GYITw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/typescript": "2.4.15", + "@vue/language-core": "2.2.12" + }, + "bin": { + "vue-tsc": "bin/vue-tsc.js" + }, + "peerDependencies": { + "typescript": ">=5.0.0" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/artipie-ui/package.json b/artipie-ui/package.json new file mode 100644 index 000000000..b2a6de410 --- /dev/null +++ b/artipie-ui/package.json @@ -0,0 +1,40 @@ +{ + "name": "artipie-ui", + "version": "1.22.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vue-tsc --noEmit && vite build", + "preview": "vite preview", + "test": "vitest run", + "test:watch": "vitest", + "lint": "eslint src/", + "type-check": "vue-tsc --noEmit" + }, + "dependencies": { + "@primevue/themes": "^4.3.0", + "axios": "^1.7.0", + "chart.js": "^4.4.0", + "pinia": "^2.3.0", + "primeicons": "^7.0.0", + "primevue": "^4.3.0", + "vue": "^3.5.0", + "vue-chartjs": "^5.3.0", + "vue-router": "^4.5.0" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.0", + "@tsconfig/node22": "^22.0.0", + "@types/node": "^22.0.0", + "@vitejs/plugin-vue": "^5.2.0", + "@vue/test-utils": "^2.4.0", + "eslint": "^9.0.0", + "happy-dom": "^16.0.0", + "tailwindcss": "^4.1.0", + "typescript": "^5.7.0", + "vite": "^6.2.0", + "vitest": "^3.0.0", + "vue-tsc": "^2.2.0" + } +} diff --git a/artipie-ui/public/config.json b/artipie-ui/public/config.json new file mode 100644 index 000000000..f86415705 --- /dev/null +++ b/artipie-ui/public/config.json @@ -0,0 +1,6 @@ +{ + "apiBaseUrl": "http://localhost:8086/api/v1", + "grafanaUrl": "http://localhost:3000", + "appTitle": "Artipie", + "defaultPageSize": 20 +} \ No newline at end of file diff --git a/artipie-ui/public/config.json.template b/artipie-ui/public/config.json.template new file mode 100644 index 000000000..a9fb80695 --- /dev/null +++ b/artipie-ui/public/config.json.template @@ -0,0 +1,6 @@ +{ + "apiBaseUrl": "$API_BASE_URL", + "grafanaUrl": "$GRAFANA_URL", + "appTitle": "$APP_TITLE", + "defaultPageSize": $DEFAULT_PAGE_SIZE +} diff --git a/artipie-ui/src/App.vue b/artipie-ui/src/App.vue new file mode 100644 index 000000000..8206d2c52 --- /dev/null +++ b/artipie-ui/src/App.vue @@ -0,0 +1,14 @@ + + + diff --git a/artipie-ui/src/api/__tests__/client.test.ts b/artipie-ui/src/api/__tests__/client.test.ts new file mode 100644 index 000000000..bdc2c46e9 --- /dev/null +++ b/artipie-ui/src/api/__tests__/client.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { initApiClient, getApiClient } from '../client' + +describe('API Client', () => { + beforeEach(() => { + sessionStorage.clear() + }) + + it('creates axios instance with base URL', () => { + const client = initApiClient('/api/v1') + expect(client.defaults.baseURL).toBe('/api/v1') + expect(client.defaults.timeout).toBe(10_000) + }) + + it('getApiClient returns initialized instance', () => { + initApiClient('/api/v1') + const client = getApiClient() + expect(client).toBeDefined() + expect(client.defaults.baseURL).toBe('/api/v1') + }) + + it('sets Authorization header when JWT is in sessionStorage', () => { + sessionStorage.setItem('jwt', 'test-token-123') + const client = initApiClient('/api/v1') + // Verify interceptor is registered (request interceptors array) + expect(client.interceptors.request).toBeDefined() + }) +}) diff --git a/artipie-ui/src/api/auth.ts b/artipie-ui/src/api/auth.ts new file mode 100644 index 000000000..aae42e574 --- /dev/null +++ b/artipie-ui/src/api/auth.ts @@ -0,0 +1,82 @@ +import { getApiClient } from './client' +import type { AuthProvidersResponse, TokenResponse, UserInfo } from '@/types' + +export async function getProviders(): Promise { + const { data } = await getApiClient().get('/auth/providers') + return data +} + +export async function login(name: string, pass: string): Promise { + const { data } = await getApiClient().post('/auth/token', { name, pass }) + return data +} + +export async function getMe(): Promise { + const { data } = await getApiClient().get('/auth/me') + return data +} + +export async function getProviderRedirect( + name: string, + callbackUrl: string, +): Promise<{ url: string; state: string }> { + const { data } = await getApiClient().get<{ url: string; state: string }>( + `/auth/providers/${name}/redirect`, + { params: { callback_url: callbackUrl } }, + ) + return data +} + +export interface GenerateTokenResponse { + token: string + id: string + label: string + expires_at?: string + permanent: boolean +} + +/** + * Generate an API token for the currently authenticated user. + * Uses the session JWT — no password required. + */ +export async function generateTokenForSession( + expiryDays = 30, + label = 'API Token', +): Promise { + const { data } = await getApiClient().post('/auth/token/generate', { + expiry_days: expiryDays, + label, + }) + return data +} + +export interface ApiToken { + id: string + label: string + created_at: string + expires_at?: string + expired?: boolean + permanent?: boolean +} + +export async function listTokens(): Promise { + const { data } = await getApiClient().get<{ tokens: ApiToken[] }>('/auth/tokens') + return data.tokens +} + +export async function revokeToken(tokenId: string): Promise { + await getApiClient().delete(`/auth/tokens/${tokenId}`) +} + +export async function exchangeOAuthCode( + code: string, + provider: string, + callbackUrl: string, +): Promise { + const { data } = await getApiClient().post('/auth/callback', { + code, + provider, + callback_url: callbackUrl, + }) + return data +} diff --git a/artipie-ui/src/api/client.ts b/artipie-ui/src/api/client.ts new file mode 100644 index 000000000..c57325e12 --- /dev/null +++ b/artipie-ui/src/api/client.ts @@ -0,0 +1,40 @@ +import axios, { type AxiosInstance, type InternalAxiosRequestConfig } from 'axios' + +let apiClient: AxiosInstance | null = null + +export function initApiClient(baseUrl: string): AxiosInstance { + apiClient = axios.create({ + baseURL: baseUrl, + timeout: 10_000, + headers: { 'Content-Type': 'application/json' }, + }) + apiClient.interceptors.request.use((config: InternalAxiosRequestConfig) => { + const token = sessionStorage.getItem('jwt') + if (token) { + config.headers.Authorization = `Bearer ${token}` + } + return config + }) + apiClient.interceptors.response.use( + (response) => response, + (error) => { + if (error.response?.status === 401) { + // Don't logout on 401 from public auth endpoints (token generation, login) + const url = error.config?.url ?? '' + if (!url.includes('/auth/token') && !url.includes('/auth/providers') && !url.includes('/auth/callback')) { + sessionStorage.removeItem('jwt') + window.location.href = '/login' + } + } + return Promise.reject(error) + }, + ) + return apiClient +} + +export function getApiClient(): AxiosInstance { + if (!apiClient) { + throw new Error('API client not initialized. Call initApiClient() first.') + } + return apiClient +} diff --git a/artipie-ui/src/api/repos.ts b/artipie-ui/src/api/repos.ts new file mode 100644 index 000000000..76543d30c --- /dev/null +++ b/artipie-ui/src/api/repos.ts @@ -0,0 +1,97 @@ +import { getApiClient } from './client' +import type { + PaginatedResponse, CursorResponse, Repository, RepoMember, RepoListItem, + TreeEntry, ArtifactDetail, PullInstructions, StorageAlias, +} from '@/types' + +export async function listRepos(params: { + page?: number; size?: number; type?: string; q?: string +} = {}): Promise> { + const { data } = await getApiClient().get('/repositories', { params }) + return data +} + +export async function getRepo(name: string): Promise> { + const { data } = await getApiClient().get>(`/repositories/${name}`) + return data +} + +export async function repoExists(name: string): Promise { + try { + await getApiClient().head(`/repositories/${name}`) + return true + } catch { + return false + } +} + +export async function putRepo(name: string, config: Record): Promise { + await getApiClient().put(`/repositories/${name}`, config) +} + +export async function deleteRepo(name: string): Promise { + await getApiClient().delete(`/repositories/${name}`) +} + +export async function moveRepo(name: string, newName: string): Promise { + await getApiClient().put(`/repositories/${name}/move`, { new_name: newName }) +} + +export async function getMembers(name: string): Promise { + const { data } = await getApiClient().get<{ members: RepoMember[] }>(`/repositories/${name}/members`) + return data.members ?? [] +} + +export async function getTree(name: string, params: { + path?: string; limit?: number; marker?: string +} = {}): Promise> { + const { data } = await getApiClient().get(`/repositories/${name}/tree`, { params }) + return data +} + +export async function getArtifactDetail(name: string, path: string): Promise { + const { data } = await getApiClient().get(`/repositories/${name}/artifact`, { + params: { path }, + }) + return data +} + +export async function getPullInstructions(name: string, path: string): Promise { + const { data } = await getApiClient().get(`/repositories/${name}/artifact/pull`, { + params: { path }, + }) + return data +} + +export async function deleteArtifacts(name: string, path: string): Promise { + await getApiClient().delete(`/repositories/${name}/artifacts`, { data: { path } }) +} + +export async function deletePackages(name: string, packageName: string): Promise { + await getApiClient().delete(`/repositories/${name}/packages`, { + params: { package: packageName }, + }) +} + +// Storage aliases per repo +export async function getRepoStorages(name: string): Promise { + const { data } = await getApiClient().get(`/repositories/${name}/storages`) + return data.items ?? data +} + +export async function putRepoStorage(repoName: string, alias: string, config: Record): Promise { + await getApiClient().put(`/repositories/${repoName}/storages/${alias}`, config) +} + +export async function deleteRepoStorage(repoName: string, alias: string): Promise { + await getApiClient().delete(`/repositories/${repoName}/storages/${alias}`) +} + +// Cooldown per repo +export async function unblockArtifact(name: string, body: Record): Promise { + await getApiClient().post(`/repositories/${name}/cooldown/unblock`, body) +} + +export async function unblockAll(name: string): Promise { + await getApiClient().post(`/repositories/${name}/cooldown/unblock-all`) +} diff --git a/artipie-ui/src/api/roles.ts b/artipie-ui/src/api/roles.ts new file mode 100644 index 000000000..be7b313bc --- /dev/null +++ b/artipie-ui/src/api/roles.ts @@ -0,0 +1,30 @@ +import { getApiClient } from './client' +import type { PaginatedResponse, Role } from '@/types' + +export async function listRoles(params: { + page?: number; size?: number +} = {}): Promise> { + const { data } = await getApiClient().get('/roles', { params }) + return data +} + +export async function getRole(name: string): Promise { + const { data } = await getApiClient().get(`/roles/${name}`) + return data +} + +export async function putRole(name: string, body: Record): Promise { + await getApiClient().put(`/roles/${name}`, body) +} + +export async function deleteRole(name: string): Promise { + await getApiClient().delete(`/roles/${name}`) +} + +export async function enableRole(name: string): Promise { + await getApiClient().post(`/roles/${name}/enable`) +} + +export async function disableRole(name: string): Promise { + await getApiClient().post(`/roles/${name}/disable`) +} diff --git a/artipie-ui/src/api/search.ts b/artipie-ui/src/api/search.ts new file mode 100644 index 000000000..2c8854d2d --- /dev/null +++ b/artipie-ui/src/api/search.ts @@ -0,0 +1,26 @@ +import { getApiClient } from './client' +import type { PaginatedResponse, SearchResult, LocateResponse, ReindexResponse } from '@/types' + +export async function search(params: { + q: string; page?: number; size?: number +}): Promise> { + const { data } = await getApiClient().get('/search', { params }) + return data +} + +export async function locate(path: string): Promise { + const { data } = await getApiClient().get('/search/locate', { + params: { path }, + }) + return data +} + +export async function reindex(): Promise { + const { data } = await getApiClient().post('/search/reindex') + return data +} + +export async function searchStats(): Promise> { + const { data } = await getApiClient().get('/search/stats') + return data +} diff --git a/artipie-ui/src/api/settings.ts b/artipie-ui/src/api/settings.ts new file mode 100644 index 000000000..b3d4dbe60 --- /dev/null +++ b/artipie-ui/src/api/settings.ts @@ -0,0 +1,90 @@ +import { getApiClient } from './client' +import type { + Settings, HealthResponse, DashboardStats, ReposByType, + StorageAlias, PaginatedResponse, CooldownRepo, BlockedArtifact, + CooldownConfig, +} from '@/types' + +// Settings +export async function getSettings(): Promise { + const { data } = await getApiClient().get('/settings') + return data +} + +export async function updatePrefixes(prefixes: string[]): Promise { + await getApiClient().put('/settings/prefixes', { prefixes }) +} + +// Health +export async function getHealth(): Promise { + const { data } = await getApiClient().get('/health') + return data +} + +// Dashboard +export async function getDashboardStats(): Promise { + const { data } = await getApiClient().get('/dashboard/stats') + return data +} + +export async function getDashboardRequests(period = '24h'): Promise> { + const { data } = await getApiClient().get('/dashboard/requests', { params: { period } }) + return data +} + +export async function getReposByType(): Promise { + const { data } = await getApiClient().get('/dashboard/repos-by-type') + return data +} + +// Global storage aliases +export async function listStorages(): Promise { + const { data } = await getApiClient().get('/storages') + return data.items ?? data +} + +export async function putStorage(name: string, config: Record): Promise { + await getApiClient().put(`/storages/${name}`, config) +} + +export async function deleteStorage(name: string): Promise { + await getApiClient().delete(`/storages/${name}`) +} + +export async function updateSettingsSection( + section: string, + data: Record, +): Promise { + await getApiClient().put(`/settings/${section}`, data) +} + +// Cooldown +export async function getCooldownOverview(): Promise { + const { data } = await getApiClient().get('/cooldown/overview') + return data.repos ?? [] +} + +export async function getCooldownBlocked(params: { + repo?: string; page?: number; size?: number; search?: string +} = {}): Promise> { + const { data } = await getApiClient().get('/cooldown/blocked', { params }) + return data +} + +export async function getCooldownConfig(): Promise { + const { data } = await getApiClient().get('/cooldown/config') + return data +} + +export async function updateCooldownConfig(config: CooldownConfig): Promise { + await getApiClient().put('/cooldown/config', config) +} + +// Auth provider management +export async function toggleAuthProvider(id: number, enabled: boolean): Promise { + await getApiClient().put(`/auth-providers/${id}/toggle`, { enabled }) +} + +export async function updateAuthProviderConfig(id: number, config: Record): Promise { + await getApiClient().put(`/auth-providers/${id}/config`, config) +} diff --git a/artipie-ui/src/api/users.ts b/artipie-ui/src/api/users.ts new file mode 100644 index 000000000..8d6876ece --- /dev/null +++ b/artipie-ui/src/api/users.ts @@ -0,0 +1,34 @@ +import { getApiClient } from './client' +import type { PaginatedResponse, User } from '@/types' + +export async function listUsers(params: { + page?: number; size?: number; q?: string; status?: string +} = {}): Promise> { + const { data } = await getApiClient().get('/users', { params }) + return data +} + +export async function getUser(name: string): Promise { + const { data } = await getApiClient().get(`/users/${name}`) + return data +} + +export async function putUser(name: string, body: Record): Promise { + await getApiClient().put(`/users/${name}`, body) +} + +export async function deleteUser(name: string): Promise { + await getApiClient().delete(`/users/${name}`) +} + +export async function changePassword(name: string, oldPass: string, newPass: string): Promise { + await getApiClient().post(`/users/${name}/password`, { old_pass: oldPass, new_pass: newPass, new_type: 'plain' }) +} + +export async function enableUser(name: string): Promise { + await getApiClient().post(`/users/${name}/enable`) +} + +export async function disableUser(name: string): Promise { + await getApiClient().post(`/users/${name}/disable`) +} diff --git a/artipie-ui/src/assets/main.css b/artipie-ui/src/assets/main.css new file mode 100644 index 000000000..7a34d44d6 --- /dev/null +++ b/artipie-ui/src/assets/main.css @@ -0,0 +1,57 @@ +@import "tailwindcss"; +@import "primeicons/primeicons.css"; + +:root { + --app-sidebar-width: 240px; + --app-sidebar-collapsed-width: 64px; + --app-header-height: 56px; +} + +body { + margin: 0; + font-family: 'Inter', system-ui, -apple-system, sans-serif; + -webkit-font-smoothing: antialiased; +} + +/* Dark-only theme */ +.dark .p-card { + background: #1f2937; + border: 1px solid #374151; + border-radius: 14px; +} + +.dark .p-inputtext:focus { + border-color: #d97b2a !important; + box-shadow: 0 0 0 3px rgba(217,123,42,0.1) !important; +} + +.dark .p-button:not(.p-button-secondary):not(.p-button-text):not(.p-button-outlined):not(.p-button-danger):not(.p-button-success):not(.p-button-warn):not(.p-button-info) { + background: #d97b2a; + border-color: #d97b2a; +} +.dark .p-button:not(.p-button-secondary):not(.p-button-text):not(.p-button-outlined):not(.p-button-danger):not(.p-button-success):not(.p-button-warn):not(.p-button-info):hover { + background: #c06a1f; + border-color: #c06a1f; +} + +.dark .p-datatable .p-datatable-tbody > tr { background: #111827; } +.dark .p-datatable .p-datatable-tbody > tr:nth-child(even) { background: rgba(255,255,255,0.02); } +.dark .p-datatable .p-datatable-tbody > tr:hover { background: rgba(217,123,42,0.04) !important; } + +.dark .p-dialog .p-dialog-content, +.dark .p-dialog .p-dialog-header, +.dark .p-dialog .p-dialog-footer { background: #1e293b; } +.dark .p-dialog .p-dialog-header { border-bottom-color: #374151; } +.dark .p-dialog .p-dialog-footer { border-top-color: #374151; } + +.dark .p-breadcrumb { background: transparent; border: none; padding: 0; } +.dark .p-paginator { background: transparent; border: none; } + +/* Tooltip: prevent line breaks */ +.p-tooltip .p-tooltip-text { white-space: nowrap !important; } + +/* Scrollbar */ +::-webkit-scrollbar { width: 6px; height: 6px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: rgba(255,255,255,0.1); border-radius: 3px; } +::-webkit-scrollbar-thumb:hover { background: rgba(255,255,255,0.2); } diff --git a/artipie-ui/src/components/common/HealthIndicator.vue b/artipie-ui/src/components/common/HealthIndicator.vue new file mode 100644 index 000000000..b8144311a --- /dev/null +++ b/artipie-ui/src/components/common/HealthIndicator.vue @@ -0,0 +1,39 @@ + + + diff --git a/artipie-ui/src/components/common/RepoTypeBadge.vue b/artipie-ui/src/components/common/RepoTypeBadge.vue new file mode 100644 index 000000000..a0010e81d --- /dev/null +++ b/artipie-ui/src/components/common/RepoTypeBadge.vue @@ -0,0 +1,32 @@ + + + diff --git a/artipie-ui/src/components/layout/AppHeader.vue b/artipie-ui/src/components/layout/AppHeader.vue new file mode 100644 index 000000000..1c3f67bd4 --- /dev/null +++ b/artipie-ui/src/components/layout/AppHeader.vue @@ -0,0 +1,147 @@ + + + diff --git a/artipie-ui/src/components/layout/AppLayout.vue b/artipie-ui/src/components/layout/AppLayout.vue new file mode 100644 index 000000000..0d86569a4 --- /dev/null +++ b/artipie-ui/src/components/layout/AppLayout.vue @@ -0,0 +1,81 @@ + + + + + diff --git a/artipie-ui/src/components/layout/AppSidebar.vue b/artipie-ui/src/components/layout/AppSidebar.vue new file mode 100644 index 000000000..7c1c3257e --- /dev/null +++ b/artipie-ui/src/components/layout/AppSidebar.vue @@ -0,0 +1,119 @@ + + + diff --git a/artipie-ui/src/components/layout/__tests__/AppLayout.test.ts b/artipie-ui/src/components/layout/__tests__/AppLayout.test.ts new file mode 100644 index 000000000..f2cf2b955 --- /dev/null +++ b/artipie-ui/src/components/layout/__tests__/AppLayout.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { mount } from '@vue/test-utils' +import { createPinia, setActivePinia } from 'pinia' +import AppLayout from '../AppLayout.vue' +import PrimeVue from 'primevue/config' +import Aura from '@primevue/themes/aura' + +describe('AppLayout', () => { + beforeEach(() => { + setActivePinia(createPinia()) + }) + + it('renders header, sidebar, and main content area', () => { + const wrapper = mount(AppLayout, { + global: { + plugins: [[PrimeVue, { theme: { preset: Aura } }]], + stubs: ['router-link', 'router-view', 'AppHeader', 'AppSidebar', 'HealthIndicator'], + }, + slots: { default: '

Page Content
' }, + }) + expect(wrapper.find('[data-testid="app-layout"]').exists()).toBe(true) + }) +}) diff --git a/artipie-ui/src/composables/__tests__/usePagination.test.ts b/artipie-ui/src/composables/__tests__/usePagination.test.ts new file mode 100644 index 000000000..f4351e101 --- /dev/null +++ b/artipie-ui/src/composables/__tests__/usePagination.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect, vi } from 'vitest' +import { usePagination } from '../usePagination' + +describe('usePagination', () => { + it('starts at page 0 with default size', () => { + const mockFn = vi.fn().mockResolvedValue({ items: [], page: 0, size: 20, total: 0, hasMore: false }) + const { page, size } = usePagination(mockFn) + expect(page.value).toBe(0) + expect(size.value).toBe(20) + }) + + it('fetches data and updates state', async () => { + const mockFn = vi.fn().mockResolvedValue({ + items: [{ name: 'a' }], page: 0, size: 20, total: 1, hasMore: false, + }) + const { items, total, fetch } = usePagination(mockFn) + await fetch() + expect(items.value).toEqual([{ name: 'a' }]) + expect(total.value).toBe(1) + expect(mockFn).toHaveBeenCalledWith({ page: 0, size: 20 }) + }) + + it('advances to next page', async () => { + const mockFn = vi.fn().mockResolvedValue({ + items: [{ name: 'b' }], page: 1, size: 20, total: 40, hasMore: true, + }) + const { page, nextPage } = usePagination(mockFn) + page.value = 0 + await nextPage() + expect(page.value).toBe(1) + }) +}) diff --git a/artipie-ui/src/composables/useConfirmDelete.ts b/artipie-ui/src/composables/useConfirmDelete.ts new file mode 100644 index 000000000..9f861e89f --- /dev/null +++ b/artipie-ui/src/composables/useConfirmDelete.ts @@ -0,0 +1,29 @@ +import { ref } from 'vue' + +export function useConfirmDelete() { + const visible = ref(false) + const targetName = ref('') + let resolvePromise: ((confirmed: boolean) => void) | null = null + + function confirm(name: string): Promise { + targetName.value = name + visible.value = true + return new Promise((resolve) => { + resolvePromise = resolve + }) + } + + function accept() { + visible.value = false + resolvePromise?.(true) + resolvePromise = null + } + + function reject() { + visible.value = false + resolvePromise?.(false) + resolvePromise = null + } + + return { visible, targetName, confirm, accept, reject } +} diff --git a/artipie-ui/src/composables/useCursorPagination.ts b/artipie-ui/src/composables/useCursorPagination.ts new file mode 100644 index 000000000..db5e71049 --- /dev/null +++ b/artipie-ui/src/composables/useCursorPagination.ts @@ -0,0 +1,47 @@ +import { ref, type Ref } from 'vue' +import type { CursorResponse } from '@/types' + +export function useCursorPagination( + apiFn: (params: { marker?: string; limit?: number }) => Promise>, + defaultLimit = 100, +) { + const items: Ref = ref([]) + const marker = ref(null) + const hasMore = ref(false) + const loading = ref(false) + const error = ref(null) + + async function fetch() { + loading.value = true + error.value = null + items.value = [] + marker.value = null + try { + const resp = await apiFn({ limit: defaultLimit }) + items.value = resp.items + marker.value = resp.marker + hasMore.value = resp.hasMore + } catch (e: unknown) { + error.value = e instanceof Error ? e.message : 'Fetch failed' + } finally { + loading.value = false + } + } + + async function loadMore() { + if (!marker.value) return + loading.value = true + try { + const resp = await apiFn({ marker: marker.value, limit: defaultLimit }) + items.value.push(...resp.items) + marker.value = resp.marker + hasMore.value = resp.hasMore + } catch (e: unknown) { + error.value = e instanceof Error ? e.message : 'Load more failed' + } finally { + loading.value = false + } + } + + return { items, marker, hasMore, loading, error, fetch, loadMore } +} diff --git a/artipie-ui/src/composables/useDebouncedSearch.ts b/artipie-ui/src/composables/useDebouncedSearch.ts new file mode 100644 index 000000000..27d379d56 --- /dev/null +++ b/artipie-ui/src/composables/useDebouncedSearch.ts @@ -0,0 +1,23 @@ +import { ref, watch } from 'vue' + +export function useDebouncedSearch( + searchFn: (query: string) => Promise, + delay = 300, +) { + const query = ref('') + let timeout: ReturnType | null = null + + watch(query, (val) => { + if (timeout) clearTimeout(timeout) + timeout = setTimeout(() => { + searchFn(val) + }, delay) + }) + + function clear() { + query.value = '' + if (timeout) clearTimeout(timeout) + } + + return { query, clear } +} diff --git a/artipie-ui/src/composables/usePagination.ts b/artipie-ui/src/composables/usePagination.ts new file mode 100644 index 000000000..03ac8fa65 --- /dev/null +++ b/artipie-ui/src/composables/usePagination.ts @@ -0,0 +1,49 @@ +import { ref, type Ref } from 'vue' +import type { PaginatedResponse } from '@/types' + +export function usePagination( + apiFn: (params: { page: number; size: number }) => Promise>, + defaultSize = 20, +) { + const items: Ref = ref([]) + const page = ref(0) + const size = ref(defaultSize) + const total = ref(0) + const hasMore = ref(false) + const loading = ref(false) + const error = ref(null) + + async function fetch() { + loading.value = true + error.value = null + try { + const resp = await apiFn({ page: page.value, size: size.value }) + items.value = resp.items + total.value = resp.total + hasMore.value = resp.hasMore + } catch (e: unknown) { + error.value = e instanceof Error ? e.message : 'Fetch failed' + } finally { + loading.value = false + } + } + + async function nextPage() { + page.value++ + await fetch() + } + + async function prevPage() { + if (page.value > 0) { + page.value-- + await fetch() + } + } + + async function goToPage(p: number) { + page.value = p + await fetch() + } + + return { items, page, size, total, hasMore, loading, error, fetch, nextPage, prevPage, goToPage } +} diff --git a/artipie-ui/src/composables/usePermission.ts b/artipie-ui/src/composables/usePermission.ts new file mode 100644 index 000000000..72f594e1d --- /dev/null +++ b/artipie-ui/src/composables/usePermission.ts @@ -0,0 +1,13 @@ +import { computed } from 'vue' +import { useAuthStore } from '@/stores/auth' + +export function usePermission(resource: string) { + const auth = useAuthStore() + const hasPermission = computed(() => { + if (!auth.user) return false + const perms = auth.user.permissions ?? {} + const key = `api_${resource}_permissions` + return perms[key] === true + }) + return { hasPermission } +} diff --git a/artipie-ui/src/main.ts b/artipie-ui/src/main.ts new file mode 100644 index 000000000..44ac29e0b --- /dev/null +++ b/artipie-ui/src/main.ts @@ -0,0 +1,65 @@ +import { createApp } from 'vue' +import { createPinia } from 'pinia' +import PrimeVue from 'primevue/config' +import Aura from '@primevue/themes/aura' +import { definePreset } from '@primevue/themes' +import ToastService from 'primevue/toastservice' +import ConfirmationService from 'primevue/confirmationservice' +import Tooltip from 'primevue/tooltip' +import App from './App.vue' +import { useConfigStore } from './stores/config' +import { initApiClient } from './api/client' +import { createAppRouter } from './router' +import type { RuntimeConfig } from './types' + +import './assets/main.css' + +const Auto1Preset = definePreset(Aura, { + semantic: { + primary: { + 50: '#fdf5ec', + 100: '#fae6cf', + 200: '#f5cda0', + 300: '#edb06d', + 400: '#e8944a', + 500: '#d97b2a', + 600: '#c06a1f', + 700: '#a0571a', + 800: '#804518', + 900: '#673816', + 950: '#3a1d0a', + }, + }, +}) + +async function bootstrap() { + const resp = await fetch('/config.json') + const config: RuntimeConfig = await resp.json() + + const app = createApp(App) + const pinia = createPinia() + app.use(pinia) + + const configStore = useConfigStore() + configStore.loadConfig(config) + initApiClient(configStore.apiBaseUrl) + + app.use(PrimeVue, { + theme: { + preset: Auto1Preset, + options: { + darkModeSelector: '.dark', + }, + }, + }) + app.use(ToastService) + app.use(ConfirmationService) + app.directive('tooltip', Tooltip) + + const router = createAppRouter() + app.use(router) + + app.mount('#app') +} + +bootstrap() diff --git a/artipie-ui/src/router/__tests__/index.test.ts b/artipie-ui/src/router/__tests__/index.test.ts new file mode 100644 index 000000000..6edff6bae --- /dev/null +++ b/artipie-ui/src/router/__tests__/index.test.ts @@ -0,0 +1,24 @@ +import { describe, it, expect } from 'vitest' +import { routes } from '../index' + +describe('Router', () => { + it('has login route', () => { + expect(routes.find((r) => r.path === '/login')).toBeDefined() + }) + + it('has dashboard route at /', () => { + expect(routes.find((r) => r.path === '/')).toBeDefined() + }) + + it('has admin routes', () => { + const adminRoutes = routes.filter((r) => r.path.startsWith('/admin')) + expect(adminRoutes.length).toBeGreaterThanOrEqual(5) + }) + + it('all admin routes have admin meta', () => { + const adminRoutes = routes.filter((r) => r.path.startsWith('/admin')) + adminRoutes.forEach((r) => { + expect(r.meta?.requiresAdmin).toBe(true) + }) + }) +}) diff --git a/artipie-ui/src/router/index.ts b/artipie-ui/src/router/index.ts new file mode 100644 index 000000000..67e0f96b6 --- /dev/null +++ b/artipie-ui/src/router/index.ts @@ -0,0 +1,149 @@ +import { createRouter, createWebHistory, type RouteRecordRaw } from 'vue-router' +import { useAuthStore } from '@/stores/auth' + +export const routes: RouteRecordRaw[] = [ + { + path: '/login', + name: 'login', + component: () => import('@/views/auth/LoginView.vue'), + meta: { public: true }, + }, + { + path: '/auth/callback', + name: 'oauth-callback', + component: () => import('@/views/auth/OAuthCallbackView.vue'), + meta: { public: true }, + }, + { + path: '/', + name: 'dashboard', + component: () => import('@/views/dashboard/DashboardView.vue'), + }, + { + path: '/repositories', + name: 'repositories', + component: () => import('@/views/repos/RepoListView.vue'), + }, + { + path: '/repositories/:name', + name: 'repo-detail', + component: () => import('@/views/repos/RepoDetailView.vue'), + props: true, + }, + { + path: '/search', + name: 'search', + component: () => import('@/views/search/SearchView.vue'), + }, + { + path: '/profile', + name: 'profile', + component: () => import('@/views/profile/ProfileView.vue'), + }, + // Admin routes — each guarded by its specific permission + { + path: '/admin/repositories', + name: 'admin-repositories', + component: () => import('@/views/admin/RepoManagementView.vue'), + meta: { requiredPermission: 'api_repository_permissions' }, + }, + { + path: '/admin/repositories/create', + name: 'admin-repo-create', + component: () => import('@/views/admin/RepoCreateView.vue'), + meta: { requiredPermission: 'api_repository_permissions' }, + }, + { + path: '/admin/repositories/:name/edit', + name: 'admin-repo-edit', + component: () => import('@/views/admin/RepoEditView.vue'), + props: true, + meta: { requiredPermission: 'api_repository_permissions' }, + }, + { + path: '/admin/users', + name: 'admin-users', + component: () => import('@/views/admin/UserListView.vue'), + meta: { requiredPermission: 'api_user_permissions' }, + }, + { + path: '/admin/users/:name', + name: 'admin-user-detail', + component: () => import('@/views/admin/UserDetailView.vue'), + props: true, + meta: { requiredPermission: 'api_user_permissions' }, + }, + { + path: '/admin/roles', + name: 'admin-roles', + component: () => import('@/views/admin/RoleListView.vue'), + meta: { requiredPermission: 'api_role_permissions' }, + }, + { + path: '/admin/roles/:name', + name: 'admin-role-detail', + component: () => import('@/views/admin/RoleDetailView.vue'), + props: true, + meta: { requiredPermission: 'api_role_permissions' }, + }, + { + path: '/admin/storage', + name: 'admin-storage', + component: () => import('@/views/admin/StorageAliasView.vue'), + meta: { requiredPermission: 'api_alias_permissions' }, + }, + { + path: '/admin/cooldown', + name: 'admin-cooldown', + component: () => import('@/views/admin/CooldownView.vue'), + meta: { requiredPermission: 'api_cooldown_permissions' }, + }, + { + path: '/admin/settings', + name: 'admin-settings', + component: () => import('@/views/admin/SettingsView.vue'), + meta: { requiresAdmin: true }, + }, + { + path: '/:pathMatch(.*)*', + name: 'not-found', + component: () => import('@/views/NotFoundView.vue'), + }, +] + +export function createAppRouter() { + const router = createRouter({ + history: createWebHistory(), + routes, + }) + + router.beforeEach(async (to) => { + if (to.meta.public) return true + const auth = useAuthStore() + if (!auth.isAuthenticated) { + return { name: 'login', query: { redirect: to.fullPath } } + } + // On page refresh the token survives in sessionStorage but user data is lost. + // Re-fetch it so permissions, admin status, and profile info are available. + if (!auth.user) { + await auth.fetchUser() + // fetchUser calls logout() on 401 (expired JWT), which clears the token. + if (!auth.isAuthenticated) { + return { name: 'login', query: { redirect: to.fullPath } } + } + } + if (to.meta.requiresAdmin && !auth.isAdmin) { + return { name: 'dashboard' } + } + const requiredPerm = to.meta.requiredPermission as string | undefined + if (requiredPerm) { + const val = auth.user?.permissions?.[requiredPerm] + if (!Array.isArray(val) || val.length === 0) { + return { name: 'dashboard' } + } + } + return true + }) + + return router +} diff --git a/artipie-ui/src/stores/__tests__/config.test.ts b/artipie-ui/src/stores/__tests__/config.test.ts new file mode 100644 index 000000000..9b468dac9 --- /dev/null +++ b/artipie-ui/src/stores/__tests__/config.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useConfigStore } from '../config' + +describe('configStore', () => { + beforeEach(() => { + setActivePinia(createPinia()) + }) + + it('has default values', () => { + const store = useConfigStore() + expect(store.apiBaseUrl).toBe('/api/v1') + expect(store.defaultPageSize).toBe(20) + }) + + it('loads runtime config', () => { + const store = useConfigStore() + store.loadConfig({ + apiBaseUrl: '/custom/api', + grafanaUrl: 'http://grafana:3000', + appTitle: 'Test', + defaultPageSize: 50, + }) + expect(store.apiBaseUrl).toBe('/custom/api') + expect(store.appTitle).toBe('Test') + expect(store.defaultPageSize).toBe(50) + }) +}) diff --git a/artipie-ui/src/stores/__tests__/theme.test.ts b/artipie-ui/src/stores/__tests__/theme.test.ts new file mode 100644 index 000000000..fb37ea26f --- /dev/null +++ b/artipie-ui/src/stores/__tests__/theme.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useThemeStore } from '../theme' + +describe('themeStore', () => { + beforeEach(() => { + setActivePinia(createPinia()) + localStorage.clear() + }) + + it('defaults to light mode', () => { + const store = useThemeStore() + expect(store.mode).toBe('light') + }) + + it('toggles to dark mode', () => { + const store = useThemeStore() + store.toggle() + expect(store.mode).toBe('dark') + }) + + it('persists to localStorage', () => { + const store = useThemeStore() + store.toggle() + expect(localStorage.getItem('theme')).toBe('dark') + }) + + it('isDark computed', () => { + const store = useThemeStore() + expect(store.isDark).toBe(false) + store.toggle() + expect(store.isDark).toBe(true) + }) +}) diff --git a/artipie-ui/src/stores/auth.ts b/artipie-ui/src/stores/auth.ts new file mode 100644 index 000000000..e0ac56303 --- /dev/null +++ b/artipie-ui/src/stores/auth.ts @@ -0,0 +1,108 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import type { UserInfo, AuthProvider } from '@/types' +import * as authApi from '@/api/auth' + +export const useAuthStore = defineStore('auth', () => { + const token = ref(sessionStorage.getItem('jwt')) + const user = ref(null) + const providers = ref([]) + const loading = ref(false) + + const isAuthenticated = computed(() => !!token.value) + const isAdmin = computed(() => { + if (!user.value) return false + const perms = user.value.permissions ?? {} + return hasPerm(perms, 'api_user_permissions') + || hasPerm(perms, 'api_role_permissions') + }) + const username = computed(() => user.value?.name ?? '') + + async function login(uname: string, password: string) { + loading.value = true + try { + const resp = await authApi.login(uname, password) + token.value = resp.token + sessionStorage.setItem('jwt', resp.token) + await fetchUser() + } finally { + loading.value = false + } + } + + async function fetchUser() { + if (!token.value) return + try { + user.value = await authApi.getMe() + } catch { + logout() + } + } + + async function fetchProviders() { + try { + const resp = await authApi.getProviders() + providers.value = resp.providers + } catch { + providers.value = [] + } + } + + async function ssoRedirect(providerName: string) { + const callbackUrl = window.location.origin + '/auth/callback' + const resp = await authApi.getProviderRedirect(providerName, callbackUrl) + sessionStorage.setItem('sso_state', resp.state) + sessionStorage.setItem('sso_provider', providerName) + sessionStorage.setItem('sso_callback_url', callbackUrl) + window.location.href = resp.url + } + + async function handleOAuthCallback(code: string, state: string) { + const savedState = sessionStorage.getItem('sso_state') + const provider = sessionStorage.getItem('sso_provider') + const callbackUrl = sessionStorage.getItem('sso_callback_url') + sessionStorage.removeItem('sso_state') + sessionStorage.removeItem('sso_provider') + sessionStorage.removeItem('sso_callback_url') + if (!savedState || savedState !== state) { + throw new Error('Invalid OAuth state — possible CSRF attack') + } + if (!provider || !callbackUrl) { + throw new Error('Missing SSO session data') + } + loading.value = true + try { + const resp = await authApi.exchangeOAuthCode(code, provider, callbackUrl) + token.value = resp.token + sessionStorage.setItem('jwt', resp.token) + await fetchUser() + } finally { + loading.value = false + } + } + + function logout() { + token.value = null + user.value = null + sessionStorage.removeItem('jwt') + } + + function hasPerm(perms: Record, key: string): boolean { + const val = perms[key] + return Array.isArray(val) && val.length > 0 + } + + function hasAction(key: string, action: string): boolean { + const perms = user.value?.permissions ?? {} + const val = perms[key] + return Array.isArray(val) && (val.includes(action) || val.includes('*')) + } + + return { + token, user, providers, loading, + isAuthenticated, isAdmin, username, + login, logout, fetchUser, fetchProviders, + ssoRedirect, handleOAuthCallback, + hasAction, + } +}) diff --git a/artipie-ui/src/stores/config.ts b/artipie-ui/src/stores/config.ts new file mode 100644 index 000000000..27c8bdaad --- /dev/null +++ b/artipie-ui/src/stores/config.ts @@ -0,0 +1,19 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import type { RuntimeConfig } from '@/types' + +export const useConfigStore = defineStore('config', () => { + const apiBaseUrl = ref('/api/v1') + const grafanaUrl = ref('') + const appTitle = ref('Artipie') + const defaultPageSize = ref(20) + + function loadConfig(cfg: RuntimeConfig) { + apiBaseUrl.value = cfg.apiBaseUrl + grafanaUrl.value = cfg.grafanaUrl + appTitle.value = cfg.appTitle + defaultPageSize.value = cfg.defaultPageSize + } + + return { apiBaseUrl, grafanaUrl, appTitle, defaultPageSize, loadConfig } +}) diff --git a/artipie-ui/src/stores/notifications.ts b/artipie-ui/src/stores/notifications.ts new file mode 100644 index 000000000..95d3970e3 --- /dev/null +++ b/artipie-ui/src/stores/notifications.ts @@ -0,0 +1,34 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' + +export interface Toast { + id: number + severity: 'success' | 'error' | 'warn' | 'info' + summary: string + detail?: string + life?: number +} + +let nextId = 0 + +export const useNotificationStore = defineStore('notifications', () => { + const toasts = ref([]) + + function add(severity: Toast['severity'], summary: string, detail?: string) { + const id = nextId++ + const life = severity === 'error' ? 5000 : 3000 + toasts.value.push({ id, severity, summary, detail, life }) + setTimeout(() => remove(id), life) + } + + function remove(id: number) { + toasts.value = toasts.value.filter((t) => t.id !== id) + } + + function success(summary: string, detail?: string) { add('success', summary, detail) } + function error(summary: string, detail?: string) { add('error', summary, detail) } + function warn(summary: string, detail?: string) { add('warn', summary, detail) } + function info(summary: string, detail?: string) { add('info', summary, detail) } + + return { toasts, add, remove, success, error, warn, info } +}) diff --git a/artipie-ui/src/stores/theme.ts b/artipie-ui/src/stores/theme.ts new file mode 100644 index 000000000..f2a32884a --- /dev/null +++ b/artipie-ui/src/stores/theme.ts @@ -0,0 +1,25 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' + +function detectInitialTheme(): 'dark' { + return 'dark' +} + +export const useThemeStore = defineStore('theme', () => { + const mode = ref<'dark'>('dark') + const isDark = computed(() => true) + + function toggle() { + // Dark mode only + } + + function applyTheme() { + if (mode.value === 'dark') { + document.documentElement.classList.add('dark') + } else { + document.documentElement.classList.remove('dark') + } + } + + return { mode, isDark, toggle, applyTheme } +}) diff --git a/artipie-ui/src/types/index.ts b/artipie-ui/src/types/index.ts new file mode 100644 index 000000000..7ae189c74 --- /dev/null +++ b/artipie-ui/src/types/index.ts @@ -0,0 +1,238 @@ +// Pagination +export interface PaginatedResponse { + items: T[] + page: number + size: number + total: number + hasMore: boolean +} + +export interface CursorResponse { + items: T[] + marker: string | null + hasMore: boolean +} + +export interface ApiError { + error: string + message: string + status: number +} + +// Auth +export interface AuthProvider { + type: string + enabled: boolean + priority?: number +} + +export interface AuthProvidersResponse { + providers: AuthProvider[] +} + +export interface TokenResponse { + token: string +} + +export interface UserInfo { + name: string + context: string + permissions: Record + can_delete_artifacts?: boolean + email?: string + groups?: string[] +} + +export interface RepoListItem { + name: string + type: string +} + +// Repository +export interface Repository { + name: string + type: string + config: Record + enabled?: boolean + created_at?: string + updated_at?: string +} + +export interface RepoMember { + name: string + type: string + enabled: boolean +} + +// Artifact +export interface TreeEntry { + name: string + path: string + type: 'file' | 'directory' + size?: number + modified?: string +} + +export interface ArtifactDetail { + path: string + name: string + size: number + modified?: string + checksums?: Record +} + +export interface PullInstructions { + type: string + instructions: string[] +} + +// Search +export interface SearchResult { + repo_type: string + repo_name: string + artifact_path: string + artifact_name?: string + version?: string + size: number + created_at?: string + owner?: string +} + +export interface LocateResponse { + repositories: string[] + count: number +} + +export interface ReindexResponse { + status: string + message: string +} + +// Dashboard +export interface DashboardStats { + repo_count: number + artifact_count: number + total_storage: number | string + blocked_count: number + top_repos?: { name: string; type: string; artifact_count: number; size?: number }[] +} + +export interface ReposByType { + types: Record +} + +// User +export interface User { + name: string + email?: string + enabled?: boolean + auth_provider?: string + roles?: string[] + created_at?: string +} + +// Role +export interface Role { + name: string + permissions: Record + enabled?: boolean + created_at?: string +} + +// Storage Alias +export interface StorageAlias { + name: string + repo_name?: string | null + config: Record + type?: string +} + +// Cooldown +export interface CooldownRepo { + name: string + type: string + cooldown: string + active_blocks?: number +} + +export interface BlockedArtifact { + package_name: string + version: string + repo: string + repo_type: string + reason: string + blocked_date: string + blocked_until: string + remaining_hours: number +} + +// Cooldown config +export interface CooldownConfig { + enabled: boolean + minimum_allowed_age: string + repo_types?: Record +} + +// Settings +export interface Settings { + port: number + version: string + prefixes: string[] + jwt?: { + expires: boolean + expiry_seconds: number + } + http_client?: { + proxy_timeout: number + connection_timeout: number + idle_timeout: number + follow_redirects: boolean + connection_acquire_timeout: number + max_connections_per_destination: number + max_requests_queued_per_destination: number + } + http_server?: { + request_timeout: string + } + metrics?: { + enabled: boolean + endpoint?: string + port?: number + jvm: boolean + http: boolean + storage: boolean + } + cooldown?: { + enabled: boolean + minimum_allowed_age: string + } + credentials?: Array<{ + id: number + type: string + priority: number + enabled: boolean + config?: Record + }> + database?: { + configured: boolean + } + caches?: { + valkey_configured: boolean + } +} + +// Health +export interface HealthResponse { + status: string +} + +// Runtime config (config.json) +export interface RuntimeConfig { + apiBaseUrl: string + grafanaUrl: string + appTitle: string + defaultPageSize: number +} diff --git a/artipie-ui/src/utils/repoTypes.ts b/artipie-ui/src/utils/repoTypes.ts new file mode 100644 index 000000000..0c2fbb7ab --- /dev/null +++ b/artipie-ui/src/utils/repoTypes.ts @@ -0,0 +1,184 @@ +/** + * Single source of truth for repository type display. + * + * Each technology gets a unique color derived from its real logo/brand: + * Maven — #C71A36 (red, from Maven logo) + * Docker — #2496ED (blue, from Docker logo) + * npm — #CB3837 (dark red, from npm logo) → shifted to #CC4444 to differ from Maven + * PyPI — #3775A9 (blue) → shifted to #0C7C59 teal-green to differ from Docker + * Helm — #0F1689 (navy, from Helm logo) → #277CC5 medium blue + * Go — #00ADD8 (cyan, from Go logo) + * NuGet — #6F42C1 (purple, from NuGet logo) + * Debian — #D70751 (magenta, from Debian logo) + * RPM — #EE0000 (red) → #E06020 to differ + * Conda — #44A833 (green, from Anaconda logo) + * RubyGems— #E9573F (ruby red, from RubyGems logo) → #CC6699 pink + * Conan — #6699CB (steel blue, from Conan logo) + * Hex — #6E4A7E (purple, from Hex logo) + * PHP — #777BB4 (lavender, from PHP logo) + * File — #8B8B8B (gray) + * Binary — #6B7280 (gray) + */ + +interface TechInfo { + label: string + icon: string + /** Hex color for the technology — used for dot/bar/accent */ + color: string + /** Tailwind bg for icon container */ + bgClass: string + /** Tailwind text for icon */ + textClass: string +} + +const TECH_MAP: Record = { + maven: { label: 'Maven', icon: 'pi pi-box', color: '#C71A36', bgClass: 'bg-[#C71A36]/10', textClass: 'text-[#C71A36]' }, + gradle: { label: 'Gradle', icon: 'pi pi-box', color: '#02303A', bgClass: 'bg-[#02303A]/15', textClass: 'text-[#5A9A8C]' }, + docker: { label: 'Docker', icon: 'pi pi-server', color: '#2496ED', bgClass: 'bg-[#2496ED]/10', textClass: 'text-[#2496ED]' }, + npm: { label: 'npm', icon: 'pi pi-box', color: '#CC4444', bgClass: 'bg-[#CC4444]/10', textClass: 'text-[#CC4444]' }, + pypi: { label: 'PyPI', icon: 'pi pi-box', color: '#0C7C59', bgClass: 'bg-[#0C7C59]/10', textClass: 'text-[#0C7C59]' }, + helm: { label: 'Helm', icon: 'pi pi-box', color: '#277CC5', bgClass: 'bg-[#277CC5]/10', textClass: 'text-[#277CC5]' }, + go: { label: 'Go', icon: 'pi pi-box', color: '#00ADD8', bgClass: 'bg-[#00ADD8]/10', textClass: 'text-[#00ADD8]' }, + nuget: { label: 'NuGet', icon: 'pi pi-box', color: '#6F42C1', bgClass: 'bg-[#6F42C1]/10', textClass: 'text-[#6F42C1]' }, + debian: { label: 'Debian', icon: 'pi pi-box', color: '#D70751', bgClass: 'bg-[#D70751]/10', textClass: 'text-[#D70751]' }, + deb: { label: 'Debian', icon: 'pi pi-box', color: '#D70751', bgClass: 'bg-[#D70751]/10', textClass: 'text-[#D70751]' }, + rpm: { label: 'RPM', icon: 'pi pi-box', color: '#E06020', bgClass: 'bg-[#E06020]/10', textClass: 'text-[#E06020]' }, + conda: { label: 'Conda', icon: 'pi pi-box', color: '#44A833', bgClass: 'bg-[#44A833]/10', textClass: 'text-[#44A833]' }, + gem: { label: 'RubyGems', icon: 'pi pi-box', color: '#CC6699', bgClass: 'bg-[#CC6699]/10', textClass: 'text-[#CC6699]' }, + conan: { label: 'Conan', icon: 'pi pi-box', color: '#6699CB', bgClass: 'bg-[#6699CB]/10', textClass: 'text-[#6699CB]' }, + hex: { label: 'Hex', icon: 'pi pi-box', color: '#6E4A7E', bgClass: 'bg-[#6E4A7E]/10', textClass: 'text-[#6E4A7E]' }, + php: { label: 'PHP', icon: 'pi pi-box', color: '#777BB4', bgClass: 'bg-[#777BB4]/10', textClass: 'text-[#777BB4]' }, + file: { label: 'File', icon: 'pi pi-folder', color: '#8B8B8B', bgClass: 'bg-gray-500/10', textClass: 'text-gray-400' }, + binary: { label: 'Binary', icon: 'pi pi-file', color: '#6B7280', bgClass: 'bg-gray-500/10', textClass: 'text-gray-400' }, +} + +const SUBTYPE_LABELS: Record = { + proxy: 'Proxy', + group: 'Group', +} + +const DEFAULT_TECH: TechInfo = { + label: 'Unknown', icon: 'pi pi-box', color: '#6B7280', + bgClass: 'bg-gray-500/10', textClass: 'text-gray-400', +} + +/** Extract base type and subtype from raw repo type. + * Handles both hyphens (npm-proxy) and underscores (npm_proxy). */ +function parseType(raw: string): { base: string; subtype: string | null } { + const lower = (raw ?? 'unknown').toLowerCase() + for (const sfx of Object.keys(SUBTYPE_LABELS)) { + if (lower.endsWith(`-${sfx}`) || lower.endsWith(`_${sfx}`)) { + return { base: lower.slice(0, -(sfx.length + 1)), subtype: sfx } + } + } + return { base: lower, subtype: null } +} + +/** Get technology info. */ +export function getTechInfo(raw: string): TechInfo { + const { base } = parseType(raw) + return TECH_MAP[base] ?? DEFAULT_TECH +} + +/** Technology label (e.g. "npm", "Maven", "Docker"). */ +export function techLabel(raw: string): string { + return getTechInfo(raw).label +} + +/** Subtype label (e.g. "Proxy", "Group", "Local"). */ +export function subtypeLabel(raw: string): string { + const { subtype } = parseType(raw) + return subtype ? SUBTYPE_LABELS[subtype] ?? 'Local' : 'Local' +} + +/** Full label (e.g. "Maven Proxy"). */ +export function repoTypeLabel(raw: string): string { + const tech = techLabel(raw) + const sub = subtypeLabel(raw) + return sub ? `${tech} ${sub}` : tech +} + +/** Base label only (for filters). */ +export function repoTypeBaseLabel(raw: string): string { + return techLabel(raw) +} + +/** Icon class. */ +export function repoTypeIcon(raw: string): string { + return getTechInfo(raw).icon +} + +/** Tailwind color classes for icon container: "bg-xxx/10 text-xxx". */ +export function repoTypeColorClass(raw: string): string { + const info = getTechInfo(raw) + return `${info.bgClass} ${info.textClass}` +} + +/** Hex color for the technology (for bars, dots, inline styles). */ +export function repoTypeColor(raw: string): string { + return getTechInfo(raw).color +} + +/** PrimeVue Tag severity — not ideal for brand colors, use sparingly. */ +export function repoTypeSeverity(raw: string): string { + const { subtype } = parseType(raw) + if (subtype === 'proxy') return 'info' + if (subtype === 'group') return 'warn' + return 'secondary' +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1) +} + +/** Filter dropdown options. */ +export const REPO_TYPE_FILTERS = [ + { label: 'All Types', value: null }, + { label: 'Maven', value: 'maven' }, + { label: 'Docker', value: 'docker' }, + { label: 'npm', value: 'npm' }, + { label: 'PyPI', value: 'pypi' }, + { label: 'Helm', value: 'helm' }, + { label: 'Go', value: 'go' }, + { label: 'NuGet', value: 'nuget' }, + { label: 'Debian', value: 'debian' }, + { label: 'RPM', value: 'rpm' }, + { label: 'Conda', value: 'conda' }, + { label: 'RubyGems', value: 'gem' }, + { label: 'Conan', value: 'conan' }, + { label: 'Hex', value: 'hex' }, + { label: 'PHP', value: 'php' }, + { label: 'File', value: 'file' }, + { label: 'Binary', value: 'binary' }, +] as const + +/** Create form options. */ +export const REPO_TYPE_CREATE_OPTIONS = [ + { label: 'Maven (Local)', value: 'maven' }, + { label: 'Maven (Proxy)', value: 'maven-proxy' }, + { label: 'Maven (Group)', value: 'maven-group' }, + { label: 'Docker (Local)', value: 'docker' }, + { label: 'Docker (Proxy)', value: 'docker-proxy' }, + { label: 'Docker (Group)', value: 'docker-group' }, + { label: 'npm (Local)', value: 'npm' }, + { label: 'npm (Proxy)', value: 'npm-proxy' }, + { label: 'npm (Group)', value: 'npm-group' }, + { label: 'PyPI (Local)', value: 'pypi' }, + { label: 'PyPI (Proxy)', value: 'pypi-proxy' }, + { label: 'PyPI (Group)', value: 'pypi-group' }, + { label: 'Go (Proxy)', value: 'go-proxy' }, + { label: 'Helm (Local)', value: 'helm' }, + { label: 'NuGet (Local)', value: 'nuget' }, + { label: 'Debian (Local)', value: 'deb' }, + { label: 'RPM (Local)', value: 'rpm' }, + { label: 'Conda (Local)', value: 'conda' }, + { label: 'RubyGems (Local)', value: 'gem' }, + { label: 'Conan (Local)', value: 'conan' }, + { label: 'Hex (Local)', value: 'hex' }, + { label: 'PHP (Local)', value: 'php' }, + { label: 'PHP (Proxy)', value: 'php-proxy' }, + { label: 'File (Local)', value: 'file' }, + { label: 'File (Proxy)', value: 'file-proxy' }, + { label: 'File (Group)', value: 'file-group' }, + { label: 'Binary (Local)', value: 'binary' }, +] as const diff --git a/artipie-ui/src/views/NotFoundView.vue b/artipie-ui/src/views/NotFoundView.vue new file mode 100644 index 000000000..cd1766fc3 --- /dev/null +++ b/artipie-ui/src/views/NotFoundView.vue @@ -0,0 +1,11 @@ + diff --git a/artipie-ui/src/views/admin/CooldownView.vue b/artipie-ui/src/views/admin/CooldownView.vue new file mode 100644 index 000000000..97edcd114 --- /dev/null +++ b/artipie-ui/src/views/admin/CooldownView.vue @@ -0,0 +1,203 @@ + + + diff --git a/artipie-ui/src/views/admin/RepoCreateView.vue b/artipie-ui/src/views/admin/RepoCreateView.vue new file mode 100644 index 000000000..aabcc7a2f --- /dev/null +++ b/artipie-ui/src/views/admin/RepoCreateView.vue @@ -0,0 +1,323 @@ + + +