Add v2/beatmapsets

.env.example

@@ -18,4 +18,5 @@ MIRRORS_TO_IGNORE="someserver,otherserver,osulabs" # Optional: If you need to di
 # Options: 'mino' | 'bancho' | 'direct' | 'gatari' | 'nerinyan' | 'osulabs';
 
 DISABLE_SAFE_RATELIMIT_MODE= # Optional: By default, we use only 90% of the available APIs ratelimits to not get any hard restrictions. Add 'true' to disable this behaviour for better performance. 
-DISABLE_DAILY_RATE_LIMIT= # Optional: Add 'true' to ignore all daily ratelimits. 
+DISABLE_DAILY_RATE_LIMIT= # Optional: Add 'true' to ignore all daily ratelimits. 
+ENABLE_CRON_TO_CLEAR_OUTDATED_BEATMAPS= # Optional: Enable this if you are having problems with a disk space due to the database. Add 'true' to enable. 

package.json

@@ -16,17 +16,17 @@
     },
     "dependencies": {
         "@bogeychan/elysia-logger": "^0.1.4",
-        "@elysiajs/cors": "^1.1.1",
-        "@elysiajs/server-timing": "^1.1.0",
-        "@elysiajs/swagger": "^1.1.5",
+        "@elysiajs/cors": "^1.3.0",
+        "@elysiajs/server-timing": "^1.3.0",
+        "@elysiajs/swagger": "^1.3.0",
         "@faker-js/faker": "^10.1.0",
         "@types/adm-zip": "^0.5.6",
         "adm-zip": "^0.5.16",
         "axios": "^1.7.7",
         "dotenv": "^16.4.5",
         "drizzle-orm": "^0.35.3",
-        "elysia": "latest",
-        "elysia-autoload": "^1.4.0",
+        "elysia": "~1.3.1",
+        "elysia-autoload": "^1.6.0",
         "elysia-ip": "^1.0.7",
         "elysia-rate-limit": "^4.1.0",
         "elysia-requestid": "1.0.9",

server/src/config.ts

@@ -32,6 +32,7 @@ const {
     MIRRORS_TO_IGNORE,
     DISABLE_SAFE_RATELIMIT_MODE,
     DISABLE_DAILY_RATE_LIMIT,
+    ENABLE_CRON_TO_CLEAR_OUTDATED_BEATMAPS,
 } = process.env;
 
 if (!POSTGRES_USER || !POSTGRES_PASSWORD) {
@@ -72,6 +73,7 @@ const config: {
     MirrorsToIgnore: string[];
     DisableSafeRatelimitMode: boolean;
     DisableDailyRateLimit: boolean;
+    EnableCronToClearOutdatedBeatmaps: boolean;
 } = {
     PORT: PORT || '3000',
     POSTGRES_USER: POSTGRES_USER || 'admin',
@@ -95,6 +97,8 @@ const config: {
     MirrorsToIgnore: MIRRORS_TO_IGNORE?.split(',').map((v) => v.trim()) ?? [],
     DisableSafeRatelimitMode: DISABLE_SAFE_RATELIMIT_MODE === 'true',
     DisableDailyRateLimit: DISABLE_DAILY_RATE_LIMIT === 'true',
+    EnableCronToClearOutdatedBeatmaps:
+        ENABLE_CRON_TO_CLEAR_OUTDATED_BEATMAPS === 'true',
 };
 
 export const observationaryConfigPublic = {
@@ -108,6 +112,8 @@ export const observationaryConfigPublic = {
     MirrorsToIgnore: MIRRORS_TO_IGNORE?.split(',').map((v) => v.trim()) ?? [],
     DisableSafeRatelimitMode: DISABLE_SAFE_RATELIMIT_MODE === 'true',
     DisableDailyRateLimit: DISABLE_DAILY_RATE_LIMIT === 'true',
+    EnableCronToClearOutdatedBeatmaps:
+        ENABLE_CRON_TO_CLEAR_OUTDATED_BEATMAPS === 'true',
 };
 
 export default config;

server/src/controllers/api/index.ts

@@ -164,6 +164,31 @@ export default (app: App) => {
                 }),
                 tags: ['v2'],
             },
+        )
+        .get(
+            'v2/beatmapsets',
+            async ({ BeatmapsManagerInstance, query, set }) => {
+                const data =
+                    await BeatmapsManagerInstance.getBeatmapsetsByBeatmapIds({
+                        beatmapIds: query.beatmapIds,
+                    });
+
+                if (data.source) {
+                    set.headers['X-Data-Source'] = data.source;
+                }
+
+                data.source = undefined;
+
+                if (!data.data) return data;
+
+                return data.data;
+            },
+            {
+                query: t.Object({
+                    beatmapIds: t.Array(t.Numeric()),
+                }),
+                tags: ['v2'],
+            },
         );
 
     return app;

server/src/core/abstracts/client/base-client.abstract.ts

@@ -6,6 +6,7 @@ import {
     DownloadOsuBeatmap,
     GetBeatmapOptions,
     GetBeatmapSetOptions,
+    GetBeatmapsetsByBeatmapIdsOptions,
     GetBeatmapsOptions,
     ResultWithStatus,
     SearchBeatmapsetsOptions,
@@ -78,6 +79,12 @@ export class BaseClient {
         throw new Error('Method not implemented.');
     }
 
+    async getBeatmapsetsByBeatmapIds(
+        ctx: GetBeatmapsetsByBeatmapIdsOptions,
+    ): Promise<ResultWithStatus<Beatmapset[] | null>> {
+        throw new Error('Method not implemented.');
+    }
+
     async downloadBeatmapSet(
         ctx: DownloadBeatmapSetOptions,
     ): Promise<ResultWithStatus<ArrayBuffer | null>> {

server/src/core/abstracts/client/base-client.types.ts

@@ -24,6 +24,10 @@ export type GetBeatmapsOptions = {
     ids: number[];
 };
 
+export type GetBeatmapsetsByBeatmapIdsOptions = {
+    beatmapIds: number[];
+};
+
 export type DownloadBeatmapSetOptions = {
     beatmapSetId: number;
     noVideo?: boolean;
@@ -52,6 +56,7 @@ export enum ClientAbilities {
     SearchBeatmapsets = 1 << 8, // 256
     GetBeatmaps = 1 << 9, // 512
     DownloadOsuBeatmap = 1 << 10, // 1024
+    GetBeatmapsetsByBeatmapIds = 1 << 11, // 2048
 }
 
 export type MirrorClient<T extends BaseClient = BaseClient> = {

server/src/core/domains/osu.ppy.sh/bancho.client.ts

@@ -6,6 +6,7 @@ import {
     DownloadOsuBeatmap,
     GetBeatmapOptions,
     GetBeatmapSetOptions,
+    GetBeatmapsetsByBeatmapIdsOptions,
     GetBeatmapsOptions,
     ResultWithStatus,
 } from '../../abstracts/client/base-client.types';
@@ -24,6 +25,7 @@ export class BanchoClient extends BaseClient {
                     ClientAbilities.GetBeatmapSetById,
                     ClientAbilities.GetBeatmaps,
                     ClientAbilities.DownloadOsuBeatmap,
+                    ClientAbilities.GetBeatmapsetsByBeatmapIds,
                 ],
             },
             {
@@ -34,6 +36,7 @@ export class BanchoClient extends BaseClient {
                             ClientAbilities.GetBeatmapSetById,
                             ClientAbilities.GetBeatmaps,
                             ClientAbilities.DownloadOsuBeatmap,
+                            ClientAbilities.GetBeatmapsetsByBeatmapIds,
                         ],
                         routes: ['/'],
                         limit: 1200,
@@ -94,6 +97,39 @@ export class BanchoClient extends BaseClient {
         };
     }
 
+    async getBeatmapsetsByBeatmapIds(
+        ctx: GetBeatmapsetsByBeatmapIdsOptions,
+    ): Promise<ResultWithStatus<Beatmapset[] | null>> {
+        const { beatmapIds } = ctx;
+
+        const result = await this.api.get<{ beatmaps: BanchoBeatmap[] }>(
+            `api/v2/beatmaps?${beatmapIds.map((id) => `ids[]=${id}`).join('&')}`,
+            {
+                config: {
+                    headers: {
+                        Authorization: `Bearer ${await this.osuApiKey}`,
+                    },
+                },
+            },
+        );
+
+        if (!result || result.status !== 200) {
+            return { result: null, status: result?.status ?? 500 };
+        }
+
+        return {
+            result: result.data?.beatmaps?.map((b: BanchoBeatmap) =>
+                this.convertService.convertBeatmapset({
+                    ...b.beatmapset,
+                    ...(b.convert
+                        ? { converts: [b.convert] }
+                        : { beatmaps: [b] }),
+                } as BanchoBeatmapset),
+            ),
+            status: result.status,
+        };
+    }
+
     async downloadOsuBeatmap(
         ctx: DownloadOsuBeatmap,
     ): Promise<ResultWithStatus<ArrayBuffer | null>> {

server/src/core/managers/beatmaps/beatmaps.manager.ts

@@ -8,6 +8,7 @@ import {
     DownloadOsuBeatmap,
     ResultWithStatus,
     SearchBeatmapsetsOptions,
+    GetBeatmapsetsByBeatmapIdsOptions,
 } from '../../abstracts/client/base-client.types';
 import { MirrorsManager } from '../mirrors/mirrors.manager';
 import { ServerResponse } from './beatmaps-manager.types';
@@ -147,6 +148,82 @@ export class BeatmapsManager {
         };
     }
 
+    async getBeatmapsetsByBeatmapIds(
+        ctx: GetBeatmapsetsByBeatmapIdsOptions,
+    ): Promise<ServerResponse<Beatmapset[]>> {
+        const beatmapsets =
+            await this.StorageManager.getBeatmapSetsByBeatmapIds(ctx);
+
+        const missingIds =
+            ctx.beatmapIds.filter(
+                (id) =>
+                    beatmapsets !== undefined &&
+                    !beatmapsets
+                        ?.flatMap(
+                            (set) => set.beatmaps?.map((map) => map.id) ?? [],
+                        )
+                        .includes(id),
+            ) ?? ctx.beatmapIds;
+
+        if (beatmapsets && missingIds.length === 0) {
+            return {
+                data: beatmapsets,
+                status: HttpStatusCode.Ok,
+                message: undefined,
+                source: 'storage',
+            };
+        }
+
+        let result = await this.MirrorsManager.getBeatmapsetsByBeatmapIds({
+            beatmapIds: missingIds,
+        });
+
+        if (result.status >= 500) {
+            return this.formatResultAsServerError(result);
+        }
+
+        // ! NOTE: Results from getBeatmapsetsByBeatmapIds will include not all beatmaps, so we need to fetch beatmapsets again to fetch all beatmaps with them
+        result.result = await Promise.all(
+            result.result?.map(
+                async (beatmapset) =>
+                    await this.getBeatmapSet({
+                        beatmapSetId: beatmapset.id,
+                    }),
+            ) ?? [],
+        ).then(
+            (results) =>
+                results
+                    .map((result) => result.data ?? null)
+                    .filter((result) => result !== null) as Beatmapset[],
+        );
+
+        if (result.result && result.result.length > 0) {
+            for (const beatmapset of result.result) {
+                this.StorageManager.insertBeatmapset(beatmapset, {
+                    beatmapSetId: beatmapset.id,
+                });
+            }
+        }
+
+        const missingIdsFromResult =
+            result.result?.filter(
+                (set) =>
+                    !beatmapsets
+                        ?.flatMap((set) => set.beatmaps?.map((map) => map.id))
+                        .includes(set.id),
+            ) ?? [];
+
+        return {
+            data: [...(beatmapsets ?? []), ...(result.result ?? [])],
+            status: result.status,
+            message:
+                missingIdsFromResult.length > 0
+                    ? `Some beatmapsets not found: ${missingIdsFromResult.join(', ')}`
+                    : undefined,
+            source: 'mirror',
+        };
+    }
+
     async downloadBeatmapSet(
         ctx: DownloadBeatmapSetOptions,
     ): Promise<ServerResponse<null | ArrayBuffer>> {

server/src/core/managers/mirrors/mirrors.manager.ts

@@ -8,6 +8,7 @@ import {
     SearchBeatmapsetsOptions,
     GetBeatmapsOptions,
     DownloadOsuBeatmap,
+    GetBeatmapsetsByBeatmapIdsOptions,
 } from '../../abstracts/client/base-client.types';
 import { DirectClient, BanchoClient } from '../../domains';
 import { MirrorsManagerService } from './mirrors-manager.service';
@@ -165,6 +166,24 @@ export class MirrorsManager {
         return await this.useMirror<Beatmap[]>(ctx, criteria, 'getBeatmaps');
     }
 
+    async getBeatmapsetsByBeatmapIds(
+        ctx: GetBeatmapsetsByBeatmapIdsOptions,
+    ): Promise<ResultWithStatus<Beatmapset[]>> {
+        const { beatmapIds } = ctx;
+
+        if (!beatmapIds || beatmapIds.length === 0) {
+            throw new Error('beatmapIds is required to fetch beatmapsets');
+        }
+
+        const criteria = ClientAbilities.GetBeatmapsetsByBeatmapIds;
+
+        return await this.useMirror<Beatmapset[]>(
+            ctx,
+            criteria,
+            'getBeatmapsetsByBeatmapIds',
+        );
+    }
+
     async downloadBeatmapSet(
         ctx: DownloadBeatmapSetOptions,
     ): Promise<ResultWithStatus<ArrayBuffer>> {
@@ -274,7 +293,8 @@ export class MirrorsManager {
             | GetBeatmapSetOptions
             | SearchBeatmapsetsOptions
             | GetBeatmapsOptions
-            | DownloadOsuBeatmap,
+            | DownloadOsuBeatmap
+            | GetBeatmapsetsByBeatmapIdsOptions,
         criteria: ClientAbilities,
         action: keyof MirrorClient['client'],
     ): Promise<ResultWithStatus<T>> {

server/src/core/managers/storage/storage.manager.ts

@@ -3,6 +3,7 @@ import {
     DownloadOsuBeatmap,
     GetBeatmapOptions,
     GetBeatmapSetOptions,
+    GetBeatmapsetsByBeatmapIdsOptions,
     SearchBeatmapsetsOptions,
 } from '../../abstracts/client/base-client.types';
 import { Beatmap, Beatmapset } from '../../../types/general/beatmap';
@@ -17,13 +18,15 @@ import {
     deleteBeatmapsets,
     getBeatmapSetById,
     getBeatmapSetCount,
+    getBeatmapSetsByBeatmapIds,
     getUnvalidBeatmapSets,
 } from '../../../database/models/beatmapset';
 import { StorageCacheService } from './storage-cache.service';
 import { StorageFilesService } from './storage-files.service';
 import { getBeatmapSetsFilesCount } from '../../../database/models/beatmapsetFile';
 import { getBeatmapOsuFileCount } from '../../../database/models/beatmapOsuFile';
 import logger from '../../../utils/logger';
+import config from '../../../config';
 
 export class StorageManager {
     private readonly cacheService: StorageCacheService;
@@ -85,6 +88,18 @@ export class StorageManager {
         return entity ?? undefined;
     }
 
+    async getBeatmapSetsByBeatmapIds(
+        ctx: GetBeatmapsetsByBeatmapIdsOptions,
+    ): Promise<Beatmapset[] | null | undefined> {
+        const entities = await getBeatmapSetsByBeatmapIds(ctx.beatmapIds, true);
+
+        if (entities === null) {
+            return null;
+        }
+
+        return entities ?? undefined;
+    }
+
     async getBeatmapsetFile(
         ctx: DownloadBeatmapSetOptions,
     ): Promise<ArrayBuffer | undefined | null> {
@@ -174,6 +189,10 @@ export class StorageManager {
     }
 
     private async clearOldBeatmapsets() {
+        if (!config.EnableCronToClearOutdatedBeatmaps) {
+            return;
+        }
+
         const beatmapsetsForRemoval = await getUnvalidBeatmapSets();
 
         const forRemoval = [...beatmapsetsForRemoval];