diff --git a/backend/package.json b/backend/package.json index 510fc4bb..083b56cd 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,7 +10,6 @@ "migrate": "knex migrate:latest", "migrate:rollback": "knex migrate:rollback", "seed": "knex seed:run", - "etl": "ts-node src/scripts/run-etl.ts", "lint": "eslint src --ext .ts", "format": "prettier --write \"src/**/*.ts\"" }, @@ -24,7 +23,6 @@ "dependencies": { "@azure/identity": "^4.0.1", "@azure/keyvault-secrets": "^4.7.0", - "@neondatabase/serverless": "^0.9.0", "axios": "^1.13.2", "bcrypt": "^5.1.1", "connect-redis": "^7.1.0", diff --git a/backend/src/database/connection.ts b/backend/src/database/connection.ts index b71d3cfe..a5a2eb11 100644 --- a/backend/src/database/connection.ts +++ b/backend/src/database/connection.ts @@ -11,11 +11,14 @@ export const getDatabasePool = (): Pool => { throw new Error('DATABASE_URL environment variable is not set'); } + const isSupabase = connectionString.includes('supabase.co'); const config: PoolConfig = { connectionString, - ssl: { - rejectUnauthorized: false - }, + ssl: isSupabase ? { + rejectUnauthorized: false + } : connectionString.includes('sslmode=require') ? { + rejectUnauthorized: false + } : undefined, max: parseInt(process.env.DB_POOL_MAX || '10'), min: parseInt(process.env.DB_POOL_MIN || '2'), idleTimeoutMillis: 30000, diff --git a/backend/src/database/migrations/009_add_mbid_to_music_items.ts b/backend/src/database/migrations/009_add_mbid_to_music_items.ts new file mode 100644 index 00000000..e3261235 --- /dev/null +++ b/backend/src/database/migrations/009_add_mbid_to_music_items.ts @@ -0,0 +1,21 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('music_items', (table) => { + table.uuid('mbid').nullable(); + table.index('mbid'); + }); + + await knex.raw(` + CREATE UNIQUE INDEX IF NOT EXISTS music_items_mbid_unique + ON music_items (mbid) + WHERE mbid IS NOT NULL + `); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('music_items', (table) => { + table.dropIndex('mbid'); + table.dropColumn('mbid'); + }); +} diff --git a/backend/src/database/seeds/002_seed_music_items.ts b/backend/src/database/seeds/002_seed_music_items.ts deleted file mode 100644 index 674610c7..00000000 --- a/backend/src/database/seeds/002_seed_music_items.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { Knex } from 'knex'; - -export async function seed(knex: Knex): Promise { - const existingItems = await knex('music_items').count('id as count').first(); - - if (existingItems && parseInt(existingItems.count as string) > 0) { - console.log('Music items already seeded, skipping...'); - return; - } - - await knex('music_items').insert([ - { - type: 'album', - title: 'ASTROWORLD', - artist: 'Travis Scott', - spotify_id: 'spotify_astroworld', - apple_music_id: 'apple_astroworld' - }, - { - type: 'song', - title: 'Blinding Lights', - artist: 'The Weeknd', - spotify_id: 'spotify_blinding_lights', - apple_music_id: 'apple_blinding_lights' - }, - { - type: 'album', - title: 'Blonde', - artist: 'Frank Ocean', - spotify_id: 'spotify_blonde', - apple_music_id: 'apple_blonde' - }, - { - type: 'artist', - title: 'Kendrick Lamar', - artist: 'Artist Profile', - spotify_id: 'spotify_kendrick', - apple_music_id: 'apple_kendrick' - }, - { - type: 'album', - title: 'folklore', - artist: 'Taylor Swift', - spotify_id: 'spotify_folklore', - apple_music_id: 'apple_folklore' - }, - { - type: 'song', - title: 'As It Was', - artist: 'Harry Styles', - spotify_id: 'spotify_as_it_was', - apple_music_id: 'apple_as_it_was' - }, - { - type: 'album', - title: 'To Pimp a Butterfly', - artist: 'Kendrick Lamar', - spotify_id: 'spotify_tpab', - apple_music_id: 'apple_tpab' - }, - { - type: 'album', - title: 'My Beautiful Dark Twisted Fantasy', - artist: 'Kanye West', - spotify_id: 'spotify_mbdtf', - apple_music_id: 'apple_mbdtf' - } - ]); - - console.log('Music items seeded successfully'); -} diff --git a/backend/src/index.ts b/backend/src/index.ts index 2cb27e03..a7d38f48 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -74,7 +74,6 @@ app.use('/interactions', webhookRoutes); app.use(errorMiddleware); import { testConnection } from './database/connection'; -import { startETLScheduler } from './jobs/music-etl.job'; app.listen(PORT, async () => { logger.info('Service: musiq-api'); @@ -87,10 +86,6 @@ app.listen(PORT, async () => { } else { logger.error('Database connection failed - check DATABASE_URL'); } - - if (process.env.ENABLE_ETL === 'true') { - startETLScheduler(); - } }); export default app; diff --git a/backend/src/jobs/music-etl.job.ts b/backend/src/jobs/music-etl.job.ts deleted file mode 100644 index a2a45b0a..00000000 --- a/backend/src/jobs/music-etl.job.ts +++ /dev/null @@ -1,50 +0,0 @@ -import * as cron from 'node-cron'; -import { MusicETLService } from '../services/music-etl.service'; -import { logger } from '../config/logger'; - -let etlService: MusicETLService; -let isRunning = false; - -export function startETLScheduler(): void { - etlService = new MusicETLService(); - - const cronSchedule = process.env.ETL_CRON_SCHEDULE || '0 */6 * * *'; - - logger.info(`Starting ETL scheduler with schedule: ${cronSchedule}`); - - cron.schedule(cronSchedule, async () => { - if (isRunning) { - logger.warn('ETL job already running, skipping...'); - return; - } - - isRunning = true; - try { - await etlService.runETLJob(); - } catch (error) { - logger.error('Scheduled ETL job failed', { error }); - } finally { - isRunning = false; - } - }); - - logger.info('ETL scheduler started'); -} - -export async function runETLJobManually(): Promise { - if (!etlService) { - etlService = new MusicETLService(); - } - - if (isRunning) { - throw new Error('ETL job is already running'); - } - - isRunning = true; - try { - await etlService.runETLJob(); - } finally { - isRunning = false; - } -} - diff --git a/backend/src/routes/admin.ts b/backend/src/routes/admin.ts index 0c1b62a2..1e05b26d 100644 --- a/backend/src/routes/admin.ts +++ b/backend/src/routes/admin.ts @@ -1,36 +1,8 @@ import { Router } from 'express'; import { authMiddleware, AuthRequest } from '../middleware/auth.middleware'; import { requireRole } from '../middleware/rbac.middleware'; -import { runETLJobManually } from '../jobs/music-etl.job'; -import { logger } from '../config/logger'; const router = Router(); -router.post( - '/etl/run', - authMiddleware, - requireRole('admin'), - async (req: AuthRequest, res, next) => { - try { - logger.info('Manual ETL job triggered', { userId: req.userId }); - - runETLJobManually() - .then(() => { - logger.info('Manual ETL job completed'); - }) - .catch((error) => { - logger.error('Manual ETL job failed', { error }); - }); - - res.json({ - success: true, - message: 'ETL job started. Check logs for progress.', - }); - } catch (error) { - next(error); - } - } -); - export default router; diff --git a/backend/src/scripts/run-etl.ts b/backend/src/scripts/run-etl.ts deleted file mode 100644 index ad717190..00000000 --- a/backend/src/scripts/run-etl.ts +++ /dev/null @@ -1,20 +0,0 @@ -import dotenv from 'dotenv'; -import { runETLJobManually } from '../jobs/music-etl.job'; -import { logger } from '../config/logger'; - -dotenv.config(); - -async function main() { - try { - logger.info('Running ETL job manually...'); - await runETLJobManually(); - logger.info('ETL job completed successfully'); - process.exit(0); - } catch (error) { - logger.error('ETL job failed', { error }); - process.exit(1); - } -} - -main(); - diff --git a/backend/src/services/music-etl.service.ts b/backend/src/services/music-etl.service.ts deleted file mode 100644 index 6bcb2823..00000000 --- a/backend/src/services/music-etl.service.ts +++ /dev/null @@ -1,283 +0,0 @@ -import axios from 'axios'; -import { getDatabasePool } from '../database/connection'; -import { logger } from '../config/logger'; - -interface SpotifyTrack { - id: string; - name: string; - artists: Array<{ name: string }>; - album: { - name: string; - images: Array<{ url: string }>; - }; - external_ids?: { - isrc?: string; - }; -} - -interface SpotifyAlbum { - id: string; - name: string; - artists: Array<{ name: string }>; - images: Array<{ url: string }>; - external_ids?: { - upc?: string; - }; -} - -export class MusicETLService { - private pool = getDatabasePool(); - private spotifyClientId: string; - private spotifyClientSecret: string; - private spotifyAccessToken: string | null = null; - private spotifyTokenExpiry: number = 0; - - constructor() { - this.spotifyClientId = process.env.SPOTIFY_CLIENT_ID || ''; - this.spotifyClientSecret = process.env.SPOTIFY_CLIENT_SECRET || ''; - } - - async getSpotifyAccessToken(): Promise { - if (this.spotifyAccessToken && Date.now() < this.spotifyTokenExpiry) { - return this.spotifyAccessToken; - } - - try { - const response = await axios.post( - 'https://accounts.spotify.com/api/token', - new URLSearchParams({ - grant_type: 'client_credentials', - }).toString(), - { - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - Authorization: `Basic ${Buffer.from( - `${this.spotifyClientId}:${this.spotifyClientSecret}` - ).toString('base64')}`, - }, - } - ); - - const accessToken = response.data.access_token; - if (!accessToken || typeof accessToken !== 'string') { - throw new Error('Failed to get Spotify access token'); - } - - this.spotifyAccessToken = accessToken; - this.spotifyTokenExpiry = Date.now() + (response.data.expires_in * 1000) - 60000; - - return accessToken; - } catch (error) { - logger.error('Failed to get Spotify access token', { error }); - throw error; - } - } - - async fetchSpotifyNewReleases(limit: number = 50): Promise { - try { - const token = await this.getSpotifyAccessToken(); - const response = await axios.get<{ albums: { items: SpotifyAlbum[] } }>( - 'https://api.spotify.com/v1/browse/new-releases', - { - headers: { - Authorization: `Bearer ${token}`, - }, - params: { - limit, - country: 'US', - }, - } - ); - - return response.data.albums.items; - } catch (error) { - logger.error('Failed to fetch Spotify new releases', { error }); - throw error; - } - } - - async fetchSpotifyFeaturedPlaylists(limit: number = 50): Promise { - try { - const token = await this.getSpotifyAccessToken(); - - const playlistsResponse = await axios.get<{ playlists: { items: Array<{ id: string }> } }>( - 'https://api.spotify.com/v1/browse/featured-playlists', - { - headers: { - Authorization: `Bearer ${token}`, - }, - params: { - limit: 10, - country: 'US', - }, - } - ); - - const tracks: SpotifyTrack[] = []; - - for (const playlist of playlistsResponse.data.playlists.items.slice(0, 5)) { - try { - const tracksResponse = await axios.get<{ items: Array<{ track: SpotifyTrack }> }>( - `https://api.spotify.com/v1/playlists/${playlist.id}/tracks`, - { - headers: { - Authorization: `Bearer ${token}`, - }, - params: { - limit: 10, - }, - } - ); - - tracks.push(...tracksResponse.data.items.map(item => item.track).filter(Boolean)); - } catch (error) { - logger.warn('Failed to fetch playlist tracks', { playlistId: playlist.id, error }); - } - } - - return tracks.slice(0, limit); - } catch (error) { - logger.error('Failed to fetch Spotify featured playlists', { error }); - throw error; - } - } - - async fetchSpotifyTopTracks(limit: number = 50): Promise { - try { - const token = await this.getSpotifyAccessToken(); - const response = await axios.get<{ items: Array<{ track: SpotifyTrack | null }> }>( - 'https://api.spotify.com/v1/playlists/37i9dQZEVXbMDoHDwfc2tq/tracks', - { - headers: { - Authorization: `Bearer ${token}`, - }, - params: { - limit, - }, - } - ); - - return response.data.items - .map((item: { track: SpotifyTrack | null }) => item.track) - .filter((track): track is SpotifyTrack => track !== null); - } catch (error) { - logger.error('Failed to fetch Spotify top tracks', { error }); - throw error; - } - } - - async transformAndStoreAlbums(albums: SpotifyAlbum[]): Promise { - let inserted = 0; - - for (const album of albums) { - try { - const artistName = album.artists.map(a => a.name).join(', '); - const imageUrl = album.images?.[0]?.url || null; - - const result = await this.pool.query( - `INSERT INTO music_items (type, title, artist, image_url, spotify_id, metadata) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (spotify_id) DO UPDATE SET - title = EXCLUDED.title, - artist = EXCLUDED.artist, - image_url = EXCLUDED.image_url, - metadata = EXCLUDED.metadata, - updated_at = NOW() - RETURNING id`, - [ - 'album', - album.name, - artistName, - imageUrl, - album.id, - JSON.stringify({ - release_date: null, - total_tracks: null, - genres: [], - }), - ] - ); - - if (result.rows.length > 0) { - inserted++; - } - } catch (error) { - logger.warn('Failed to insert album', { album: album.name, error }); - } - } - - return inserted; - } - - async transformAndStoreTracks(tracks: SpotifyTrack[]): Promise { - let inserted = 0; - - for (const track of tracks) { - try { - const artistName = track.artists.map(a => a.name).join(', '); - const imageUrl = track.album?.images?.[0]?.url || null; - - const result = await this.pool.query( - `INSERT INTO music_items (type, title, artist, image_url, spotify_id, metadata) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (spotify_id) DO UPDATE SET - title = EXCLUDED.title, - artist = EXCLUDED.artist, - image_url = EXCLUDED.image_url, - metadata = EXCLUDED.metadata, - updated_at = NOW() - RETURNING id`, - [ - 'song', - track.name, - artistName, - imageUrl, - track.id, - JSON.stringify({ - album: track.album?.name, - duration_ms: null, - explicit: null, - }), - ] - ); - - if (result.rows.length > 0) { - inserted++; - } - } catch (error) { - logger.warn('Failed to insert track', { track: track.name, error }); - } - } - - return inserted; - } - - async runETLJob(): Promise { - logger.info('Starting music ETL job...'); - - try { - let totalInserted = 0; - - const newReleases = await this.fetchSpotifyNewReleases(50); - const albumsInserted = await this.transformAndStoreAlbums(newReleases); - totalInserted += albumsInserted; - logger.info(`Inserted ${albumsInserted} new albums from Spotify new releases`); - - const topTracks = await this.fetchSpotifyTopTracks(50); - const tracksInserted = await this.transformAndStoreTracks(topTracks); - totalInserted += tracksInserted; - logger.info(`Inserted ${tracksInserted} new tracks from Spotify top tracks`); - - const featuredTracks = await this.fetchSpotifyFeaturedPlaylists(30); - const featuredInserted = await this.transformAndStoreTracks(featuredTracks); - totalInserted += featuredInserted; - logger.info(`Inserted ${featuredInserted} new tracks from Spotify featured playlists`); - - logger.info(`ETL job completed. Total items inserted/updated: ${totalInserted}`); - } catch (error) { - logger.error('ETL job failed', { error }); - throw error; - } - } -} - diff --git a/etl/src/database/connection.ts b/etl/src/database/connection.ts index 295852d8..c41d14e6 100644 --- a/etl/src/database/connection.ts +++ b/etl/src/database/connection.ts @@ -13,9 +13,12 @@ export function getDatabasePool(): Pool { throw new Error('DATABASE_URL environment variable is required'); } + const isSupabase = connectionString.includes('supabase.co'); pool = new Pool({ connectionString, - ssl: connectionString.includes('sslmode=require') ? { + ssl: isSupabase ? { + rejectUnauthorized: false + } : connectionString.includes('sslmode=require') ? { rejectUnauthorized: false } : undefined, max: 10, diff --git a/etl/src/services/musicbrainz-etl.service.ts b/etl/src/services/musicbrainz-etl.service.ts index ac84d2e3..df028611 100644 --- a/etl/src/services/musicbrainz-etl.service.ts +++ b/etl/src/services/musicbrainz-etl.service.ts @@ -1,6 +1,7 @@ import { MusicBrainzExtractService } from './musicbrainz-extract.service'; import { MusicBrainzTransformService } from './musicbrainz-transform.service'; import { MusicBrainzLoadService } from './musicbrainz-load.service'; +import { MusicBrainzSyncService } from './musicbrainz-sync.service'; import { CoverArtEnrichmentService } from './cover-art-enrichment.service'; import { MusicBrainzConfig } from '../config/musicbrainz.config'; import { logger } from '../config/logger'; @@ -8,6 +9,9 @@ import { logger } from '../config/logger'; interface ETLOptions { albumsPerGenre?: number; genres?: string[]; + maxTotalAlbums?: number; + maxTotalArtists?: number; + maxTotalTracks?: number; skipArtists?: boolean; skipAlbums?: boolean; skipTracks?: boolean; @@ -18,12 +22,14 @@ export class MusicBrainzETLService { private extractService: MusicBrainzExtractService; private transformService: MusicBrainzTransformService; private loadService: MusicBrainzLoadService; + private syncService: MusicBrainzSyncService; private coverArtService: CoverArtEnrichmentService; constructor() { this.extractService = new MusicBrainzExtractService(); this.transformService = new MusicBrainzTransformService(); this.loadService = new MusicBrainzLoadService(); + this.syncService = new MusicBrainzSyncService(); this.coverArtService = new CoverArtEnrichmentService(); } @@ -37,19 +43,20 @@ export class MusicBrainzETLService { ...MusicBrainzConfig.nicheGenres ]; - const genresToProcess = options.genres || allGenres; - const albumsPerGenre = options.albumsPerGenre || - (genresToProcess.some(g => MusicBrainzConfig.majorGenres.includes(g)) - ? MusicBrainzConfig.albumsPerMajorGenre - : MusicBrainzConfig.albumsPerNicheGenre); + const top5Genres = ['hip-hop', 'pop', 'rock', 'electronic', 'jazz']; + const genresToProcess = options.genres || top5Genres; + const maxTotalAlbums = options.maxTotalAlbums || 2000; + const maxTotalArtists = options.maxTotalArtists || 2000; + const maxTotalTracks = options.maxTotalTracks || 2000; + const albumsPerGenre = options.albumsPerGenre || Math.ceil(maxTotalAlbums / genresToProcess.length); - logger.info(`Processing ${genresToProcess.length} genres with ${albumsPerGenre} albums per genre`); + logger.info(`Processing ${genresToProcess.length} genres with ${albumsPerGenre} albums per genre (max: ${maxTotalAlbums} albums, ${maxTotalArtists} artists, ${maxTotalTracks} tracks)`); let allAlbums: any[] = []; if (!options.skipAlbums) { logger.info('=== EXTRACT PHASE: Albums ==='); - allAlbums = await this.extractService.extractAllGenres(genresToProcess, albumsPerGenre); + allAlbums = await this.extractService.extractAllGenres(genresToProcess, albumsPerGenre, maxTotalAlbums); logger.info(`Extracted ${allAlbums.length} total albums`); } @@ -59,10 +66,13 @@ export class MusicBrainzETLService { } logger.info('=== TRANSFORM PHASE ==='); - const { artists, albumArtists } = this.transformService.transformArtists(allAlbums); + const { artists: allArtists, albumArtists: allAlbumArtists } = this.transformService.transformArtists(allAlbums); const albums = this.transformService.transformAlbums(allAlbums); - logger.info(`Transformed: ${artists.length} artists, ${albums.length} albums`); + const artists = allArtists.slice(0, maxTotalArtists); + const albumArtists = allAlbumArtists.filter(rel => artists.some(a => a.mbid === rel.artist_mbid)); + + logger.info(`Transformed: ${artists.length}/${allArtists.length} artists (limited to ${maxTotalArtists}), ${albums.length} albums`); if (options.enrichCoverArt) { logger.info('=== ENRICHMENT PHASE: Cover Art ==='); @@ -101,15 +111,23 @@ export class MusicBrainzETLService { const allAlbumTracks: any[] = []; for (let i = 0; i < albums.length; i++) { + if (allTracks.length >= maxTotalTracks) { + logger.info(`Reached max tracks limit (${maxTotalTracks}). Stopping track extraction.`); + break; + } + const album = albums[i]; logger.info(`Extracting tracks for album ${i + 1}/${albums.length}: ${album.title}`); const recordings = await this.extractService.extractRecordingsForReleaseGroup(album.mbid); if (recordings.length > 0) { - const positions = recordings.map((_, idx) => idx + 1); + const remainingSlots = maxTotalTracks - allTracks.length; + const recordingsToProcess = recordings.slice(0, remainingSlots); + + const positions = recordingsToProcess.map((_, idx) => idx + 1); const { tracks, albumTracks } = this.transformService.transformTracks( - recordings, + recordingsToProcess, album.mbid, positions ); @@ -118,20 +136,32 @@ export class MusicBrainzETLService { allAlbumTracks.push(...albumTracks); } + if (allTracks.length >= maxTotalTracks) { + logger.info(`Reached max tracks limit (${maxTotalTracks}). Stopping track extraction.`); + break; + } + if ((i + 1) % 10 === 0) { - logger.info(`Processed ${i + 1}/${albums.length} albums for tracks`); + logger.info(`Processed ${i + 1}/${albums.length} albums for tracks. Total tracks: ${allTracks.length}`); } } - logger.info(`Transformed ${allTracks.length} tracks`); + const finalTracks = allTracks.slice(0, maxTotalTracks); + const finalAlbumTracks = allAlbumTracks.slice(0, maxTotalTracks); - const tracksLoaded = await this.loadService.loadTracks(allTracks); + logger.info(`Transformed ${finalTracks.length}/${allTracks.length} tracks (limited to ${maxTotalTracks})`); + + const tracksLoaded = await this.loadService.loadTracks(finalTracks); logger.info(`Loaded ${tracksLoaded} tracks`); - const albumTracksLoaded = await this.loadService.loadAlbumTracks(allAlbumTracks); + const albumTracksLoaded = await this.loadService.loadAlbumTracks(finalAlbumTracks); logger.info(`Loaded ${albumTracksLoaded} album-track relations`); } + logger.info('=== SYNC PHASE: music_items ==='); + const syncResults = await this.syncService.syncAll(); + logger.info(`Sync completed: ${syncResults.albums} albums, ${syncResults.tracks} tracks, ${syncResults.artists} artists synced to music_items`); + const duration = ((Date.now() - startTime) / 1000 / 60).toFixed(2); logger.info(`ETL pipeline completed successfully in ${duration} minutes`); diff --git a/etl/src/services/musicbrainz-extract.service.ts b/etl/src/services/musicbrainz-extract.service.ts index c141d043..b12e680c 100644 --- a/etl/src/services/musicbrainz-extract.service.ts +++ b/etl/src/services/musicbrainz-extract.service.ts @@ -175,17 +175,28 @@ export class MusicBrainzExtractService { } } - async extractAllGenres(genres: string[], albumsPerGenre: number): Promise { + async extractAllGenres(genres: string[], albumsPerGenre: number, maxTotalAlbums: number = 2000): Promise { const allAlbums: MusicBrainzReleaseGroup[] = []; const seenIds = new Set(); for (const genre of genres) { + if (allAlbums.length >= maxTotalAlbums) { + logger.info(`Reached max total albums limit (${maxTotalAlbums}). Stopping extraction.`); + break; + } + logger.info(`Extracting albums for genre: ${genre}`); + const remainingSlots = maxTotalAlbums - allAlbums.length; + const albumsToExtract = Math.min(albumsPerGenre, remainingSlots); + try { - const albums = await this.extractTopAlbumsByGenre(genre, albumsPerGenre); + const albums = await this.extractTopAlbumsByGenre(genre, albumsToExtract); for (const album of albums) { + if (allAlbums.length >= maxTotalAlbums) { + break; + } if (!seenIds.has(album.id)) { seenIds.add(album.id); allAlbums.push(album); @@ -198,7 +209,7 @@ export class MusicBrainzExtractService { } } - return allAlbums; + return allAlbums.slice(0, maxTotalAlbums); } } diff --git a/etl/src/services/musicbrainz-sync.service.ts b/etl/src/services/musicbrainz-sync.service.ts new file mode 100644 index 00000000..1c378eda --- /dev/null +++ b/etl/src/services/musicbrainz-sync.service.ts @@ -0,0 +1,158 @@ +import { Pool } from 'pg'; +import { getDatabasePool } from '../database/connection'; +import { logger } from '../config/logger'; + +export class MusicBrainzSyncService { + private readonly pool: Pool; + + constructor() { + this.pool = getDatabasePool(); + } + + async syncAlbumsToMusicItems(): Promise { + logger.info('Syncing albums from mb_albums to music_items...'); + + const query = ` + INSERT INTO music_items (type, title, artist, image_url, mbid, metadata) + SELECT + 'album' as type, + ma.title, + COALESCE( + STRING_AGG(DISTINCT mba.name, ', ' ORDER BY mba.name), + 'Unknown Artist' + ) as artist, + ma.cover_art_url as image_url, + ma.mbid, + jsonb_build_object( + 'release_date', ma.release_date, + 'primary_type', ma.primary_type, + 'secondary_types', ma.secondary_types, + 'status', ma.status + ) as metadata + FROM mb_albums ma + LEFT JOIN mb_album_artists maa ON ma.mbid = maa.album_mbid + LEFT JOIN mb_artists mba ON maa.artist_mbid = mba.mbid + WHERE ma.mbid IS NOT NULL + GROUP BY ma.mbid, ma.title, ma.cover_art_url, ma.release_date, ma.primary_type, ma.secondary_types, ma.status + ON CONFLICT (mbid) + DO UPDATE SET + title = EXCLUDED.title, + artist = EXCLUDED.artist, + image_url = EXCLUDED.image_url, + metadata = EXCLUDED.metadata, + updated_at = NOW() + `; + + try { + const result = await this.pool.query(query); + const count = result.rowCount || 0; + logger.info(`Synced ${count} albums to music_items`); + return count; + } catch (error: any) { + logger.error('Error syncing albums', { error: error.message }); + throw error; + } + } + + async syncTracksToMusicItems(): Promise { + logger.info('Syncing tracks from mb_tracks to music_items...'); + + const query = ` + INSERT INTO music_items (type, title, artist, image_url, mbid, metadata) + SELECT + 'song' as type, + mt.title, + COALESCE( + STRING_AGG(DISTINCT mba.name, ', ' ORDER BY mba.name), + 'Unknown Artist' + ) as artist, + ma.cover_art_url as image_url, + mt.mbid, + jsonb_build_object( + 'album_title', ma.title, + 'album_mbid', ma.mbid, + 'length', mt.length, + 'position', mat.position, + 'disc_number', mat.disc_number + ) as metadata + FROM mb_tracks mt + INNER JOIN mb_album_tracks mat ON mt.mbid = mat.track_mbid + INNER JOIN mb_albums ma ON mat.album_mbid = ma.mbid + LEFT JOIN mb_album_artists maa ON ma.mbid = maa.album_mbid + LEFT JOIN mb_artists mba ON maa.artist_mbid = mba.mbid + WHERE mt.mbid IS NOT NULL + GROUP BY mt.mbid, mt.title, mt.length, ma.cover_art_url, ma.title, ma.mbid, mat.position, mat.disc_number + ON CONFLICT (mbid) + DO UPDATE SET + title = EXCLUDED.title, + artist = EXCLUDED.artist, + image_url = EXCLUDED.image_url, + metadata = EXCLUDED.metadata, + updated_at = NOW() + `; + + try { + const result = await this.pool.query(query); + const count = result.rowCount || 0; + logger.info(`Synced ${count} tracks to music_items`); + return count; + } catch (error: any) { + logger.error('Error syncing tracks', { error: error.message }); + throw error; + } + } + + async syncArtistsToMusicItems(): Promise { + logger.info('Syncing artists from mb_artists to music_items...'); + + const query = ` + INSERT INTO music_items (type, title, artist, image_url, mbid, metadata) + SELECT + 'artist' as type, + mba.name as title, + mba.name as artist, + NULL as image_url, + mba.mbid, + jsonb_build_object( + 'sort_name', mba.sort_name, + 'type', mba.type, + 'area', mba.area, + 'disambiguation', mba.disambiguation + ) as metadata + FROM mb_artists mba + WHERE mba.mbid IS NOT NULL + AND mba.mbid NOT IN ( + SELECT mbid FROM music_items WHERE mbid IS NOT NULL AND type = 'artist' + ) + ON CONFLICT (mbid) + DO UPDATE SET + title = EXCLUDED.title, + artist = EXCLUDED.artist, + metadata = EXCLUDED.metadata, + updated_at = NOW() + `; + + try { + const result = await this.pool.query(query); + const count = result.rowCount || 0; + logger.info(`Synced ${count} artists to music_items`); + return count; + } catch (error: any) { + logger.error('Error syncing artists', { error: error.message }); + throw error; + } + } + + async syncAll(): Promise<{ albums: number; tracks: number; artists: number }> { + logger.info('Starting sync of all MusicBrainz data to music_items...'); + + const albums = await this.syncAlbumsToMusicItems(); + const tracks = await this.syncTracksToMusicItems(); + const artists = await this.syncArtistsToMusicItems(); + + const total = albums + tracks + artists; + logger.info(`Sync completed: ${albums} albums, ${tracks} tracks, ${artists} artists (${total} total)`); + + return { albums, tracks, artists }; + } +} diff --git a/frontend/MusicApp/Views/FeedCardView.swift b/frontend/MusicApp/Views/FeedCardView.swift index bf7e2f46..c88ddf10 100644 --- a/frontend/MusicApp/Views/FeedCardView.swift +++ b/frontend/MusicApp/Views/FeedCardView.swift @@ -8,36 +8,39 @@ struct FeedCardView: View { @State private var isFavorited = false + private var iconName: String { + switch item.type { + case .album: + return "opticaldisc.fill" + case .song: + return "music.note" + case .artist: + return "person.fill" + } + } + + private var iconColor: Color { + switch item.type { + case .album: + return AppColors.primary + case .song: + return AppColors.secondary + case .artist: + return AppColors.accent + } + } + var body: some View { HStack(spacing: 16) { ZStack(alignment: .topTrailing) { - AsyncImage(url: URL(string: item.imageUrl)) { image in - image - .resizable() - .aspectRatio(contentMode: .fill) - } placeholder: { - Rectangle() - .fill(AppColors.secondaryBackground) - .overlay( - ProgressView() - .tint(AppColors.primary) - ) - } - .frame(width: 96, height: 96) - .cornerRadius(AppStyles.cornerRadiusMedium) - .clipped() - - if item.type == .song { - ZStack { - Color.black.opacity(0.4) - .frame(width: 96, height: 96) - .cornerRadius(AppStyles.cornerRadiusMedium) - - Image(systemName: "play.fill") - .font(.system(size: 32)) - .foregroundColor(.white) - } - } + RoundedRectangle(cornerRadius: AppStyles.cornerRadiusMedium) + .fill(AppColors.secondaryBackground) + .frame(width: 96, height: 96) + .overlay( + Image(systemName: iconName) + .font(.system(size: 40)) + .foregroundColor(iconColor) + ) if item.trending == true { ZStack { @@ -147,27 +150,3 @@ struct FeedCardView: View { .cardStyle() } } - -#Preview { - FeedCardView( - item: MusicItem( - id: "1", - type: .album, - title: "ASTROWORLD", - artist: "Travis Scott", - imageUrl: "https://upload.wikimedia.org/wikipedia/en/0/0b/Astroworld_by_Travis_Scott.jpg", - rating: 8.7, - ratingCount: 234500, - trending: true, - trendingChange: 12, - spotifyId: nil, - appleMusicId: nil, - metadata: nil - ), - onRate: {}, - onFavorite: {}, - onComment: {} - ) - .padding() - .background(AppColors.background) -} diff --git a/frontend/MusicApp/Views/RatingModalView.swift b/frontend/MusicApp/Views/RatingModalView.swift index 0257ffee..1d046af5 100644 --- a/frontend/MusicApp/Views/RatingModalView.swift +++ b/frontend/MusicApp/Views/RatingModalView.swift @@ -6,6 +6,28 @@ struct RatingModalView: View { let onClose: () -> Void let onSubmit: (Int, [String]) -> Void + private func iconName(for type: MusicItemType) -> String { + switch type { + case .album: + return "opticaldisc.fill" + case .song: + return "music.note" + case .artist: + return "person.fill" + } + } + + private func iconColor(for type: MusicItemType) -> Color { + switch type { + case .album: + return AppColors.primary + case .song: + return AppColors.secondary + case .artist: + return AppColors.accent + } + } + var body: some View { ZStack { Color.black.opacity(0.8) @@ -39,17 +61,14 @@ struct RatingModalView: View { if let item = item { HStack(spacing: 16) { - AsyncImage(url: URL(string: item.imageUrl)) { image in - image - .resizable() - .aspectRatio(contentMode: .fill) - } placeholder: { - Rectangle() - .fill(AppColors.secondaryBackground) - } - .frame(width: 80, height: 80) - .cornerRadius(AppStyles.cornerRadiusMedium) - .clipped() + RoundedRectangle(cornerRadius: AppStyles.cornerRadiusMedium) + .fill(AppColors.secondaryBackground) + .frame(width: 80, height: 80) + .overlay( + Image(systemName: iconName(for: item.type)) + .font(.system(size: 32)) + .foregroundColor(iconColor(for: item.type)) + ) VStack(alignment: .leading, spacing: 8) { Text(item.title) @@ -240,25 +259,3 @@ struct FlowLayout: Layout { } } } - -#Preview { - RatingModalView( - viewModel: RatingViewModel(), - item: MusicItem( - id: "1", - type: .album, - title: "ASTROWORLD", - artist: "Travis Scott", - imageUrl: "https://upload.wikimedia.org/wikipedia/en/0/0b/Astroworld_by_Travis_Scott.jpg", - rating: 8.7, - ratingCount: 234500, - trending: true, - trendingChange: 12, - spotifyId: nil, - appleMusicId: nil, - metadata: nil - ), - onClose: {}, - onSubmit: { _, _ in } - ) -}