From c7764ce88a06b29547f706c499116a5025d14cc5 Mon Sep 17 00:00:00 2001 From: Ivaylo Badinov Date: Tue, 19 Aug 2025 18:47:20 +0300 Subject: [PATCH 01/14] chore(ci): disable trendminer deploy --- .github/workflows/deploy_develop.yaml | 37 ++++++++++++++------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy_develop.yaml b/.github/workflows/deploy_develop.yaml index 33eb4c3..b366be2 100644 --- a/.github/workflows/deploy_develop.yaml +++ b/.github/workflows/deploy_develop.yaml @@ -34,21 +34,22 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDING_TAGS_API_KEY }} - deploy_trendminer: - name: trendminer - uses: ./.github/workflows/ssh_deploy.yaml - with: - VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "develop-trendminerfun-api-mainnet" - secrets: - AE_NETWORK_ID: "ae_mainnet" - API_HOST_PORT: "3043" - DB_DATABASE: "api" - DEPLOY_HOST: "api.dev.trendminer.fun" - DEPLOY_KEY: ${{ secrets.DEV_TRENDMINERFUN_DEPLOY_KEY }} - DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} - DB_USER: ${{ secrets.DEV_DB_USER }} - DB_PASSWORD: ${{ secrets.DEV_DB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} + # deploy_trendminer: + # name: trendminer + # uses: ./.github/workflows/ssh_deploy.yaml + # with: + # VERSION: ${{ inputs.VERSION }} + # CONTAINER_NAME: "develop-trendminerfun-api-mainnet" + # secrets: + # AE_NETWORK_ID: "ae_mainnet" + # API_HOST_PORT: "3043" + # DB_DATABASE: "api" + # DEPLOY_HOST: "api.dev.trendminer.fun" + # DEPLOY_KEY: ${{ secrets.DEV_TRENDMINERFUN_DEPLOY_KEY }} + # DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} + # DB_USER: ${{ secrets.DEV_DB_USER }} + # DB_PASSWORD: ${{ secrets.DEV_DB_PASSWORD }} + # DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} + # TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDMINERFUN_TRENDING_TAGS_API_KEY }} From 8c284696b576649943a19a102aee5865ce75e4d0 Mon Sep 17 00:00:00 2001 From: Ivaylo Badinov Date: Tue, 19 Aug 2025 19:40:30 +0300 Subject: [PATCH 02/14] chore(ci): enable trendminer deploy --- .github/workflows/deploy_develop.yaml | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy_develop.yaml b/.github/workflows/deploy_develop.yaml index b366be2..694c954 100644 --- a/.github/workflows/deploy_develop.yaml +++ b/.github/workflows/deploy_develop.yaml @@ -34,22 +34,22 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDING_TAGS_API_KEY }} - # deploy_trendminer: - # name: trendminer - # uses: ./.github/workflows/ssh_deploy.yaml - # with: - # VERSION: ${{ inputs.VERSION }} - # CONTAINER_NAME: "develop-trendminerfun-api-mainnet" - # secrets: - # AE_NETWORK_ID: "ae_mainnet" - # API_HOST_PORT: "3043" - # DB_DATABASE: "api" - # DEPLOY_HOST: "api.dev.trendminer.fun" - # DEPLOY_KEY: ${{ secrets.DEV_TRENDMINERFUN_DEPLOY_KEY }} - # DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} - # DB_USER: ${{ secrets.DEV_DB_USER }} - # DB_PASSWORD: ${{ secrets.DEV_DB_PASSWORD }} - # DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - # DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} - # TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDMINERFUN_TRENDING_TAGS_API_KEY }} + deploy_trendminer: + name: trendminer + uses: ./.github/workflows/ssh_deploy.yaml + with: + VERSION: ${{ inputs.VERSION }} + CONTAINER_NAME: "develop-trendminerfun-api-mainnet" + secrets: + AE_NETWORK_ID: "ae_mainnet" + API_HOST_PORT: "3043" + DB_DATABASE: "api" + DEPLOY_HOST: "api.dev.trendminer.fun" + DEPLOY_KEY: ${{ secrets.DEV_TRENDMINERFUN_DEPLOY_KEY }} + DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} + DB_USER: ${{ secrets.DEV_DB_USER }} + DB_PASSWORD: ${{ secrets.DEV_DB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} + TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDMINERFUN_TRENDING_TAGS_API_KEY }} From 75ca65a4a05f59a203b6394b2b7e47fbb9ced1cc Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Sat, 23 Aug 2025 20:20:24 +0100 Subject: [PATCH 03/14] feat: add social posts module with post creation, content parsing, and API endpoints --- src/app.module.ts | 2 + src/app.service.ts | 31 +- src/bcl/bcl.module.ts | 2 +- src/bcl/services/sync-transactions.service.ts | 31 +- src/configs/constants.ts | 14 +- src/social/README.md | 157 ++++++ src/social/config/post-contracts.config.ts | 46 ++ src/social/controllers/posts.controller.ts | 102 ++++ src/social/dto/index.ts | 1 + src/social/dto/post.dto.ts | 75 +++ src/social/entities/post.entity.ts | 44 ++ src/social/interfaces/post.interfaces.ts | 75 +++ src/social/post.module.ts | 17 + src/social/services/post.service.spec.ts | 332 +++++++++++++ src/social/services/post.service.ts | 446 ++++++++++++++++++ src/social/utils/content-parser.util.spec.ts | 283 +++++++++++ src/social/utils/content-parser.util.ts | 147 ++++++ 17 files changed, 1769 insertions(+), 36 deletions(-) create mode 100644 src/social/README.md create mode 100644 src/social/config/post-contracts.config.ts create mode 100644 src/social/controllers/posts.controller.ts create mode 100644 src/social/dto/index.ts create mode 100644 src/social/dto/post.dto.ts create mode 100644 src/social/entities/post.entity.ts create mode 100644 src/social/interfaces/post.interfaces.ts create mode 100644 src/social/post.module.ts create mode 100644 src/social/services/post.service.spec.ts create mode 100644 src/social/services/post.service.ts create mode 100644 src/social/utils/content-parser.util.spec.ts create mode 100644 src/social/utils/content-parser.util.ts diff --git a/src/app.module.ts b/src/app.module.ts index 80b1c16..e8be73f 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -23,6 +23,7 @@ import { AnalyticsModule } from './analytics/analytics.module'; import { AffiliationModule } from './affiliation/affiliation.module'; import { AccountModule } from './account/account.module'; import { TrendingTagsModule } from './trending-tags/trending-tags.module'; +import { PostModule } from './social/post.module'; @Module({ imports: [ @@ -59,6 +60,7 @@ import { TrendingTagsModule } from './trending-tags/trending-tags.module'; AffiliationModule, AccountModule, TrendingTagsModule, + PostModule, ], controllers: [AppController], providers: [AppService], diff --git a/src/app.service.ts b/src/app.service.ts index d19a556..c8293ae 100644 --- a/src/app.service.ts +++ b/src/app.service.ts @@ -2,22 +2,31 @@ import { InjectQueue } from '@nestjs/bull'; import { Injectable } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { Queue } from 'bull'; +import moment, { Moment } from 'moment'; import { AePricingService } from './ae-pricing/ae-pricing.service'; import { CommunityFactoryService } from './ae/community-factory.service'; +import { WebSocketService } from './ae/websocket.service'; +import { SyncTransactionsService } from './bcl/services/sync-transactions.service'; +import { PostService } from './social/services/post.service'; import { DELETE_OLD_TOKENS_QUEUE } from './tokens/queues/constants'; -import moment, { Moment } from 'moment'; +import { ITransaction } from './utils/types'; + @Injectable() export class AppService { startedAt: Moment; constructor( private communityFactoryService: CommunityFactoryService, private aePricingService: AePricingService, + private websocketService: WebSocketService, + private syncTransactionsService: SyncTransactionsService, + private postService: PostService, @InjectQueue(DELETE_OLD_TOKENS_QUEUE) private readonly deleteOldTokensQueue: Queue, ) { this.init(); this.startedAt = moment(); + this.setupLiveSync(); } async init() { @@ -30,6 +39,26 @@ export class AppService { }); } + setupLiveSync() { + let syncedTransactions = []; + + this.websocketService.subscribeForTransactionsUpdates( + (transaction: ITransaction) => { + // Prevent duplicate transactions + if (!syncedTransactions.includes(transaction.hash)) { + this.syncTransactionsService.handleLiveTransaction(transaction); + this.postService.handleLiveTransaction(transaction); + } + syncedTransactions.push(transaction.hash); + + // Reset synced transactions after 100 transactions + if (syncedTransactions.length > 100) { + syncedTransactions = []; + } + }, + ); + } + @Cron(CronExpression.EVERY_HOUR) syncAeCoinPricing() { this.aePricingService.pullAndSaveCoinCurrencyRates(); diff --git a/src/bcl/bcl.module.ts b/src/bcl/bcl.module.ts index ba04445..cdf5ed6 100644 --- a/src/bcl/bcl.module.ts +++ b/src/bcl/bcl.module.ts @@ -43,7 +43,7 @@ import { VerifyTransactionsService } from './services/verify-transactions.servic FixHoldersService, VerifyTransactionsService, ], - exports: [SyncBlocksService], + exports: [SyncBlocksService, SyncTransactionsService], controllers: [DebugFailedTransactionsController], }) export class BclModule { diff --git a/src/bcl/services/sync-transactions.service.ts b/src/bcl/services/sync-transactions.service.ts index 585d784..6c7f6b2 100644 --- a/src/bcl/services/sync-transactions.service.ts +++ b/src/bcl/services/sync-transactions.service.ts @@ -1,5 +1,4 @@ -import { WebSocketService } from '@/ae/websocket.service'; -import { ACTIVE_NETWORK, LIVE_SYNCING_ENABLED, TX_FUNCTIONS } from '@/configs'; +import { ACTIVE_NETWORK, TX_FUNCTIONS } from '@/configs'; import { TransactionService } from '@/transactions/services/transaction.service'; import { fetchJson } from '@/utils/common'; import { ITransaction } from '@/utils/types'; @@ -14,7 +13,6 @@ export class SyncTransactionsService { private readonly logger = new Logger(SyncTransactionsService.name); constructor( - private websocketService: WebSocketService, private readonly transactionService: TransactionService, @InjectRepository(FailedTransaction) @@ -23,31 +21,10 @@ export class SyncTransactionsService { // } - onModuleInit() { - this.setupLiveSync(); - } - - setupLiveSync() { - if (!LIVE_SYNCING_ENABLED) { - return; + async handleLiveTransaction(transaction: ITransaction) { + if (Object.values(TX_FUNCTIONS).includes(transaction.tx.function)) { + this.transactionService.saveTransaction(transaction, null, true); } - let syncedTransactions = []; - - this.websocketService.subscribeForTransactionsUpdates( - (transaction: ITransaction) => { - if (Object.values(TX_FUNCTIONS).includes(transaction.tx.function)) { - // Prevent duplicate transactions - if (!syncedTransactions.includes(transaction.hash)) { - syncedTransactions.push(transaction.hash); - this.transactionService.saveTransaction(transaction, null, true); - } - } - // Reset synced transactions after 100 transactions - if (syncedTransactions.length > 100) { - syncedTransactions = []; - } - }, - ); } async fetchAndSyncTransactions( diff --git a/src/configs/constants.ts b/src/configs/constants.ts index 2b4551b..c9fba9e 100644 --- a/src/configs/constants.ts +++ b/src/configs/constants.ts @@ -99,13 +99,13 @@ export const MAX_RETRIES_FOR_FAILED_TRANSACTIONS = 10; export const MAX_TOKENS_TO_CHECK_WITHOUT_HOLDERS = 20; -export const SYNCING_ENABLED = true; -export const LIVE_SYNCING_ENABLED = true; -export const PERIODIC_SYNCING_ENABLED = true; -export const UPDATE_TRENDING_TOKENS_ENABLED = true; -export const PULL_INVITATIONS_ENABLED = true; -export const PULL_ACCOUNTS_ENABLED = true; -export const PULL_TRENDING_TAGS_ENABLED = true; +export const SYNCING_ENABLED = false; +export const LIVE_SYNCING_ENABLED = false; +export const PERIODIC_SYNCING_ENABLED = false; +export const UPDATE_TRENDING_TOKENS_ENABLED = false; +export const PULL_INVITATIONS_ENABLED = false; +export const PULL_ACCOUNTS_ENABLED = false; +export const PULL_TRENDING_TAGS_ENABLED = false; /** * API Keys and Security diff --git a/src/social/README.md b/src/social/README.md new file mode 100644 index 0000000..e4e102f --- /dev/null +++ b/src/social/README.md @@ -0,0 +1,157 @@ +# Social Posts Module + +This module handles social media posts on the Aeternity blockchain, including post creation, content parsing, and data retrieval. + +## Overview + +The Social Posts module processes blockchain transactions to extract and store social media posts. It supports multiple contract versions and provides comprehensive content parsing, media extraction, and topic analysis. + +## Architecture + +### Core Components + +- **PostService**: Main service handling post processing and storage +- **Post Entity**: Database entity representing social posts +- **PostsController**: REST API endpoints for post retrieval +- **Content Parser**: Utility for parsing post content and extracting metadata +- **Contract Configuration**: Centralized contract management + +### Key Features + +- ✅ Multi-contract support with versioning +- ✅ Real-time transaction processing +- ✅ Content sanitization and validation +- ✅ Topic extraction (hashtags) +- ✅ Media URL validation and extraction +- ✅ Comprehensive error handling and logging +- ✅ Database transaction safety +- ✅ Retry mechanisms for API failures +- ✅ Concurrent processing with locks + +## Configuration + +### Contract Configuration + +Contracts are configured in `src/social/config/post-contracts.config.ts`: + +```typescript +export const POST_CONTRACTS: IPostContract[] = [ + { + contractAddress: 'ct_2Hyt9ZxzXra5NAzhePkRsDPDWppoatVD7CtHnUoHVbuehwR8Nb', + version: 3, + description: 'Current social posting contract' + }, +]; +``` + +### Content Parsing Options + +Content parsing can be customized via `IContentParsingOptions`: + +- `maxTopics`: Maximum number of hashtags to extract (default: 10) +- `maxMediaItems`: Maximum number of media URLs to extract (default: 5) +- `sanitizeContent`: Whether to sanitize content (default: true) + +## API Endpoints + +### GET /posts + +Retrieve paginated list of posts with optional sorting. + +**Query Parameters:** +- `page`: Page number (default: 1) +- `limit`: Items per page (default: 100) +- `order_by`: Sort field ('total_comments' | 'created_at') +- `order_direction`: Sort direction ('ASC' | 'DESC') + +### GET /posts/:id + +Retrieve a specific post by ID. + +## Data Processing Flow + +1. **Transaction Reception**: Live transactions or batch processing from middleware +2. **Contract Validation**: Check if transaction is from supported contract +3. **Content Extraction**: Extract content and metadata from transaction arguments +4. **Content Parsing**: Parse content for topics and media URLs +5. **Validation**: Validate content structure and data integrity +6. **Storage**: Save to database with transaction safety +7. **Error Handling**: Log errors and handle failures gracefully + +## Content Processing + +### Topic Extraction + +- Extracts hashtags starting with '#' +- Converts to lowercase +- Filters invalid characters +- Removes duplicates +- Limits to reasonable length (1-50 characters) + +### Media Validation + +- Validates URL format and protocol (http/https) +- Checks for common media file extensions +- Supports major media hosting platforms +- Limits number of media items per post + +### Content Sanitization + +- Trims whitespace +- Normalizes line endings +- Limits consecutive line breaks +- Enforces maximum content length (5000 chars) + +## Error Handling + +The module implements comprehensive error handling: + +- **Validation Errors**: Invalid transaction data or content +- **Network Errors**: Middleware API failures with retry logic +- **Database Errors**: Transaction rollback and error logging +- **Processing Errors**: Individual transaction failures don't stop batch processing + +## Monitoring and Logging + +Structured logging provides visibility into: + +- Transaction processing status +- Error details with stack traces +- Performance metrics +- Contract processing statistics +- Retry attempts and failures + +## Development + +### Running Tests + +```bash +npm test src/social +``` + +### Adding New Contracts + +1. Add contract configuration to `post-contracts.config.ts` +2. Update contract version handling if needed +3. Test with sample transactions + +### Extending Content Parsing + +1. Modify `content-parser.util.ts` +2. Add new parsing options to interfaces +3. Update tests and documentation + +## Performance Considerations + +- **Concurrent Processing**: Uses processing locks to prevent duplicate work +- **Batch Processing**: Processes multiple transactions efficiently +- **Database Transactions**: Ensures data consistency +- **Error Isolation**: Individual failures don't affect batch processing +- **Retry Logic**: Handles temporary network failures gracefully + +## Security + +- **Input Validation**: All transaction data is validated +- **Content Sanitization**: User content is sanitized before storage +- **URL Validation**: Media URLs are validated for security +- **Error Information**: Sensitive data is not logged in errors diff --git a/src/social/config/post-contracts.config.ts b/src/social/config/post-contracts.config.ts new file mode 100644 index 0000000..3c5ec10 --- /dev/null +++ b/src/social/config/post-contracts.config.ts @@ -0,0 +1,46 @@ +import { IPostContract } from '../interfaces/post.interfaces'; + +/** + * Configuration for supported post contracts + * Each contract represents a different version or type of social posting functionality + */ +export const POST_CONTRACTS: IPostContract[] = [ + // Commented out older contract - kept for reference + // { + // contractAddress: 'ct_2AfnEfCSZCTEkxL5Yoi4Yfq6fF7YapHRaFKDJK3THMXMBspp5z', + // version: 1, + // description: 'Legacy tip/retip contract' + // }, + { + contractAddress: 'ct_2Hyt9ZxzXra5NAzhePkRsDPDWppoatVD7CtHnUoHVbuehwR8Nb', + version: 3, + description: 'Current social posting contract', + }, +]; + +/** + * Get contract configuration by address + */ +export function getContractByAddress( + address: string, +): IPostContract | undefined { + return POST_CONTRACTS.find( + (contract) => contract.contractAddress === address, + ); +} + +/** + * Get all active contract addresses + */ +export function getActiveContractAddresses(): string[] { + return POST_CONTRACTS.map((contract) => contract.contractAddress); +} + +/** + * Check if a contract address is supported + */ +export function isContractSupported(address: string): boolean { + return POST_CONTRACTS.some( + (contract) => contract.contractAddress === address, + ); +} diff --git a/src/social/controllers/posts.controller.ts b/src/social/controllers/posts.controller.ts new file mode 100644 index 0000000..5f3ee5a --- /dev/null +++ b/src/social/controllers/posts.controller.ts @@ -0,0 +1,102 @@ +import { + Controller, + DefaultValuePipe, + Get, + NotFoundException, + Param, + ParseIntPipe, + Query, +} from '@nestjs/common'; +import { + ApiOperation, + ApiParam, + ApiQuery, + ApiTags, + ApiOkResponse, +} from '@nestjs/swagger'; +import { InjectRepository } from '@nestjs/typeorm'; +import { paginate } from 'nestjs-typeorm-paginate'; +import { Repository } from 'typeorm'; +import { Post } from '../entities/post.entity'; +import { PostDto } from '../dto'; +import { ApiOkResponsePaginated } from '@/utils/api-type'; + +@Controller('posts') +@ApiTags('Posts') +export class PostsController { + constructor( + @InjectRepository(Post) + private readonly postRepository: Repository, + ) { + // + } + + @ApiQuery({ name: 'page', type: 'number', required: false }) + @ApiQuery({ name: 'limit', type: 'number', required: false }) + @ApiQuery({ + name: 'order_by', + enum: ['total_comments', 'created_at'], + required: false, + }) + @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) + @ApiQuery({ + name: 'search', + type: 'string', + required: false, + description: 'Search term to filter posts by content or topics', + }) + @ApiOperation({ + operationId: 'listAll', + summary: 'Get all posts', + description: + 'Retrieve a paginated list of all posts with optional sorting and search functionality', + }) + @ApiOkResponsePaginated(PostDto) + @Get() + async listAll( + @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, + @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, + @Query('order_by') orderBy: string = 'created_at', + @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', + @Query('search') search?: string, + ) { + const query = this.postRepository.createQueryBuilder('post'); + + // Add search functionality + if (search) { + const searchTerm = `%${search}%`; + query.where( + '(post.content ILIKE :searchTerm OR CAST(post.topics AS TEXT) ILIKE :searchTerm)', + { searchTerm }, + ); + } + + // Add ordering + if (orderBy) { + query.orderBy(`post.${orderBy}`, orderDirection); + } + + return paginate(query, { page, limit }); + } + + @ApiParam({ name: 'id', type: 'string', description: 'Post ID' }) + @ApiOperation({ + operationId: 'getById', + summary: 'Get post by ID', + description: 'Retrieve a specific post by its unique identifier', + }) + @ApiOkResponse({ + type: PostDto, + description: 'Post retrieved successfully', + }) + @Get(':id') + async getById(@Param('id') id: string) { + const post = await this.postRepository.findOne({ + where: { id }, + }); + if (!post) { + throw new NotFoundException(`Post with ID ${id} not found`); + } + return post; + } +} diff --git a/src/social/dto/index.ts b/src/social/dto/index.ts new file mode 100644 index 0000000..4171984 --- /dev/null +++ b/src/social/dto/index.ts @@ -0,0 +1 @@ +export { PostDto } from './post.dto'; diff --git a/src/social/dto/post.dto.ts b/src/social/dto/post.dto.ts new file mode 100644 index 0000000..983ab08 --- /dev/null +++ b/src/social/dto/post.dto.ts @@ -0,0 +1,75 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class PostDto { + @ApiProperty({ + description: 'Unique identifier for the post', + example: '12345_v1', + }) + id: string; + + @ApiProperty({ + description: 'Transaction hash associated with the post', + example: 'th_1234567890abcdef...', + }) + tx_hash: string; + + @ApiProperty({ + description: 'Transaction arguments as JSON array', + type: 'array', + items: { type: 'object' }, + example: [{ type: 'string', value: 'Hello world!' }], + }) + tx_args: any[]; + + @ApiProperty({ + description: 'Address of the post sender/creator', + example: 'ak_2a1j2Mk9YSmC1gioUq4PWRm3bsv887MbuRVwyv4KaUGoR1eiKi', + }) + sender_address: string; + + @ApiProperty({ + description: 'Address of the smart contract', + example: 'ct_2AfnEfCSPx4A6UYXj2XHDqHXcC7EF2bgbp8UN1KPAJDysPJT32', + }) + contract_address: string; + + @ApiProperty({ + description: 'Type of the post/transaction', + example: 'post', + }) + type: string; + + @ApiProperty({ + description: 'Main content of the post', + example: 'Hello world! This is my first post on the blockchain.', + }) + content: string; + + @ApiProperty({ + description: 'Array of topics/hashtags associated with the post', + type: [String], + example: ['#blockchain', '#hello', '#firstpost'], + }) + topics: string[]; + + @ApiProperty({ + description: 'Array of media URLs associated with the post', + type: [String], + example: ['https://example.com/image.jpg', 'https://example.com/video.mp4'], + }) + media: string[]; + + @ApiProperty({ + description: 'Total number of comments on this post', + example: 5, + }) + total_comments: number; + + @ApiProperty({ + description: 'Timestamp when the post was created', + type: 'string', + format: 'date-time', + example: '2023-12-01T10:30:00.000Z', + }) + created_at: Date; +} diff --git a/src/social/entities/post.entity.ts b/src/social/entities/post.entity.ts new file mode 100644 index 0000000..51a2ca8 --- /dev/null +++ b/src/social/entities/post.entity.ts @@ -0,0 +1,44 @@ +import { Column, CreateDateColumn, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ + name: 'posts', +}) +export class Post { + @PrimaryColumn() + id: string; + + @Column({ + unique: true, + }) + tx_hash: string; + + @Column('json') + tx_args: any[]; + + @Column() + sender_address: string; + + @Column() + contract_address: string; + + @Column() + type: string; + + @Column() + content: string; + + @Column('json', { default: [] }) + topics: string[]; + + @Column('json', { default: [] }) + media: string[]; + + @Column() + total_comments: number; + + @CreateDateColumn({ + type: 'timestamp', + default: () => 'CURRENT_TIMESTAMP(6)', + }) + public created_at: Date; +} diff --git a/src/social/interfaces/post.interfaces.ts b/src/social/interfaces/post.interfaces.ts new file mode 100644 index 0000000..3c777f8 --- /dev/null +++ b/src/social/interfaces/post.interfaces.ts @@ -0,0 +1,75 @@ +import { ITransaction } from '@/utils/types'; + +/** + * Configuration for a post contract + */ +export interface IPostContract { + contractAddress: string; + version: number; + description?: string; +} + +/** + * Parsed post content with extracted metadata + */ +export interface IParsedPostContent { + content: string; + topics: string[]; + media: string[]; +} + +/** + * Data structure for creating a new post + */ +export interface ICreatePostData { + id: string; + type: string; + tx_hash: string; + sender_address: string; + contract_address: string; + content: string; + topics: string[]; + media: string[]; + total_comments: number; + tx_args: any[]; + created_at: Date; +} + +/** + * Result of post processing operation + */ +export interface IPostProcessingResult { + success: boolean; + post?: any; + error?: string; + skipped?: boolean; + reason?: string; +} + +/** + * Configuration for middleware API requests + */ +export interface IMiddlewareRequestConfig { + direction: 'forward' | 'backward'; + limit: number; + type: string; + contract: string; +} + +/** + * Response structure from middleware API + */ +export interface IMiddlewareResponse { + data: ITransaction[]; + next?: string; + prev?: string; +} + +/** + * Options for content parsing + */ +export interface IContentParsingOptions { + maxTopics?: number; + maxMediaItems?: number; + sanitizeContent?: boolean; +} diff --git a/src/social/post.module.ts b/src/social/post.module.ts new file mode 100644 index 0000000..4429bb7 --- /dev/null +++ b/src/social/post.module.ts @@ -0,0 +1,17 @@ +import { AeModule } from '@/ae/ae.module'; +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { Post } from './entities/post.entity'; +import { PostService } from './services/post.service'; +import { TransactionsModule } from '@/transactions/transactions.module'; +import { PostsController } from './controllers/posts.controller'; + +@Module({ + imports: [AeModule, TransactionsModule, TypeOrmModule.forFeature([Post])], + providers: [PostService], + exports: [PostService], + controllers: [PostsController], +}) +export class PostModule { + // +} diff --git a/src/social/services/post.service.spec.ts b/src/social/services/post.service.spec.ts new file mode 100644 index 0000000..7f3eff9 --- /dev/null +++ b/src/social/services/post.service.spec.ts @@ -0,0 +1,332 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { PostService } from './post.service'; +import { Post } from '../entities/post.entity'; +import { ITransaction } from '@/utils/types'; +import { Logger } from '@nestjs/common'; + +// Mock the external dependencies +jest.mock('@/utils/common'); +jest.mock('../config/post-contracts.config'); +jest.mock('../utils/content-parser.util'); + +describe('PostService', () => { + let service: PostService; + let repository: jest.Mocked>; + let logger: jest.Mocked; + + const mockRepository = { + findOne: jest.fn(), + create: jest.fn(), + save: jest.fn(), + manager: { + transaction: jest.fn(), + }, + }; + + const createMockTransaction = ( + overrides: Partial = {}, + ): ITransaction => ({ + blockHeight: 123456, + claim: null, + hash: 'th_testHash123', + microIndex: 1, + microTime: Date.now(), + pending: false, + tx: { + abiVersion: 1, + amount: 0, + microTime: Date.now(), + arguments: [], + callerId: 'ak_testCaller123', + code: '', + commitmentId: null, + contractId: 'ct_testContract123', + fee: 1000, + gas: 5000, + gasPrice: 1000000000, + gasUsed: 3000, + name: null, + nameFee: 0, + nameId: null, + nameSalt: '', + nonce: 1, + pointers: null, + result: 'ok', + return: { type: 'tuple', value: 'test-return-value' }, + returnType: 'ok', + type: 'ContractCallTx' as const, + VSN: '1', + }, + ...overrides, + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + PostService, + { + provide: getRepositoryToken(Post), + useValue: mockRepository, + }, + ], + }).compile(); + + service = module.get(PostService); + repository = module.get(getRepositoryToken(Post)); + logger = service['logger'] as jest.Mocked; + + // Mock logger methods + logger.log = jest.fn(); + logger.error = jest.fn(); + logger.warn = jest.fn(); + logger.debug = jest.fn(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('validateTransaction', () => { + it('should return false for null transaction', () => { + const result = service['validateTransaction'](null as any); + expect(result).toBe(false); + }); + + it('should return false for transaction without required fields', () => { + const transaction = {} as ITransaction; + const result = service['validateTransaction'](transaction); + expect(result).toBe(false); + }); + + it('should return true for valid transaction', () => { + const transaction = createMockTransaction({ + tx: { + ...createMockTransaction().tx, + callerId: 'ak_testCaller', + contractId: 'ct_testContract', + arguments: [], + }, + }); + + const result = service['validateTransaction'](transaction); + expect(result).toBe(true); + }); + }); + + describe('generatePostId', () => { + it('should generate ID with return value when available', () => { + const transaction = createMockTransaction({ + tx: { + ...createMockTransaction().tx, + return: { + type: 'tuple', + value: 'return-value', + }, + }, + }); + + const contract = { version: 3, contractAddress: 'test' }; + const result = service['generatePostId'](transaction, contract); + + expect(result).toBe('return-value_v3'); + }); + + it('should generate fallback ID when return value is not available', () => { + const transaction = createMockTransaction({ + hash: 'th_testHash12345678', + tx: { + ...createMockTransaction().tx, + return: undefined as any, + }, + }); + + const contract = { version: 3, contractAddress: 'test' }; + const result = service['generatePostId'](transaction, contract); + + expect(result).toBe('12345678_v3'); + }); + }); + + describe('handleLiveTransaction', () => { + it('should return error for transaction without contract ID', async () => { + const transaction = createMockTransaction({ + tx: { + ...createMockTransaction().tx, + contractId: undefined as any, + }, + }); + + const result = await service.handleLiveTransaction(transaction); + + expect(result.success).toBe(false); + expect(result.error).toBe('Missing contract ID or unsupported contract'); + expect(result.skipped).toBe(true); + }); + + it('should return error for unsupported contract', async () => { + // Mock the contract support check + const { + isContractSupported, + } = require('../config/post-contracts.config'); + isContractSupported.mockReturnValue(false); + + const transaction = createMockTransaction({ + tx: { + ...createMockTransaction().tx, + contractId: 'ct_unsupportedContract', + }, + }); + + const result = await service.handleLiveTransaction(transaction); + + expect(result.success).toBe(false); + expect(result.error).toBe('Missing contract ID or unsupported contract'); + expect(result.skipped).toBe(true); + }); + + it('should process supported contract successfully', async () => { + // Mock the contract support and configuration + const { + isContractSupported, + getContractByAddress, + } = require('../config/post-contracts.config'); + isContractSupported.mockReturnValue(true); + getContractByAddress.mockReturnValue({ + contractAddress: 'ct_testContract', + version: 3, + }); + + const mockPost = { id: 'test-post-id' }; + jest + .spyOn(service, 'savePostFromTransaction') + .mockResolvedValue(mockPost as Post); + + const transaction = createMockTransaction({ + tx: { + ...createMockTransaction().tx, + contractId: 'ct_testContract', + }, + }); + + const result = await service.handleLiveTransaction(transaction); + + expect(result.success).toBe(true); + expect(result.post).toBe(mockPost); + }); + }); + + describe('savePostFromTransaction', () => { + it('should return null for invalid transaction', async () => { + jest.spyOn(service as any, 'validateTransaction').mockReturnValue(false); + + const result = await service.savePostFromTransaction({} as ITransaction, { + contractAddress: 'test', + version: 3, + }); + + expect(result).toBeNull(); + }); + + it('should return existing post if already exists', async () => { + jest.spyOn(service as any, 'validateTransaction').mockReturnValue(true); + + const existingPost = { id: 'existing-post', tx_hash: 'th_testHash' }; + repository.findOne.mockResolvedValue(existingPost as Post); + + const transaction = createMockTransaction({ + hash: 'th_testHash', + tx: { + ...createMockTransaction().tx, + arguments: [{ type: 'tuple', value: 'test content' }], + }, + }); + + const result = await service.savePostFromTransaction(transaction, { + contractAddress: 'test', + version: 3, + }); + + expect(result).toBe(existingPost); + }); + + it('should create new post for valid transaction', async () => { + jest.spyOn(service as any, 'validateTransaction').mockReturnValue(true); + jest + .spyOn(service as any, 'generatePostId') + .mockReturnValue('new-post-id'); + + // Mock content parser + const { parsePostContent } = require('../utils/content-parser.util'); + parsePostContent.mockReturnValue({ + content: 'parsed content', + topics: ['#test'], + media: [], + }); + + repository.findOne.mockResolvedValue(null); + + const newPost = { id: 'new-post-id', tx_hash: 'th_testHash' }; + const mockManager = { + create: jest.fn().mockReturnValue(newPost), + save: jest.fn().mockResolvedValue(newPost), + }; + (repository.manager.transaction as jest.Mock).mockImplementation( + (callback) => callback(mockManager), + ); + + const transaction = createMockTransaction({ + hash: 'th_testHash', + tx: { + ...createMockTransaction().tx, + callerId: 'ak_testCaller', + contractId: 'ct_testContract', + function: 'create_community', + arguments: [ + { type: 'tuple', value: 'test content' }, + { type: 'list', value: [] }, + ], + }, + }); + + const result = await service.savePostFromTransaction(transaction, { + contractAddress: 'ct_testContract', + version: 3, + }); + + expect(result).toBe(newPost); + expect(parsePostContent).toHaveBeenCalledWith('test content', []); + }); + }); +}); + +/** + * Additional test cases to implement: + * + * 1. loadPostsFromMdw tests: + * - Successful data loading + * - Retry mechanism on failures + * - Pagination handling + * - Empty response handling + * + * 2. pullLatestPosts tests: + * - Processing lock behavior + * - Error handling and recovery + * - URL construction + * + * 3. pullLatestPostsForContracts tests: + * - Multiple contract processing + * - Parallel execution + * - Error aggregation + * + * 4. Integration tests: + * - End-to-end transaction processing + * - Database interaction testing + * - Error scenarios + * + * 5. Performance tests: + * - Large batch processing + * - Memory usage + * - Concurrent request handling + */ diff --git a/src/social/services/post.service.ts b/src/social/services/post.service.ts new file mode 100644 index 0000000..bde6819 --- /dev/null +++ b/src/social/services/post.service.ts @@ -0,0 +1,446 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { Post } from '../entities/post.entity'; +import { + ACTIVE_NETWORK, + MAX_RETRIES_WHEN_REQUEST_FAILED, + WAIT_TIME_WHEN_REQUEST_FAILED, +} from '@/configs'; +import { fetchJson } from '@/utils/common'; +import moment from 'moment'; +import { ITransaction } from '@/utils/types'; +import camelcaseKeysDeep from 'camelcase-keys-deep'; +import { + POST_CONTRACTS, + getContractByAddress, + isContractSupported, +} from '../config/post-contracts.config'; +import { + IPostContract, + ICreatePostData, + IPostProcessingResult, + IMiddlewareResponse, + IMiddlewareRequestConfig, +} from '../interfaces/post.interfaces'; +import { parsePostContent } from '../utils/content-parser.util'; + +@Injectable() +export class PostService { + private readonly logger = new Logger(PostService.name); + private readonly isProcessing = new Map(); + + constructor( + @InjectRepository(Post) + private readonly postRepository: Repository, + ) { + this.logger.log('PostService initialized'); + } + + async onModuleInit(): Promise { + this.logger.log('Initializing PostService module...'); + try { + await this.pullLatestPostsForContracts(); + this.logger.log('PostService module initialized successfully'); + } catch (error) { + this.logger.error('Failed to initialize PostService module', error); + // Don't throw - allow the service to start even if initial sync fails + } + } + + async handleLiveTransaction( + transaction: ITransaction, + ): Promise { + const contractAddress = transaction?.tx?.contractId; + + if (!contractAddress || !isContractSupported(contractAddress)) { + return { + success: false, + error: 'Missing contract ID or unsupported contract', + skipped: true, + }; + } + + const contract = getContractByAddress(contractAddress); + if (!contract) { + this.logger.error('Contract configuration not found', { + contractAddress, + }); + return { success: false, error: 'Contract configuration missing' }; + } + + try { + const post = await this.savePostFromTransaction(transaction, contract); + this.logger.log('Live transaction processed successfully', { + hash: transaction.hash, + contractAddress, + postId: post?.id, + }); + return { success: true, post }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + const errorStack = error instanceof Error ? error.stack : undefined; + + this.logger.error('Failed to process live transaction', { + hash: transaction.hash, + contractAddress, + error: errorMessage, + stack: errorStack, + }); + return { success: false, error: errorMessage }; + } + } + + async pullLatestPostsForContracts(): Promise { + const contractsProcessingKey = 'all_contracts'; + + if (this.isProcessing.get(contractsProcessingKey)) { + this.logger.warn('Contract processing already in progress, skipping...'); + return; + } + + this.isProcessing.set(contractsProcessingKey, true); + + try { + this.logger.log( + `Starting to pull posts for ${POST_CONTRACTS.length} contracts`, + ); + + const results = await Promise.allSettled( + POST_CONTRACTS.map((contract) => this.pullLatestPosts(contract)), + ); + + const successful = results.filter((r) => r.status === 'fulfilled').length; + const failed = results.filter((r) => r.status === 'rejected').length; + + this.logger.log( + `Contract processing completed: ${successful} successful, ${failed} failed`, + ); + + // Log any failures + results.forEach((result, index) => { + if (result.status === 'rejected') { + const error = + result.reason instanceof Error + ? result.reason + : new Error(String(result.reason)); + this.logger.error( + `Failed to process contract ${POST_CONTRACTS[index].contractAddress}`, + { + error: error.message, + stack: error.stack, + }, + ); + } + }); + } finally { + this.isProcessing.delete(contractsProcessingKey); + } + } + + async pullLatestPosts(contract: IPostContract): Promise { + const processingKey = `contract_${contract.contractAddress}`; + + if (this.isProcessing.get(processingKey)) { + this.logger.warn( + `Contract ${contract.contractAddress} already being processed, skipping...`, + ); + return []; + } + + this.isProcessing.set(processingKey, true); + + try { + const config: IMiddlewareRequestConfig = { + direction: 'backward', + limit: 100, + type: 'contract_call', + contract: contract.contractAddress, + }; + + const queryString = new URLSearchParams({ + direction: config.direction, + limit: config.limit.toString(), + type: config.type, + contract: config.contract, + }).toString(); + + const url = `${ACTIVE_NETWORK.middlewareUrl}/v3/transactions?${queryString}`; + + this.logger.log( + `Pulling latest posts for contract ${contract.contractAddress}`, + { url }, + ); + + const posts = await this.loadPostsFromMdw(url, contract); + + this.logger.log( + `Successfully pulled ${posts.length} posts for contract ${contract.contractAddress}`, + ); + + return posts; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + const errorStack = error instanceof Error ? error.stack : undefined; + + this.logger.error( + `Failed to pull posts for contract ${contract.contractAddress}`, + { + error: errorMessage, + stack: errorStack, + }, + ); + throw error; + } finally { + this.isProcessing.delete(processingKey); + } + } + + async loadPostsFromMdw( + url: string, + contract: IPostContract, + posts: any[] = [], + totalRetries = 0, + ): Promise { + let result: IMiddlewareResponse; + + try { + result = await fetchJson(url); + } catch (error) { + if (totalRetries < MAX_RETRIES_WHEN_REQUEST_FAILED) { + const nextRetry = totalRetries + 1; + const errorMessage = + error instanceof Error ? error.message : String(error); + + this.logger.warn( + `Middleware request failed, retrying (${nextRetry}/${MAX_RETRIES_WHEN_REQUEST_FAILED})`, + { + url, + error: errorMessage, + retryIn: WAIT_TIME_WHEN_REQUEST_FAILED, + }, + ); + + await new Promise((resolve) => + setTimeout(resolve, WAIT_TIME_WHEN_REQUEST_FAILED), + ); + return this.loadPostsFromMdw(url, contract, posts, nextRetry); + } + + const errorMessage = + error instanceof Error ? error.message : String(error); + const errorStack = error instanceof Error ? error.stack : undefined; + + this.logger.error( + 'Failed to load posts from middleware after all retries', + { + url, + error: errorMessage, + stack: errorStack, + totalRetries, + }, + ); + return posts; + } + + if (!result?.data?.length) { + this.logger.debug('No data received from middleware', { url }); + return posts; + } + + // Process transactions with better error handling + const processingResults = await Promise.allSettled( + result.data.map(async (transaction) => { + try { + const camelCasedTransaction = camelcaseKeysDeep( + transaction, + ) as ITransaction; + const post = await this.savePostFromTransaction( + camelCasedTransaction, + contract, + ); + return post; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.warn('Failed to process individual transaction', { + txHash: transaction?.hash, + error: errorMessage, + }); + throw error; + } + }), + ); + + // Collect successful results + const successfulPosts = processingResults + .filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled', + ) + .map((result) => result.value) + .filter((post) => post !== null); + + posts.push(...successfulPosts); + + const failedCount = processingResults.filter( + (r) => r.status === 'rejected', + ).length; + if (failedCount > 0) { + this.logger.warn( + `${failedCount} transactions failed to process in this batch`, + ); + } + + // Continue with pagination if available + if (result.next) { + const nextUrl = `${ACTIVE_NETWORK.middlewareUrl}${result.next}`; + return this.loadPostsFromMdw(nextUrl, contract, posts, 0); + } + + return posts; + } + + async savePostFromTransaction( + transaction: ITransaction, + contract: IPostContract, + ): Promise { + if (!this.validateTransaction(transaction)) { + this.logger.warn('Invalid transaction data', { + hash: transaction?.hash, + }); + return null; + } + + const txHash = transaction.hash; + + try { + // Check if post already exists + const existingPost = await this.postRepository.findOne({ + where: { tx_hash: txHash }, + }); + + if (existingPost) { + return existingPost; + } + + // Validate required transaction data + if (!transaction.tx?.arguments?.[0]?.value) { + this.logger.warn('Transaction missing content argument', { txHash }); + return null; + } + + const content = transaction.tx.arguments[0].value; + if (typeof content !== 'string' || content.trim().length === 0) { + this.logger.warn('Invalid or empty content', { txHash }); + return null; + } + + // Parse content and extract metadata + const parsedContent = parsePostContent( + content, + transaction.tx.arguments[1]?.value || [], + ); + + // Create post data with proper validation + const postData: ICreatePostData = { + id: this.generatePostId(transaction, contract), + type: transaction.tx.function || 'unknown', + tx_hash: txHash, + sender_address: transaction.tx.callerId, + contract_address: transaction.tx.contractId, + content: parsedContent.content, + topics: parsedContent.topics, + media: parsedContent.media, + total_comments: 0, + tx_args: transaction.tx.arguments, + created_at: moment(transaction.microTime).toDate(), + }; + + this.logger.debug('Creating new post', { + txHash, + postId: postData.id, + topicsCount: postData.topics.length, + mediaCount: postData.media.length, + }); + + // Use database transaction for consistency + const post = await this.postRepository.manager.transaction( + async (manager) => { + const newPost = manager.create(Post, postData); + return await manager.save(newPost); + }, + ); + + this.logger.log('Post saved successfully', { + txHash, + postId: post.id, + contractAddress: contract.contractAddress, + }); + + return post; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + const errorStack = error instanceof Error ? error.stack : undefined; + + this.logger.error('Failed to save post from transaction', { + txHash, + contractAddress: contract.contractAddress, + error: errorMessage, + stack: errorStack, + }); + throw error; + } + } + + /** + * Validates transaction data structure + */ + private validateTransaction(transaction: ITransaction): boolean { + if (!transaction) { + return false; + } + + const requiredFields = ['hash', 'microTime']; + for (const field of requiredFields) { + if (!transaction[field]) { + this.logger.warn(`Transaction missing required field: ${field}`); + return false; + } + } + + if (!transaction.tx) { + this.logger.warn('Transaction missing tx data'); + return false; + } + + const requiredTxFields = ['callerId', 'contractId', 'arguments']; + for (const field of requiredTxFields) { + if (!transaction.tx[field]) { + this.logger.warn(`Transaction.tx missing required field: ${field}`); + return false; + } + } + + return true; + } + + /** + * Generates a unique post ID + */ + private generatePostId( + transaction: ITransaction, + contract: IPostContract, + ): string { + const returnValue = transaction.tx?.return?.value; + if (returnValue) { + return `${returnValue}_v${contract.version}`; + } + + // Fallback to hash-based ID if return value is not available + return `${transaction.hash.slice(-8)}_v${contract.version}`; + } +} diff --git a/src/social/utils/content-parser.util.spec.ts b/src/social/utils/content-parser.util.spec.ts new file mode 100644 index 0000000..382b10a --- /dev/null +++ b/src/social/utils/content-parser.util.spec.ts @@ -0,0 +1,283 @@ +import { + parsePostContent, + extractTopics, + extractMedia, + sanitizeContent, + isValidMediaUrl, +} from './content-parser.util'; + +describe('Content Parser Utilities', () => { + describe('parsePostContent', () => { + it('should parse content with topics and media', () => { + const content = 'Hello #world #test this is a post'; + const mediaArguments = [ + { value: 'https://example.com/image.jpg' }, + { value: 'https://example.com/video.mp4' }, + ]; + + const result = parsePostContent(content, mediaArguments); + + expect(result.content).toBe(content); + expect(result.topics).toEqual(['#world', '#test']); + expect(result.media).toEqual([ + 'https://example.com/image.jpg', + 'https://example.com/video.mp4', + ]); + }); + + it('should handle empty content and media', () => { + const result = parsePostContent('', []); + + expect(result.content).toBe(''); + expect(result.topics).toEqual([]); + expect(result.media).toEqual([]); + }); + + it('should apply custom options', () => { + const content = '#one #two #three #four #five'; + const options = { maxTopics: 2, sanitizeContent: false }; + + const result = parsePostContent(content, [], options); + + expect(result.topics).toHaveLength(2); + }); + }); + + describe('extractTopics', () => { + it('should extract hashtags from content', () => { + const content = 'Check out this #awesome #blockchain #dapp'; + const topics = extractTopics(content); + + expect(topics).toEqual(['#awesome', '#blockchain', '#dapp']); + }); + + it('should handle mixed case and convert to lowercase', () => { + const content = 'Testing #CamelCase #UPPERCASE #lowercase'; + const topics = extractTopics(content); + + expect(topics).toEqual(['#camelcase', '#uppercase', '#lowercase']); + }); + + it('should filter out invalid hashtags', () => { + const content = '# #valid_tag #123 #'; + const topics = extractTopics(content); + + expect(topics).toEqual(['#valid_tag', '#123']); + }); + + it('should remove duplicates while preserving order', () => { + const content = '#first #second #first #third #second'; + const topics = extractTopics(content); + + expect(topics).toEqual(['#first', '#second', '#third']); + }); + + it('should respect maxTopics limit', () => { + const content = '#one #two #three #four #five'; + const topics = extractTopics(content, 3); + + expect(topics).toHaveLength(3); + expect(topics).toEqual(['#one', '#two', '#three']); + }); + + it('should handle empty or invalid input', () => { + expect(extractTopics('')).toEqual([]); + expect(extractTopics(null as any)).toEqual([]); + expect(extractTopics(undefined as any)).toEqual([]); + expect(extractTopics(123 as any)).toEqual([]); + }); + + it('should filter out very long hashtags', () => { + const longTag = '#' + 'a'.repeat(60); // 61 characters total + const validTag = '#valid'; + const content = `${longTag} ${validTag}`; + + const topics = extractTopics(content); + + expect(topics).toEqual(['#valid']); + }); + }); + + describe('extractMedia', () => { + it('should extract valid media URLs', () => { + const mediaArguments = [ + { value: 'https://example.com/image.jpg' }, + { value: 'https://example.com/video.mp4' }, + { value: 'not-a-url' }, + { value: null }, + ]; + + const media = extractMedia(mediaArguments); + + expect(media).toEqual([ + 'https://example.com/image.jpg', + 'https://example.com/video.mp4', + ]); + }); + + it('should respect maxMediaItems limit', () => { + const mediaArguments = [ + { value: 'https://example.com/1.jpg' }, + { value: 'https://example.com/2.jpg' }, + { value: 'https://example.com/3.jpg' }, + ]; + + const media = extractMedia(mediaArguments, 2); + + expect(media).toHaveLength(2); + }); + + it('should handle invalid input gracefully', () => { + expect(extractMedia(null as any)).toEqual([]); + expect(extractMedia(undefined as any)).toEqual([]); + expect(extractMedia('not-array' as any)).toEqual([]); + }); + + it('should handle extraction errors', () => { + const mediaArguments = [ + { + value: { + toString: () => { + throw new Error('Test error'); + }, + }, + }, + { value: 'https://example.com/valid.jpg' }, + ]; + + const media = extractMedia(mediaArguments); + + expect(media).toEqual(['https://example.com/valid.jpg']); + }); + }); + + describe('sanitizeContent', () => { + it('should trim whitespace', () => { + const content = ' Hello world '; + const sanitized = sanitizeContent(content); + + expect(sanitized).toBe('Hello world'); + }); + + it('should normalize line endings', () => { + const content = 'Line 1\r\nLine 2\r\nLine 3'; + const sanitized = sanitizeContent(content); + + expect(sanitized).toBe('Line 1\nLine 2\nLine 3'); + }); + + it('should limit consecutive line breaks', () => { + const content = 'Line 1\n\n\n\n\nLine 2'; + const sanitized = sanitizeContent(content); + + expect(sanitized).toBe('Line 1\n\nLine 2'); + }); + + it('should enforce maximum length', () => { + const content = 'a'.repeat(6000); + const sanitized = sanitizeContent(content); + + expect(sanitized).toHaveLength(5000); + }); + + it('should handle invalid input', () => { + expect(sanitizeContent(null as any)).toBe(''); + expect(sanitizeContent(undefined as any)).toBe(''); + expect(sanitizeContent(123 as any)).toBe(''); + }); + }); + + describe('isValidMediaUrl', () => { + it('should validate URLs with media extensions', () => { + const validUrls = [ + 'https://example.com/image.jpg', + 'http://example.com/image.jpeg', + 'https://example.com/image.png', + 'https://example.com/image.gif', + 'https://example.com/image.webp', + 'https://example.com/video.mp4', + 'https://example.com/video.webm', + 'https://example.com/video.mov', + ]; + + validUrls.forEach((url) => { + expect(isValidMediaUrl(url)).toBe(true); + }); + }); + + it('should validate URLs from known media hosts', () => { + const mediaHostUrls = [ + 'https://imgur.com/gallery/abc123', + 'https://giphy.com/gifs/abc123', + 'https://youtube.com/watch?v=abc123', + 'https://vimeo.com/123456789', + 'https://i.imgur.com/abc123', + ]; + + mediaHostUrls.forEach((url) => { + expect(isValidMediaUrl(url)).toBe(true); + }); + }); + + it('should reject invalid URLs', () => { + const invalidUrls = [ + 'not-a-url', + 'ftp://example.com/file.jpg', + 'https://example.com/document.pdf', + 'https://example.com/page.html', + '', + null, + undefined, + ]; + + invalidUrls.forEach((url) => { + expect(isValidMediaUrl(url as any)).toBe(false); + }); + }); + + it('should handle malformed URLs gracefully', () => { + const malformedUrls = [ + 'https://', + 'https://.', + 'https://example', + 'https://example.', + 'javascript:alert(1)', + ]; + + malformedUrls.forEach((url) => { + expect(isValidMediaUrl(url)).toBe(false); + }); + }); + + it('should require valid protocols', () => { + expect(isValidMediaUrl('ftp://example.com/image.jpg')).toBe(false); + expect(isValidMediaUrl('file:///local/image.jpg')).toBe(false); + expect(isValidMediaUrl('data:image/png;base64,abc')).toBe(false); + }); + }); +}); + +/** + * Additional test cases to consider: + * + * 1. Edge cases: + * - Very large content strings + * - Unicode characters in hashtags + * - International domain names + * - URL encoding in media URLs + * + * 2. Security tests: + * - XSS prevention in content + * - Malicious URL detection + * - Script injection attempts + * + * 3. Performance tests: + * - Large number of hashtags + * - Large media arrays + * - Complex regex patterns + * + * 4. Integration tests: + * - Real-world content examples + * - Multi-language content + * - Mixed content types + */ diff --git a/src/social/utils/content-parser.util.ts b/src/social/utils/content-parser.util.ts new file mode 100644 index 0000000..661febe --- /dev/null +++ b/src/social/utils/content-parser.util.ts @@ -0,0 +1,147 @@ +import { + IParsedPostContent, + IContentParsingOptions, +} from '../interfaces/post.interfaces'; + +/** + * Default options for content parsing + */ +const DEFAULT_PARSING_OPTIONS: Required = { + maxTopics: 10, + maxMediaItems: 5, + sanitizeContent: true, +}; + +/** + * Parses post content and extracts topics and media + */ +export function parsePostContent( + content: string, + mediaArguments: any[] = [], + options: IContentParsingOptions = {}, +): IParsedPostContent { + const config = { ...DEFAULT_PARSING_OPTIONS, ...options }; + + // Sanitize content if requested + const sanitizedContent = config.sanitizeContent + ? sanitizeContent(content) + : content; + + // Extract topics (hashtags) + const topics = extractTopics(sanitizedContent, config.maxTopics); + + // Extract media URLs + const media = extractMedia(mediaArguments, config.maxMediaItems); + + return { + content: sanitizedContent, + topics, + media, + }; +} + +/** + * Extracts hashtags from content + */ +export function extractTopics( + content: string, + maxTopics: number = 10, +): string[] { + if (!content || typeof content !== 'string') { + return []; + } + + const topics = content + .split(/\s+/) + .filter((word) => word.startsWith('#') && word.length > 1) + .map((topic) => topic.toLowerCase().replace(/[^a-z0-9#_]/g, '')) + .filter((topic) => topic.length > 1 && topic.length <= 50) // Reasonable length limits + .slice(0, maxTopics); + + // Remove duplicates while preserving order + return [...new Set(topics)]; +} + +/** + * Extracts media URLs from transaction arguments + */ +export function extractMedia( + mediaArguments: any[] = [], + maxMediaItems: number = 5, +): string[] { + if (!Array.isArray(mediaArguments)) { + return []; + } + + try { + const media = mediaArguments + .map((item) => item?.value) + .filter((value) => value && typeof value === 'string') + .filter((url) => isValidMediaUrl(url)) + .slice(0, maxMediaItems); + + return media; + } catch (error) { + console.warn('Error extracting media from arguments:', error); + return []; + } +} + +/** + * Basic content sanitization + */ +export function sanitizeContent(content: string): string { + if (!content || typeof content !== 'string') { + return ''; + } + + return content + .trim() + .replace(/\r\n/g, '\n') // Normalize line endings + .replace(/\n{3,}/g, '\n\n') // Limit consecutive line breaks + .slice(0, 5000); // Reasonable length limit +} + +/** + * Validates if a URL appears to be a valid media URL + */ +export function isValidMediaUrl(url: string): boolean { + if (!url || typeof url !== 'string') { + return false; + } + + try { + const parsedUrl = new URL(url); + const validProtocols = ['http:', 'https:']; + const mediaExtensions = [ + '.jpg', + '.jpeg', + '.png', + '.gif', + '.webp', + '.mp4', + '.webm', + '.mov', + ]; + + const hasValidProtocol = validProtocols.includes(parsedUrl.protocol); + const hasMediaExtension = mediaExtensions.some((ext) => + parsedUrl.pathname.toLowerCase().endsWith(ext), + ); + + // Allow URLs from common media hosting services even without explicit extensions + const commonMediaHosts = [ + 'imgur.com', + 'giphy.com', + 'youtube.com', + 'vimeo.com', + ]; + const isFromMediaHost = commonMediaHosts.some((host) => + parsedUrl.hostname.includes(host), + ); + + return hasValidProtocol && (hasMediaExtension || isFromMediaHost); + } catch { + return false; + } +} From 9af26a56c7505353fd53603b4c765e65af978b6a Mon Sep 17 00:00:00 2001 From: Ivaylo Badinov <632282+venimus@users.noreply.github.com> Date: Mon, 25 Aug 2025 21:54:38 +0300 Subject: [PATCH 04/14] chore(ci): deploy superhero --- .github/workflows/deploy_develop.yaml | 28 +++++---------------------- .github/workflows/deploy_main.yaml | 28 +++++---------------------- .github/workflows/deploy_staging.yaml | 27 ++++---------------------- .github/workflows/ssh_deploy.yaml | 2 +- 4 files changed, 15 insertions(+), 70 deletions(-) diff --git a/.github/workflows/deploy_develop.yaml b/.github/workflows/deploy_develop.yaml index 694c954..10c6ea0 100644 --- a/.github/workflows/deploy_develop.yaml +++ b/.github/workflows/deploy_develop.yaml @@ -15,17 +15,17 @@ on: required: false type: string jobs: - deploy_wordcraft: - name: wordcraft + deploy: + name: superhero uses: ./.github/workflows/ssh_deploy.yaml with: VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "develop-wordcraftfun-api-mainnet" + CONTAINER_NAME: "develop-superhero-api-mainnet" secrets: AE_NETWORK_ID: "ae_mainnet" API_HOST_PORT: "3043" DB_DATABASE: "api" - DEPLOY_HOST: "api.dev.wordcraft.fun" + DEPLOY_HOST: "api.dev.tokensale.org" DEPLOY_KEY: ${{ secrets.DEV_DEPLOY_KEY }} DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} DB_USER: ${{ secrets.DEV_DB_USER }} @@ -34,22 +34,4 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDING_TAGS_API_KEY }} - deploy_trendminer: - name: trendminer - uses: ./.github/workflows/ssh_deploy.yaml - with: - VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "develop-trendminerfun-api-mainnet" - secrets: - AE_NETWORK_ID: "ae_mainnet" - API_HOST_PORT: "3043" - DB_DATABASE: "api" - DEPLOY_HOST: "api.dev.trendminer.fun" - DEPLOY_KEY: ${{ secrets.DEV_TRENDMINERFUN_DEPLOY_KEY }} - DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} - DB_USER: ${{ secrets.DEV_DB_USER }} - DB_PASSWORD: ${{ secrets.DEV_DB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} - TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDMINERFUN_TRENDING_TAGS_API_KEY }} + diff --git a/.github/workflows/deploy_main.yaml b/.github/workflows/deploy_main.yaml index 9d43edb..65c52c1 100644 --- a/.github/workflows/deploy_main.yaml +++ b/.github/workflows/deploy_main.yaml @@ -15,17 +15,17 @@ on: required: false type: string jobs: - deploy_wordcraft: - name: wordcraft + deploy: + name: superhero uses: ./.github/workflows/ssh_deploy.yaml with: VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "production-wordcraftfun-api-mainnet" + CONTAINER_NAME: "production-superhero-api-mainnet" secrets: AE_NETWORK_ID: "ae_mainnet" API_HOST_PORT: "3033" DB_DATABASE: "api" - DEPLOY_HOST: "api.wordcraft.fun" + DEPLOY_HOST: "api.superhero.com" DEPLOY_KEY: ${{ secrets.PROD_DEPLOY_KEY }} DEPLOY_USERNAME: ${{ secrets.PROD_DEPLOY_USERNAME }} DB_USER: ${{ secrets.PROD_DB_USER }} @@ -34,22 +34,4 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.TRENDING_TAGS_API_KEY }} - # Uncomment when ready to deploy to trendminer production - # deploy_trendminer: - # name: trendminer - # uses: ./.github/workflows/ssh_deploy.yaml - # with: - # VERSION: ${{ inputs.VERSION }} - # CONTAINER_NAME: "production-trendminerfun-api-mainnet" - # secrets: - # AE_NETWORK_ID: "ae_mainnet" - # API_HOST_PORT: "3033" - # DB_DATABASE: "api" - # DEPLOY_HOST: "api.trendminer.fun" - # DEPLOY_KEY: ${{ secrets.PROD_TRENDMINERFUN_DEPLOY_KEY }} - # DEPLOY_USERNAME: ${{ secrets.PROD_DEPLOY_USERNAME }} - # DB_USER: ${{ secrets.PROD_DB_USER }} - # DB_PASSWORD: ${{ secrets.PROD_DB_PASSWORD }} - # DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - # DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} + \ No newline at end of file diff --git a/.github/workflows/deploy_staging.yaml b/.github/workflows/deploy_staging.yaml index 53b8aeb..70c57b9 100644 --- a/.github/workflows/deploy_staging.yaml +++ b/.github/workflows/deploy_staging.yaml @@ -15,17 +15,17 @@ on: required: false type: string jobs: - deploy_wordcraft: - name: wordcraft + deploy: + name: superhero uses: ./.github/workflows/ssh_deploy.yaml with: VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "staging-wordcraftfun-api-mainnet" + CONTAINER_NAME: "staging-superhero-api-mainnet" secrets: AE_NETWORK_ID: "ae_mainnet" API_HOST_PORT: "3033" DB_DATABASE: "api" - DEPLOY_HOST: "api.stag.wordcraft.fun" + DEPLOY_HOST: "api.stag.superhero.com" DEPLOY_KEY: ${{ secrets.STAG_DEPLOY_KEY }} DEPLOY_USERNAME: ${{ secrets.STAG_DEPLOY_USERNAME }} DB_USER: ${{ secrets.STAG_DB_USER }} @@ -34,22 +34,3 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.STAG_TRENDING_TAGS_API_KEY }} - # Uncomment when ready to deploy to trendminer staging - # deploy_trendminer: - # name: trendminer - # uses: ./.github/workflows/ssh_deploy.yaml - # with: - # VERSION: ${{ inputs.VERSION }} - # CONTAINER_NAME: "staging-trendminerfun-api-mainnet" - # secrets: - # AE_NETWORK_ID: "ae_mainnet" - # API_HOST_PORT: "3033" - # DB_DATABASE: "api" - # DEPLOY_HOST: "api.stag.trendminer.fun" - # DEPLOY_KEY: ${{ secrets.STAG_TRENDMINERFUN_DEPLOY_KEY }} - # DEPLOY_USERNAME: ${{ secrets.STAG_DEPLOY_USERNAME }} - # DB_USER: ${{ secrets.STAG_DB_USER }} - # DB_PASSWORD: ${{ secrets.STAG_DB_PASSWORD }} - # DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - # DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} diff --git a/.github/workflows/ssh_deploy.yaml b/.github/workflows/ssh_deploy.yaml index 18be025..78f62fb 100644 --- a/.github/workflows/ssh_deploy.yaml +++ b/.github/workflows/ssh_deploy.yaml @@ -10,7 +10,7 @@ on: type: string CONTAINER_NAME: description: "Container name" - default: "wordcraftfun-api" + default: "superhero-api" required: false type: string secrets: From badade21ba3c7f659232ee643355c2e9643ca1ce Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Tue, 26 Aug 2025 15:35:41 +0100 Subject: [PATCH 05/14] feat: implement comment functionality in posts module with parent-child relationship handling --- src/social/controllers/posts.controller.ts | 37 +- src/social/entities/post.entity.ts | 19 +- src/social/interfaces/post.interfaces.ts | 19 + src/social/services/post.service.ts | 356 ++++++++++++++++-- .../transaction-history.service.spec.ts | 4 +- 5 files changed, 396 insertions(+), 39 deletions(-) diff --git a/src/social/controllers/posts.controller.ts b/src/social/controllers/posts.controller.ts index 5f3ee5a..ebc81d2 100644 --- a/src/social/controllers/posts.controller.ts +++ b/src/social/controllers/posts.controller.ts @@ -60,7 +60,9 @@ export class PostsController { @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', @Query('search') search?: string, ) { - const query = this.postRepository.createQueryBuilder('post'); + const query = this.postRepository + .createQueryBuilder('post') + .where('post.post_id IS NULL'); // Add search functionality if (search) { @@ -99,4 +101,37 @@ export class PostsController { } return post; } + + @ApiParam({ name: 'id', type: 'string', description: 'Post ID' }) + @ApiQuery({ name: 'page', type: 'number', required: false }) + @ApiQuery({ name: 'limit', type: 'number', required: false }) + @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) + @ApiOperation({ + operationId: 'getComments', + summary: 'Get comments for a post', + description: 'Retrieve paginated comments for a specific post', + }) + @ApiOkResponsePaginated(PostDto) + @Get(':id/comments') + async getComments( + @Param('id') id: string, + @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, + @Query('limit', new DefaultValuePipe(50), ParseIntPipe) limit = 50, + @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'ASC', + ) { + // First check if the parent post exists + const parentPost = await this.postRepository.findOne({ + where: { id }, + }); + if (!parentPost) { + throw new NotFoundException(`Post with ID ${id} not found`); + } + + const query = this.postRepository + .createQueryBuilder('post') + .where('post.post_id = :parentPostId', { parentPostId: id }) + .orderBy('post.created_at', orderDirection); + + return paginate(query, { page, limit }); + } } diff --git a/src/social/entities/post.entity.ts b/src/social/entities/post.entity.ts index 51a2ca8..bac5bcc 100644 --- a/src/social/entities/post.entity.ts +++ b/src/social/entities/post.entity.ts @@ -1,4 +1,11 @@ -import { Column, CreateDateColumn, Entity, PrimaryColumn } from 'typeorm'; +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryColumn, +} from 'typeorm'; @Entity({ name: 'posts', @@ -7,6 +14,16 @@ export class Post { @PrimaryColumn() id: string; + @Column({ nullable: true }) + post_id: string; + + @ManyToOne(() => Post, (post) => post.id, { + nullable: true, + onDelete: 'CASCADE', + }) + @JoinColumn({ name: 'post_id' }) + parent_post: Post; + @Column({ unique: true, }) diff --git a/src/social/interfaces/post.interfaces.ts b/src/social/interfaces/post.interfaces.ts index 3c777f8..dfbdc34 100644 --- a/src/social/interfaces/post.interfaces.ts +++ b/src/social/interfaces/post.interfaces.ts @@ -33,6 +33,7 @@ export interface ICreatePostData { total_comments: number; tx_args: any[]; created_at: Date; + post_id?: string; } /** @@ -73,3 +74,21 @@ export interface IContentParsingOptions { maxMediaItems?: number; sanitizeContent?: boolean; } + +/** + * Comment detection result + */ +export interface ICommentInfo { + isComment: boolean; + parentPostId?: string; + commentArgument?: any; +} + +/** + * Comment processing result + */ +export interface ICommentProcessingResult { + success: boolean; + parentPostExists?: boolean; + error?: string; +} diff --git a/src/social/services/post.service.ts b/src/social/services/post.service.ts index bde6819..2e626df 100644 --- a/src/social/services/post.service.ts +++ b/src/social/services/post.service.ts @@ -22,6 +22,8 @@ import { IPostProcessingResult, IMiddlewareResponse, IMiddlewareRequestConfig, + ICommentInfo, + ICommentProcessingResult, } from '../interfaces/post.interfaces'; import { parsePostContent } from '../utils/content-parser.util'; @@ -41,6 +43,8 @@ export class PostService { this.logger.log('Initializing PostService module...'); try { await this.pullLatestPostsForContracts(); + // Run cleanup for any orphaned comments from previous runs + await this.fixOrphanedComments(); this.logger.log('PostService module initialized successfully'); } catch (error) { this.logger.error('Failed to initialize PostService module', error); @@ -151,9 +155,14 @@ export class PostService { this.isProcessing.set(processingKey, true); + // delete all posts for this contract + // await this.postRepository.delete({ + // contract_address: contract.contractAddress, + // }); + try { const config: IMiddlewareRequestConfig = { - direction: 'backward', + direction: 'forward', limit: 100, type: 'contract_call', contract: contract.contractAddress, @@ -250,44 +259,36 @@ export class PostService { return posts; } - // Process transactions with better error handling - const processingResults = await Promise.allSettled( - result.data.map(async (transaction) => { - try { - const camelCasedTransaction = camelcaseKeysDeep( - transaction, - ) as ITransaction; - const post = await this.savePostFromTransaction( - camelCasedTransaction, - contract, - ); - return post; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.warn('Failed to process individual transaction', { - txHash: transaction?.hash, - error: errorMessage, - }); - throw error; + // Process transactions sequentially to handle parent-child dependencies + // This ensures parent posts are created before their comments + const successfulPosts: any[] = []; + let failedCount = 0; + + for (const transaction of result.data) { + try { + const camelCasedTransaction = camelcaseKeysDeep( + transaction, + ) as ITransaction; + const post = await this.savePostFromTransaction( + camelCasedTransaction, + contract, + ); + if (post) { + successfulPosts.push(post); } - }), - ); - - // Collect successful results - const successfulPosts = processingResults - .filter( - (result): result is PromiseFulfilledResult => - result.status === 'fulfilled', - ) - .map((result) => result.value) - .filter((post) => post !== null); + } catch (error) { + failedCount++; + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.warn('Failed to process individual transaction', { + txHash: transaction?.hash, + error: errorMessage, + }); + } + } posts.push(...successfulPosts); - const failedCount = processingResults.filter( - (r) => r.status === 'rejected', - ).length; if (failedCount > 0) { this.logger.warn( `${failedCount} transactions failed to process in this batch`, @@ -315,6 +316,7 @@ export class PostService { } const txHash = transaction.hash; + const commentInfo = this.detectComment(transaction); try { // Check if post already exists @@ -322,6 +324,26 @@ export class PostService { where: { tx_hash: txHash }, }); + // Handle existing post that needs to be converted to comment + if (existingPost && commentInfo.isComment && !existingPost.post_id) { + const result = await this.processExistingPostAsComment( + existingPost, + commentInfo, + txHash, + ); + + if (result.success) { + return existingPost; + } else { + this.logger.warn('Failed to process existing post as comment', { + txHash, + error: result.error, + parentPostExists: result.parentPostExists, + }); + // Continue with regular flow if comment processing fails + } + } + if (existingPost) { return existingPost; } @@ -338,6 +360,24 @@ export class PostService { return null; } + // For new comments, validate parent post exists with retry logic + if (commentInfo.isComment && commentInfo.parentPostId) { + const parentPostExists = await this.validateParentPost( + commentInfo.parentPostId, + ); + if (!parentPostExists) { + this.logger.warn( + 'Cannot create comment: parent post not found after retries', + { + txHash, + parentPostId: commentInfo.parentPostId, + }, + ); + // Still create the comment but mark it as orphaned for later processing + // This prevents data loss in case of timing issues + } + } + // Parse content and extract metadata const parsedContent = parsePostContent( content, @@ -357,11 +397,14 @@ export class PostService { total_comments: 0, tx_args: transaction.tx.arguments, created_at: moment(transaction.microTime).toDate(), + post_id: commentInfo.isComment ? commentInfo.parentPostId : null, }; this.logger.debug('Creating new post', { txHash, postId: postData.id, + isComment: commentInfo.isComment, + parentPostId: postData.post_id, topicsCount: postData.topics.length, mediaCount: postData.media.length, }); @@ -374,9 +417,16 @@ export class PostService { }, ); + // Update parent post comment count if this is a comment + if (commentInfo.isComment && commentInfo.parentPostId) { + await this.updatePostCommentCount(commentInfo.parentPostId); + } + this.logger.log('Post saved successfully', { txHash, postId: post.id, + isComment: commentInfo.isComment, + parentPostId: postData.post_id, contractAddress: contract.contractAddress, }); @@ -443,4 +493,240 @@ export class PostService { // Fallback to hash-based ID if return value is not available return `${transaction.hash.slice(-8)}_v${contract.version}`; } + + /** + * Detects if a transaction represents a comment and extracts parent post information + */ + private detectComment(transaction: ITransaction): ICommentInfo { + if (!transaction?.tx?.arguments?.[1]?.value) { + return { isComment: false }; + } + + const commentArgument = transaction.tx.arguments[1].value.find((arg) => + arg.value?.includes('comment:'), + ); + + if (!commentArgument?.value) { + return { isComment: false }; + } + + const parentPostId = commentArgument.value.split('comment:')[1]; + if (!parentPostId || parentPostId.trim().length === 0) { + this.logger.warn('Invalid comment format: missing parent post ID', { + txHash: transaction.hash, + commentValue: commentArgument.value, + }); + return { isComment: false }; + } + + return { + isComment: true, + parentPostId: parentPostId.trim(), + commentArgument, + }; + } + + /** + * Validates that a parent post exists for a comment with retry logic + * This handles timing issues in parallel processing where parent posts + * might be processed concurrently + */ + private async validateParentPost( + parentPostId: string, + maxRetries: number = 3, + retryDelay: number = 100, + ): Promise { + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const parentPost = await this.postRepository + .createQueryBuilder('post') + .where('post.id = :parentPostId', { parentPostId }) + .getOne(); + if (parentPost) { + this.logger.debug('Parent post found for comment validation', { + parentPostId, + attempt, + }); + return true; + } + + // If not found and we have retries left, wait and try again + if (attempt < maxRetries) { + this.logger.debug('Parent post not found, retrying...', { + parentPostId, + attempt, + nextRetryIn: retryDelay, + }); + await new Promise((resolve) => setTimeout(resolve, retryDelay)); + retryDelay *= 2; // Exponential backoff + } + } catch (error) { + this.logger.error('Error during parent post validation', { + parentPostId, + attempt, + error: error instanceof Error ? error.message : String(error), + }); + + if (attempt === maxRetries) { + return false; + } + } + } + + this.logger.warn('Parent post not found after all retries', { + parentPostId, + maxRetries, + }); + return false; + } + + /** + * Processes comment-specific logic for existing posts + */ + private async processExistingPostAsComment( + existingPost: Post, + commentInfo: ICommentInfo, + txHash: string, + ): Promise { + if (!commentInfo.isComment || !commentInfo.parentPostId) { + return { success: false, error: 'Invalid comment information' }; + } + + // Check if parent post exists + const parentPostExists = await this.validateParentPost( + commentInfo.parentPostId, + ); + if (!parentPostExists) { + this.logger.warn('Parent post does not exist for comment', { + txHash, + parentPostId: commentInfo.parentPostId, + }); + return { + success: false, + parentPostExists: false, + error: 'Parent post not found', + }; + } + + try { + // Update existing post to be a comment + await this.postRepository.update(existingPost.id, { + post_id: commentInfo.parentPostId, + }); + + // Update comment count for parent post + await this.updatePostCommentCount(commentInfo.parentPostId); + + this.logger.log('Successfully updated existing post as comment', { + txHash, + postId: existingPost.id, + parentPostId: commentInfo.parentPostId, + }); + + return { success: true, parentPostExists: true }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.error('Failed to process existing post as comment', { + txHash, + postId: existingPost.id, + parentPostId: commentInfo.parentPostId, + error: errorMessage, + }); + return { success: false, error: errorMessage }; + } + } + + /** + * Updates the comment count for a parent post + */ + private async updatePostCommentCount(parentPostId: string): Promise { + try { + const count = await this.postRepository + .createQueryBuilder('post') + .where('post.post_id = :parentPostId', { parentPostId }) + .getCount(); + + await this.postRepository.update( + { id: parentPostId }, + { total_comments: count }, + ); + + this.logger.debug('Updated comment count for parent post', { + parentPostId, + commentCount: count, + }); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.error('Failed to update comment count', { + parentPostId, + error: errorMessage, + }); + // Don't throw - comment count update failure shouldn't break the main flow + } + } + + /** + * Fixes orphaned comments by linking them to their parent posts + * This is a cleanup method for comments that were created before their parent posts + */ + async fixOrphanedComments(): Promise { + try { + this.logger.log('Starting orphaned comments cleanup...'); + + // Find comments that have a post_id but the parent post doesn't exist + const orphanedComments = await this.postRepository + .createQueryBuilder('comment') + .leftJoin('posts', 'parent', 'parent.id = comment.post_id') + .where('comment.post_id IS NOT NULL') + .andWhere('parent.id IS NULL') + .getMany(); + + if (orphanedComments.length === 0) { + this.logger.log('No orphaned comments found'); + return; + } + + this.logger.log(`Found ${orphanedComments.length} orphaned comments`); + + let fixedCount = 0; + for (const comment of orphanedComments) { + try { + // Check if parent post now exists + const parentExists = await this.validateParentPost( + comment.post_id, + 1, + 0, + ); + if (parentExists) { + // Update comment count for the parent + await this.updatePostCommentCount(comment.post_id); + fixedCount++; + } else { + // If parent still doesn't exist, remove the post_id to make it a regular post + await this.postRepository.update(comment.id, { post_id: null }); + this.logger.warn('Converted orphaned comment to regular post', { + commentId: comment.id, + originalParentId: comment.post_id, + }); + } + } catch (error) { + this.logger.error('Failed to fix orphaned comment', { + commentId: comment.id, + parentId: comment.post_id, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + this.logger.log( + `Orphaned comments cleanup completed: ${fixedCount} fixed`, + ); + } catch (error) { + this.logger.error('Failed to run orphaned comments cleanup', { + error: error instanceof Error ? error.message : String(error), + }); + } + } } diff --git a/src/transactions/services/transaction-history.service.spec.ts b/src/transactions/services/transaction-history.service.spec.ts index 03a81c1..5a183bf 100644 --- a/src/transactions/services/transaction-history.service.spec.ts +++ b/src/transactions/services/transaction-history.service.spec.ts @@ -5,8 +5,8 @@ import { DataSource, Repository } from 'typeorm'; import { Transaction } from '../entities/transaction.entity'; import { TokensService } from '@/tokens/tokens.service'; -import { TransactionHistoryService } from '../services/transaction-history.service'; -import { TransactionService } from '../services/transaction.service'; +import { TransactionHistoryService } from './transaction-history.service'; +import { TransactionService } from './transaction.service'; describe('TransactionHistoryService', () => { let service: TransactionHistoryService; From 41577989e718d17e78820fb0d3f77725173edc86 Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Tue, 2 Sep 2025 16:02:40 +0100 Subject: [PATCH 06/14] feat(trending-tags): clear existing trending tags before adding new ones --- src/trending-tags/services/trending-tags.service.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/trending-tags/services/trending-tags.service.ts b/src/trending-tags/services/trending-tags.service.ts index a405d57..37a3e5a 100644 --- a/src/trending-tags/services/trending-tags.service.ts +++ b/src/trending-tags/services/trending-tags.service.ts @@ -65,6 +65,11 @@ export class TrendingTagsService { errors: [] as string[], }; + // if data.items length, delete all trending tags + if (data.items.length) { + await this.trendingTagRepository.delete({}); + } + for (const item of data.items) { try { const normalizedTag = this.normalizeTag(item.tag); From 53534a33834b16ba2a34e5cf8bbc13879c403eca Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Tue, 2 Sep 2025 16:55:33 +0100 Subject: [PATCH 07/14] refactor(post.service): remove commented-out code and unnecessary logging for contract posts retrieval --- src/social/services/post.service.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/social/services/post.service.ts b/src/social/services/post.service.ts index 2e626df..09ebe71 100644 --- a/src/social/services/post.service.ts +++ b/src/social/services/post.service.ts @@ -155,11 +155,6 @@ export class PostService { this.isProcessing.set(processingKey, true); - // delete all posts for this contract - // await this.postRepository.delete({ - // contract_address: contract.contractAddress, - // }); - try { const config: IMiddlewareRequestConfig = { direction: 'forward', @@ -177,11 +172,6 @@ export class PostService { const url = `${ACTIVE_NETWORK.middlewareUrl}/v3/transactions?${queryString}`; - this.logger.log( - `Pulling latest posts for contract ${contract.contractAddress}`, - { url }, - ); - const posts = await this.loadPostsFromMdw(url, contract); this.logger.log( From ef02bddc1c062e49a67d7a84791dc1b070f4eb6e Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Thu, 4 Sep 2025 12:42:24 +0100 Subject: [PATCH 08/14] feat(dex): introduce DEX module with token and pair management, including controllers, services, and DTOs --- src/app.module.ts | 2 + src/dex/config/dex-contracts.config.ts | 4 ++ src/dex/controllers/dex-tokens.controller.ts | 76 ++++++++++++++++++++ src/dex/controllers/pairs.controller.ts | 70 ++++++++++++++++++ src/dex/dex.module.ts | 24 +++++++ src/dex/dto/dex-token.dto.ts | 39 ++++++++++ src/dex/dto/index.ts | 2 + src/dex/dto/pair.dto.ts | 34 +++++++++ src/dex/entities/dex-token.entity.ts | 31 ++++++++ src/dex/entities/pair.entity.ts | 36 ++++++++++ src/dex/services/dex-token.service.ts | 38 ++++++++++ src/dex/services/pair.service.ts | 43 +++++++++++ 12 files changed, 399 insertions(+) create mode 100644 src/dex/config/dex-contracts.config.ts create mode 100644 src/dex/controllers/dex-tokens.controller.ts create mode 100644 src/dex/controllers/pairs.controller.ts create mode 100644 src/dex/dex.module.ts create mode 100644 src/dex/dto/dex-token.dto.ts create mode 100644 src/dex/dto/index.ts create mode 100644 src/dex/dto/pair.dto.ts create mode 100644 src/dex/entities/dex-token.entity.ts create mode 100644 src/dex/entities/pair.entity.ts create mode 100644 src/dex/services/dex-token.service.ts create mode 100644 src/dex/services/pair.service.ts diff --git a/src/app.module.ts b/src/app.module.ts index e8be73f..b2c3bb1 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -24,6 +24,7 @@ import { AffiliationModule } from './affiliation/affiliation.module'; import { AccountModule } from './account/account.module'; import { TrendingTagsModule } from './trending-tags/trending-tags.module'; import { PostModule } from './social/post.module'; +import { DexModule } from './dex/dex.module'; @Module({ imports: [ @@ -61,6 +62,7 @@ import { PostModule } from './social/post.module'; AccountModule, TrendingTagsModule, PostModule, + DexModule, ], controllers: [AppController], providers: [AppService], diff --git a/src/dex/config/dex-contracts.config.ts b/src/dex/config/dex-contracts.config.ts new file mode 100644 index 0000000..53baa06 --- /dev/null +++ b/src/dex/config/dex-contracts.config.ts @@ -0,0 +1,4 @@ +/** + * Configuration for supported dex contracts + */ +export const DEX_CONTRACTS = {}; diff --git a/src/dex/controllers/dex-tokens.controller.ts b/src/dex/controllers/dex-tokens.controller.ts new file mode 100644 index 0000000..953f460 --- /dev/null +++ b/src/dex/controllers/dex-tokens.controller.ts @@ -0,0 +1,76 @@ +import { + Controller, + DefaultValuePipe, + Get, + NotFoundException, + Param, + ParseIntPipe, + Query, +} from '@nestjs/common'; +import { + ApiOperation, + ApiParam, + ApiQuery, + ApiTags, + ApiOkResponse, +} from '@nestjs/swagger'; +import { DexTokenService } from '../services/dex-token.service'; +import { DexTokenDto } from '../dto'; +import { ApiOkResponsePaginated } from '@/utils/api-type'; + +@Controller('dex-tokens') +@ApiTags('DEX Tokens') +export class DexTokensController { + constructor(private readonly dexTokenService: DexTokenService) {} + + @ApiQuery({ name: 'page', type: 'number', required: false }) + @ApiQuery({ name: 'limit', type: 'number', required: false }) + @ApiQuery({ + name: 'order_by', + enum: ['pairs_count', 'name', 'symbol', 'created_at'], + required: false, + }) + @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) + @ApiOperation({ + operationId: 'listAllDexTokens', + summary: 'Get all DEX tokens', + description: + 'Retrieve a paginated list of all DEX tokens with optional sorting', + }) + @ApiOkResponsePaginated(DexTokenDto) + @Get() + async listAll( + @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, + @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, + @Query('order_by') orderBy: string = 'created_at', + @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', + ) { + return this.dexTokenService.findAll( + { page, limit }, + orderBy, + orderDirection, + ); + } + + @ApiParam({ + name: 'address', + type: 'string', + description: 'Token contract address', + }) + @ApiOperation({ + operationId: 'getDexTokenByAddress', + summary: 'Get DEX token by address', + description: 'Retrieve a specific DEX token by its contract address', + }) + @ApiOkResponse({ type: DexTokenDto }) + @Get(':address') + async getByAddress(@Param('address') address: string) { + const token = await this.dexTokenService.findByAddress(address); + if (!token) { + throw new NotFoundException( + `DEX token with address ${address} not found`, + ); + } + return token; + } +} diff --git a/src/dex/controllers/pairs.controller.ts b/src/dex/controllers/pairs.controller.ts new file mode 100644 index 0000000..ef29252 --- /dev/null +++ b/src/dex/controllers/pairs.controller.ts @@ -0,0 +1,70 @@ +import { + Controller, + DefaultValuePipe, + Get, + NotFoundException, + Param, + ParseIntPipe, + Query, +} from '@nestjs/common'; +import { + ApiOperation, + ApiParam, + ApiQuery, + ApiTags, + ApiOkResponse, +} from '@nestjs/swagger'; +import { PairService } from '../services/pair.service'; +import { PairDto } from '../dto'; +import { ApiOkResponsePaginated } from '@/utils/api-type'; + +@Controller('pairs') +@ApiTags('DEX Pairs') +export class PairsController { + constructor(private readonly pairService: PairService) {} + + @ApiQuery({ name: 'page', type: 'number', required: false }) + @ApiQuery({ name: 'limit', type: 'number', required: false }) + @ApiQuery({ + name: 'order_by', + enum: ['transactions_count', 'created_at'], + required: false, + }) + @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) + @ApiOperation({ + operationId: 'listAllPairs', + summary: 'Get all pairs', + description: + 'Retrieve a paginated list of all DEX pairs with optional sorting', + }) + @ApiOkResponsePaginated(PairDto) + @Get() + async listAll( + @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, + @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, + @Query('order_by') orderBy: string = 'created_at', + @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', + ) { + return this.pairService.findAll({ page, limit }, orderBy, orderDirection); + } + + @ApiParam({ + name: 'address', + type: 'string', + description: 'Pair contract address', + }) + @ApiOperation({ + operationId: 'getPairByAddress', + summary: 'Get pair by address', + description: 'Retrieve a specific pair by its contract address', + }) + @ApiOkResponse({ type: PairDto }) + @Get(':address') + async getByAddress(@Param('address') address: string) { + const pair = await this.pairService.findByAddress(address); + if (!pair) { + throw new NotFoundException(`Pair with address ${address} not found`); + } + return pair; + } +} diff --git a/src/dex/dex.module.ts b/src/dex/dex.module.ts new file mode 100644 index 0000000..4e9ce80 --- /dev/null +++ b/src/dex/dex.module.ts @@ -0,0 +1,24 @@ +import { AeModule } from '@/ae/ae.module'; +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { Pair } from './entities/pair.entity'; +import { DexToken } from './entities/dex-token.entity'; +import { PairService } from './services/pair.service'; +import { DexTokenService } from './services/dex-token.service'; +import { TransactionsModule } from '@/transactions/transactions.module'; +import { PairsController } from './controllers/pairs.controller'; +import { DexTokensController } from './controllers/dex-tokens.controller'; + +@Module({ + imports: [ + AeModule, + TransactionsModule, + TypeOrmModule.forFeature([Pair, DexToken]), + ], + providers: [PairService, DexTokenService], + exports: [PairService, DexTokenService], + controllers: [PairsController, DexTokensController], +}) +export class DexModule { + // +} diff --git a/src/dex/dto/dex-token.dto.ts b/src/dex/dto/dex-token.dto.ts new file mode 100644 index 0000000..1a9121b --- /dev/null +++ b/src/dex/dto/dex-token.dto.ts @@ -0,0 +1,39 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class DexTokenDto { + @ApiProperty({ + description: 'Token contract address', + example: 'ct_2AfnEfCSPx4A6VjMBfDfqHNYcqDJjuJjGV1qhqP5qNKNBvYfE2', + }) + address: string; + + @ApiProperty({ + description: 'Token name', + example: 'Wrapped Aeternity', + }) + name: string; + + @ApiProperty({ + description: 'Token symbol', + example: 'WAE', + }) + symbol: string; + + @ApiProperty({ + description: 'Token decimals', + example: 18, + }) + decimals: number; + + @ApiProperty({ + description: 'Number of pairs this token is part of', + example: 5, + }) + pairs_count: number; + + @ApiProperty({ + description: 'Token creation timestamp', + example: '2024-01-01T00:00:00.000Z', + }) + created_at: Date; +} diff --git a/src/dex/dto/index.ts b/src/dex/dto/index.ts new file mode 100644 index 0000000..21304d2 --- /dev/null +++ b/src/dex/dto/index.ts @@ -0,0 +1,2 @@ +export * from './pair.dto'; +export * from './dex-token.dto'; diff --git a/src/dex/dto/pair.dto.ts b/src/dex/dto/pair.dto.ts new file mode 100644 index 0000000..21fc036 --- /dev/null +++ b/src/dex/dto/pair.dto.ts @@ -0,0 +1,34 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { DexTokenDto } from './dex-token.dto'; + +export class PairDto { + @ApiProperty({ + description: 'Pair contract address', + example: 'ct_2AfnEfCSPx4A6VjMBfDfqHNYcqDJjuJjGV1qhqP5qNKNBvYfE2', + }) + address: string; + + @ApiProperty({ + description: 'First token in the pair', + type: () => DexTokenDto, + }) + token0: DexTokenDto; + + @ApiProperty({ + description: 'Second token in the pair', + type: () => DexTokenDto, + }) + token1: DexTokenDto; + + @ApiProperty({ + description: 'Number of transactions for this pair', + example: 150, + }) + transactions_count: number; + + @ApiProperty({ + description: 'Pair creation timestamp', + example: '2024-01-01T00:00:00.000Z', + }) + created_at: Date; +} diff --git a/src/dex/entities/dex-token.entity.ts b/src/dex/entities/dex-token.entity.ts new file mode 100644 index 0000000..abe0f41 --- /dev/null +++ b/src/dex/entities/dex-token.entity.ts @@ -0,0 +1,31 @@ +import { Column, CreateDateColumn, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ + name: 'dex_tokens', +}) +export class DexToken { + @PrimaryColumn() + address: string; + + @Column() + name: string; + + @Column() + symbol: string; + + @Column({ + default: 18, + }) + decimals: number; + + @Column({ + default: 0, + }) + pairs_count: number; + + @CreateDateColumn({ + type: 'timestamp', + default: () => 'CURRENT_TIMESTAMP(6)', + }) + public created_at: Date; +} diff --git a/src/dex/entities/pair.entity.ts b/src/dex/entities/pair.entity.ts new file mode 100644 index 0000000..ecb4ec8 --- /dev/null +++ b/src/dex/entities/pair.entity.ts @@ -0,0 +1,36 @@ +import { DexToken } from './dex-token.entity'; +import { + CreateDateColumn, + Entity, + PrimaryColumn, + ManyToOne, + JoinColumn, + Column, +} from 'typeorm'; + +@Entity({ + name: 'pairs', +}) +export class Pair { + @PrimaryColumn() + address: string; + + @ManyToOne(() => DexToken, (dexToken) => dexToken.address) + @JoinColumn({ name: 'token0_address' }) + token0: DexToken; + + @ManyToOne(() => DexToken, (dexToken) => dexToken.address) + @JoinColumn({ name: 'token1_address' }) + token1: DexToken; + + @Column({ + default: 0, + }) + transactions_count: number; + + @CreateDateColumn({ + type: 'timestamp', + default: () => 'CURRENT_TIMESTAMP(6)', + }) + public created_at: Date; +} diff --git a/src/dex/services/dex-token.service.ts b/src/dex/services/dex-token.service.ts new file mode 100644 index 0000000..de287a9 --- /dev/null +++ b/src/dex/services/dex-token.service.ts @@ -0,0 +1,38 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { DexToken } from '../entities/dex-token.entity'; +import { + IPaginationOptions, + paginate, + Pagination, +} from 'nestjs-typeorm-paginate'; + +@Injectable() +export class DexTokenService { + constructor( + @InjectRepository(DexToken) + private readonly dexTokenRepository: Repository, + ) {} + + async findAll( + options: IPaginationOptions, + orderBy: string = 'created_at', + orderDirection: 'ASC' | 'DESC' = 'DESC', + ): Promise> { + const query = this.dexTokenRepository.createQueryBuilder('dexToken'); + + if (orderBy) { + query.orderBy(`dexToken.${orderBy}`, orderDirection); + } + + return paginate(query, options); + } + + async findByAddress(address: string): Promise { + return this.dexTokenRepository + .createQueryBuilder('dexToken') + .where('dexToken.address = :address', { address }) + .getOne(); + } +} diff --git a/src/dex/services/pair.service.ts b/src/dex/services/pair.service.ts new file mode 100644 index 0000000..182eeeb --- /dev/null +++ b/src/dex/services/pair.service.ts @@ -0,0 +1,43 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { Pair } from '../entities/pair.entity'; +import { + IPaginationOptions, + paginate, + Pagination, +} from 'nestjs-typeorm-paginate'; + +@Injectable() +export class PairService { + constructor( + @InjectRepository(Pair) + private readonly pairRepository: Repository, + ) {} + + async findAll( + options: IPaginationOptions, + orderBy: string = 'created_at', + orderDirection: 'ASC' | 'DESC' = 'DESC', + ): Promise> { + const query = this.pairRepository + .createQueryBuilder('pair') + .leftJoinAndSelect('pair.token0', 'token0') + .leftJoinAndSelect('pair.token1', 'token1'); + + if (orderBy) { + query.orderBy(`pair.${orderBy}`, orderDirection); + } + + return paginate(query, options); + } + + async findByAddress(address: string): Promise { + return this.pairRepository + .createQueryBuilder('pair') + .leftJoinAndSelect('pair.token0', 'token0') + .leftJoinAndSelect('pair.token1', 'token1') + .where('pair.address = :address', { address }) + .getOne(); + } +} From c81cacbccdc236b7fd70acd2b03d6906a13e9c31 Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Thu, 4 Sep 2025 12:52:37 +0100 Subject: [PATCH 09/14] feat(post.service): enable conditional initialization of post retrieval based on PULL_SOCIAL_POSTS_ENABLED flag --- src/social/services/post.service.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/social/services/post.service.ts b/src/social/services/post.service.ts index 09ebe71..aa160df 100644 --- a/src/social/services/post.service.ts +++ b/src/social/services/post.service.ts @@ -5,6 +5,7 @@ import { Post } from '../entities/post.entity'; import { ACTIVE_NETWORK, MAX_RETRIES_WHEN_REQUEST_FAILED, + PULL_SOCIAL_POSTS_ENABLED, WAIT_TIME_WHEN_REQUEST_FAILED, } from '@/configs'; import { fetchJson } from '@/utils/common'; @@ -40,15 +41,16 @@ export class PostService { } async onModuleInit(): Promise { - this.logger.log('Initializing PostService module...'); - try { - await this.pullLatestPostsForContracts(); - // Run cleanup for any orphaned comments from previous runs - await this.fixOrphanedComments(); + if (PULL_SOCIAL_POSTS_ENABLED) { + try { + await this.pullLatestPostsForContracts(); + // Run cleanup for any orphaned comments from previous runs + await this.fixOrphanedComments(); + } catch (error) { + this.logger.error('Failed to initialize PostService module', error); + // Don't throw - allow the service to start even if initial sync fails + } this.logger.log('PostService module initialized successfully'); - } catch (error) { - this.logger.error('Failed to initialize PostService module', error); - // Don't throw - allow the service to start even if initial sync fails } } From dde46e9af43f596a00120935f6ae2fab7ae0c949 Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Thu, 4 Sep 2025 14:21:34 +0100 Subject: [PATCH 10/14] feat(dex): integrate DEX contracts and synchronization service for token and pair management --- package-lock.json | 16 +++++ package.json | 1 + src/configs/constants.ts | 3 + src/dex/config/dex-contracts.config.ts | 7 +- src/dex/dex.module.ts | 13 ++-- src/dex/services/dex-sync.service.ts | 88 ++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 src/dex/services/dex-sync.service.ts diff --git a/package-lock.json b/package-lock.json index 63b5591..93b4a05 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "camelcase-keys-deep": "^0.1.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", + "dex-contracts-v2": "github:aeternity/dex-contracts-v2", "hbs": "^4.2.0", "keyv": "^5.2.3", "moment": "^2.30.1", @@ -4653,6 +4654,15 @@ "node": ">=8" } }, + "node_modules/dex-contracts-v2": { + "version": "1.0.0", + "resolved": "git+ssh://git@github.com/aeternity/dex-contracts-v2.git#9992835ac3e2d0074e5aac03612aea1baea07839", + "hasInstallScript": true, + "license": "ISC", + "dependencies": { + "fs": "^0.0.1-security" + } + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -5734,6 +5744,12 @@ "node": ">= 0.6" } }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==", + "license": "ISC" + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", diff --git a/package.json b/package.json index ca00bf9..309f117 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "@nestjs/typeorm": "^10.0.2", "@nestjs/websockets": "^10.3.9", "bctsl-sdk": "git+ssh://git@github.com/bctsl/bctsl-sdk#v1.0.0", + "dex-contracts-v2": "github:aeternity/dex-contracts-v2", "bignumber.js": "^9.1.2", "bull": "^4.15.0", "cache-manager": "^6.4.0", diff --git a/src/configs/constants.ts b/src/configs/constants.ts index c9fba9e..f8c2337 100644 --- a/src/configs/constants.ts +++ b/src/configs/constants.ts @@ -106,6 +106,9 @@ export const UPDATE_TRENDING_TOKENS_ENABLED = false; export const PULL_INVITATIONS_ENABLED = false; export const PULL_ACCOUNTS_ENABLED = false; export const PULL_TRENDING_TAGS_ENABLED = false; +export const PULL_SOCIAL_POSTS_ENABLED = false; +export const PULL_DEX_TOKENS_ENABLED = false; +export const PULL_DEX_PAIRS_ENABLED = false; /** * API Keys and Security diff --git a/src/dex/config/dex-contracts.config.ts b/src/dex/config/dex-contracts.config.ts index 53baa06..bad223a 100644 --- a/src/dex/config/dex-contracts.config.ts +++ b/src/dex/config/dex-contracts.config.ts @@ -1,4 +1,9 @@ /** * Configuration for supported dex contracts */ -export const DEX_CONTRACTS = {}; +export const DEX_CONTRACTS = { + factory: 'ct_2mfj3FoZxnhkSw5RZMcP8BfPoB1QR4QiYGNCdkAvLZ1zfF6paW', + router: 'ct_azbNZ1XrPjXfqBqbAh1ffLNTQ1sbnuUDFvJrXjYz7JQA1saQ3', + wae: 'ct_J3zBY8xxjsRr3QojETNw48Eb38fjvEuJKkQ6KzECvubvEcvCa', + aeeth: 'ct_ryTY1mxqjCjq1yBn9i6HDaCSdA6thXUFZTA84EMzbWd1SLKdh', +}; diff --git a/src/dex/dex.module.ts b/src/dex/dex.module.ts index 4e9ce80..169daf3 100644 --- a/src/dex/dex.module.ts +++ b/src/dex/dex.module.ts @@ -1,13 +1,14 @@ import { AeModule } from '@/ae/ae.module'; +import { TransactionsModule } from '@/transactions/transactions.module'; import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { Pair } from './entities/pair.entity'; +import { DexTokensController } from './controllers/dex-tokens.controller'; +import { PairsController } from './controllers/pairs.controller'; import { DexToken } from './entities/dex-token.entity'; -import { PairService } from './services/pair.service'; +import { Pair } from './entities/pair.entity'; +import { DexSyncService } from './services/dex-sync.service'; import { DexTokenService } from './services/dex-token.service'; -import { TransactionsModule } from '@/transactions/transactions.module'; -import { PairsController } from './controllers/pairs.controller'; -import { DexTokensController } from './controllers/dex-tokens.controller'; +import { PairService } from './services/pair.service'; @Module({ imports: [ @@ -15,7 +16,7 @@ import { DexTokensController } from './controllers/dex-tokens.controller'; TransactionsModule, TypeOrmModule.forFeature([Pair, DexToken]), ], - providers: [PairService, DexTokenService], + providers: [PairService, DexTokenService, DexSyncService], exports: [PairService, DexTokenService], controllers: [PairsController, DexTokensController], }) diff --git a/src/dex/services/dex-sync.service.ts b/src/dex/services/dex-sync.service.ts new file mode 100644 index 0000000..2827f35 --- /dev/null +++ b/src/dex/services/dex-sync.service.ts @@ -0,0 +1,88 @@ +import { AeSdkService } from '@/ae/ae-sdk.service'; +import { ACTIVE_NETWORK } from '@/configs'; +import { IMiddlewareRequestConfig } from '@/social/interfaces/post.interfaces'; +import { fetchJson } from '@/utils/common'; +import ContractWithMethods, { + ContractMethodsBase, +} from '@aeternity/aepp-sdk/es/contract/Contract'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import * as routerInterface from 'dex-contracts-v2/build/AedexV2Router.aci.json'; +import { Repository } from 'typeorm'; +import { DEX_CONTRACTS } from '../config/dex-contracts.config'; +import { DexToken } from '../entities/dex-token.entity'; +import { Pair } from '../entities/pair.entity'; +import { Encoded } from '@aeternity/aepp-sdk'; + +@Injectable() +export class DexSyncService { + routerContract: ContractWithMethods; + constructor( + @InjectRepository(DexToken) + private readonly dexTokenRepository: Repository, + @InjectRepository(Pair) + private readonly dexPairRepository: Repository, + + private aeSdkService: AeSdkService, + ) { + // + } + + async onModuleInit(): Promise { + console.log('========================'); + console.log('======DexSyncService=================='); + console.log('========================'); + // + + this.routerContract = await this.aeSdkService.sdk.initializeContract({ + aci: routerInterface, + address: DEX_CONTRACTS.router as Encoded.ContractAddress, + }); + this.syncDexTokens(); + } + + async syncDexTokens() { + const config: IMiddlewareRequestConfig = { + direction: 'forward', + limit: 10, + type: 'contract_call', + contract: DEX_CONTRACTS.router, + }; + const queryString = new URLSearchParams({ + direction: config.direction, + limit: config.limit.toString(), + type: config.type, + contract: config.contract, + }).toString(); + const url = `${ACTIVE_NETWORK.middlewareUrl}/v3/transactions?${queryString}`; + await this.pullDexPairsFromMdw(url); + } + + async pullDexPairsFromMdw(url: string) { + console.log('========================'); + const result = await fetchJson(url); + const data = result?.data ?? []; + for (const item of data) { + if (item.tx.function === 'add_liquidity') { + console.log('========================'); + console.log('item', item); + console.log('logs', item.tx.log); + console.log('routerInterface', routerInterface); + console.log('this.routerContrac', this.routerContract); + console.log('========================'); + const decodedEvents = this.routerContract.$decodeEvents(item.tx.log); + console.log('decodedEvents', decodedEvents); + + break; + } + + // await this.saveDexPairFromTransaction(camelcaseKeysDeep(item)); + } + // if (result.next) { + // return await this.pullDexPairsFromMdw( + // `${ACTIVE_NETWORK.middlewareUrl}${result.next}`, + // ); + // } + return result; + } +} From 8efec17d33a39b7d59d7d64ba328fbc9953e21e0 Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Thu, 4 Sep 2025 14:21:41 +0100 Subject: [PATCH 11/14] feat(dex): add new transaction functions and enhance DEX synchronization logic for improved token pair management --- src/configs/constants.ts | 17 ++- src/dex/services/dex-sync.service.ts | 179 +++++++++++++++++++++++---- 2 files changed, 169 insertions(+), 27 deletions(-) diff --git a/src/configs/constants.ts b/src/configs/constants.ts index f8c2337..2089091 100644 --- a/src/configs/constants.ts +++ b/src/configs/constants.ts @@ -81,6 +81,17 @@ export const TX_FUNCTIONS = { buy: 'buy', sell: 'sell', create_community: 'create_community', + + // dex(swap) + swap_exact_tokens_for_tokens: 'swap_exact_tokens_for_tokens', + swap_tokens_for_exact_tokens: 'swap_tokens_for_exact_tokens', + swap_exact_ae_for_tokens: 'swap_exact_ae_for_tokens', + swap_exact_tokens_for_ae: 'swap_exact_tokens_for_ae', + swap_tokens_for_exact_ae: 'swap_tokens_for_exact_ae', + swap_ae_for_exact_tokens: 'swap_ae_for_exact_tokens', + + add_liquidity: 'add_liquidity', + add_liquidity_ae: 'add_liquidity_ae', } as const; export const WAIT_TIME_WHEN_REQUEST_FAILED = 3000; // 3 seconds @@ -106,9 +117,9 @@ export const UPDATE_TRENDING_TOKENS_ENABLED = false; export const PULL_INVITATIONS_ENABLED = false; export const PULL_ACCOUNTS_ENABLED = false; export const PULL_TRENDING_TAGS_ENABLED = false; -export const PULL_SOCIAL_POSTS_ENABLED = false; -export const PULL_DEX_TOKENS_ENABLED = false; -export const PULL_DEX_PAIRS_ENABLED = false; +export const PULL_SOCIAL_POSTS_ENABLED = true; +export const PULL_DEX_TOKENS_ENABLED = true; +export const PULL_DEX_PAIRS_ENABLED = true; /** * API Keys and Security diff --git a/src/dex/services/dex-sync.service.ts b/src/dex/services/dex-sync.service.ts index 2827f35..78bd87e 100644 --- a/src/dex/services/dex-sync.service.ts +++ b/src/dex/services/dex-sync.service.ts @@ -1,22 +1,27 @@ +import camelcaseKeysDeep from 'camelcase-keys-deep'; + import { AeSdkService } from '@/ae/ae-sdk.service'; -import { ACTIVE_NETWORK } from '@/configs'; +import { ACTIVE_NETWORK, TX_FUNCTIONS } from '@/configs'; import { IMiddlewareRequestConfig } from '@/social/interfaces/post.interfaces'; import { fetchJson } from '@/utils/common'; +import { ITransaction } from '@/utils/types'; +import { Encoded } from '@aeternity/aepp-sdk'; import ContractWithMethods, { ContractMethodsBase, } from '@aeternity/aepp-sdk/es/contract/Contract'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import * as routerInterface from 'dex-contracts-v2/build/AedexV2Router.aci.json'; +import factoryInterface from 'dex-contracts-v2/build/AedexV2Factory.aci.json'; +import routerInterface from 'dex-contracts-v2/build/AedexV2Router.aci.json'; import { Repository } from 'typeorm'; import { DEX_CONTRACTS } from '../config/dex-contracts.config'; import { DexToken } from '../entities/dex-token.entity'; import { Pair } from '../entities/pair.entity'; -import { Encoded } from '@aeternity/aepp-sdk'; @Injectable() export class DexSyncService { routerContract: ContractWithMethods; + factoryContract: ContractWithMethods; constructor( @InjectRepository(DexToken) private readonly dexTokenRepository: Repository, @@ -30,7 +35,7 @@ export class DexSyncService { async onModuleInit(): Promise { console.log('========================'); - console.log('======DexSyncService=================='); + console.log('==== DexSyncService ===='); console.log('========================'); // @@ -38,13 +43,17 @@ export class DexSyncService { aci: routerInterface, address: DEX_CONTRACTS.router as Encoded.ContractAddress, }); + this.factoryContract = await this.aeSdkService.sdk.initializeContract({ + aci: factoryInterface, + address: DEX_CONTRACTS.factory as Encoded.ContractAddress, + }); this.syncDexTokens(); } async syncDexTokens() { const config: IMiddlewareRequestConfig = { direction: 'forward', - limit: 10, + limit: 100, type: 'contract_call', contract: DEX_CONTRACTS.router, }; @@ -59,30 +68,152 @@ export class DexSyncService { } async pullDexPairsFromMdw(url: string) { - console.log('========================'); const result = await fetchJson(url); const data = result?.data ?? []; - for (const item of data) { - if (item.tx.function === 'add_liquidity') { - console.log('========================'); - console.log('item', item); - console.log('logs', item.tx.log); - console.log('routerInterface', routerInterface); - console.log('this.routerContrac', this.routerContract); - console.log('========================'); - const decodedEvents = this.routerContract.$decodeEvents(item.tx.log); - console.log('decodedEvents', decodedEvents); - - break; + for (const item of camelcaseKeysDeep(data)) { + if ( + item.tx.result !== 'ok' || + item.tx.return == 'invalid' || + !item.tx.function + ) { + continue; + } + // console.log('--------------------------------'); + const pairInfo = await this.extractPairInfoFromTransaction(item); + if (!pairInfo) { + // console.log('pairInfo', pairInfo); + continue; } + // console.log('pairInfo', pairInfo.pairAddress); + // console.log('--------------------------------'); - // await this.saveDexPairFromTransaction(camelcaseKeysDeep(item)); + await this.saveDexPair(pairInfo); + } + if (result.next) { + return await this.pullDexPairsFromMdw( + `${ACTIVE_NETWORK.middlewareUrl}${result.next}`, + ); } - // if (result.next) { - // return await this.pullDexPairsFromMdw( - // `${ACTIVE_NETWORK.middlewareUrl}${result.next}`, - // ); - // } return result; } + + async extractPairInfoFromTransaction(item: ITransaction) { + console.log('item.tx.function:', item.tx.function); + let decodedEvents = null; + try { + decodedEvents = this.routerContract.$decodeEvents(item.tx.log); + } catch (error: any) { + console.log('routerContract.$decodeEvents error', error?.message); + } + if (!decodedEvents) { + try { + decodedEvents = this.factoryContract.$decodeEvents(item.tx.log); + // console.log('factoryContract.$decodeEvents decodedEvents', decodedEvents); + } catch (error: any) { + console.log('factoryContract.$decodeEvents error', error?.message); + } + } + if (!decodedEvents) { + return null; + } + let pairAddress = decodedEvents.find( + (event) => event.contract?.name === 'IAedexV2Pair', + )?.contract?.address; + let token0Address = null; + let token1Address = null; + const args = item.tx.arguments; + + if ( + item.tx.function === TX_FUNCTIONS.swap_exact_tokens_for_tokens || + item.tx.function === TX_FUNCTIONS.swap_tokens_for_exact_tokens || + item.tx.function === TX_FUNCTIONS.swap_exact_tokens_for_ae || + item.tx.function === TX_FUNCTIONS.swap_tokens_for_exact_ae + ) { + token0Address = args[2].value[0]?.value; + token1Address = args[2].value[1]?.value; + } else if ( + item.tx.function === TX_FUNCTIONS.swap_exact_ae_for_tokens || + item.tx.function === TX_FUNCTIONS.swap_ae_for_exact_tokens + ) { + token0Address = args[1].value[0]?.value; + token1Address = args[1].value[1]?.value; + } else if (item.tx.function === TX_FUNCTIONS.add_liquidity) { + token0Address = args[0].value; + token1Address = args[1].value; + // console.log('args::', JSON.stringify(args, null, 2)); + } else if (item.tx.function === TX_FUNCTIONS.add_liquidity_ae) { + // this mean add new pair WAE -> Token + token0Address = DEX_CONTRACTS.wae; + token1Address = args[0].value; + } else { + // if (item.tx.function?.includes('liquidity')) { + // return null; + // } + // console.log('item.tx.function:', item.tx.function); + // console.log('args::', JSON.stringify(args, null, 2)); + } + if (!token0Address || !token1Address) { + return null; + } + + const token0 = await this.getOrCreateToken(token0Address); + const token1 = await this.getOrCreateToken(token1Address); + + return { pairAddress, token0, token1 }; + } + + private async getOrCreateToken(address: string) { + const token = await this.dexTokenRepository.findOne({ + where: { address }, + }); + if (token) { + return token; + } + const tokenData = await fetchJson( + `${ACTIVE_NETWORK.middlewareUrl}/v3/aex9/${address}`, + ); + return this.dexTokenRepository.save({ + address, + name: tokenData.name, + symbol: tokenData.symbol, + decimals: tokenData.decimals, + }); + } + + private async saveDexPair(pairInfo: { + pairAddress: string; + token0: DexToken; + token1: DexToken; + }) { + let pair = await this.dexPairRepository.findOne({ + where: { address: pairInfo.pairAddress }, + }); + if (pair) { + return pair; + } + + pair = await this.dexPairRepository.save({ + address: pairInfo.pairAddress, + token0: pairInfo.token0, + token1: pairInfo.token1, + }); + await this.updateTokenPairsCount(pairInfo.token0); + await this.updateTokenPairsCount(pairInfo.token1); + return pair; + } + + private async updateTokenPairsCount(token: DexToken) { + const pairsCount = await this.dexPairRepository + .createQueryBuilder('pair') + .where('pair.token0_address = :token0_address', { + token0_address: token.address, + }) + .orWhere('pair.token1_address = :token1_address', { + token1_address: token.address, + }) + .getCount(); + await this.dexTokenRepository.update(token.address, { + pairs_count: pairsCount, + }); + } } From 03783b887ea9119a11619fcd07285c511efa6cd6 Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Thu, 4 Sep 2025 14:36:27 +0100 Subject: [PATCH 12/14] feat(dex): disable social posts and DEX token pair retrieval; add PairTransaction entity for transaction management --- src/configs/constants.ts | 6 ++-- src/dex/entities/pair-transaction.entity.ts | 31 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 src/dex/entities/pair-transaction.entity.ts diff --git a/src/configs/constants.ts b/src/configs/constants.ts index 2089091..9bb90b3 100644 --- a/src/configs/constants.ts +++ b/src/configs/constants.ts @@ -117,9 +117,9 @@ export const UPDATE_TRENDING_TOKENS_ENABLED = false; export const PULL_INVITATIONS_ENABLED = false; export const PULL_ACCOUNTS_ENABLED = false; export const PULL_TRENDING_TAGS_ENABLED = false; -export const PULL_SOCIAL_POSTS_ENABLED = true; -export const PULL_DEX_TOKENS_ENABLED = true; -export const PULL_DEX_PAIRS_ENABLED = true; +export const PULL_SOCIAL_POSTS_ENABLED = false; +export const PULL_DEX_TOKENS_ENABLED = false; +export const PULL_DEX_PAIRS_ENABLED = false; /** * API Keys and Security diff --git a/src/dex/entities/pair-transaction.entity.ts b/src/dex/entities/pair-transaction.entity.ts new file mode 100644 index 0000000..09b26ca --- /dev/null +++ b/src/dex/entities/pair-transaction.entity.ts @@ -0,0 +1,31 @@ +import { DexToken } from './dex-token.entity'; +import { + CreateDateColumn, + Entity, + PrimaryColumn, + ManyToOne, + JoinColumn, + Column, +} from 'typeorm'; +import { Pair } from './pair.entity'; + +@Entity({ + name: 'pair_transactions', +}) +export class PairTransaction { + @PrimaryColumn() + tx_hash: string; + + @ManyToOne(() => Pair, (pair) => pair.address) + @JoinColumn({ name: 'pair_address' }) + pair: Pair; + + @Column() + tx_type: string; + + @CreateDateColumn({ + type: 'timestamp', + default: () => 'CURRENT_TIMESTAMP(6)', + }) + public created_at: Date; +} From a52d1785da3b349062254b1eafe9065b42c4614d Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Thu, 4 Sep 2025 14:52:53 +0100 Subject: [PATCH 13/14] feat(dex): enable DEX token and pair retrieval; add PairTransaction controller and service for transaction management --- src/configs/constants.ts | 6 +- src/dex/controllers/dex-tokens.controller.ts | 4 +- .../pair-transactions.controller.ts | 130 ++++++++++++++++++ src/dex/controllers/pairs.controller.ts | 4 +- src/dex/dex.module.ts | 20 ++- src/dex/dto/index.ts | 1 + src/dex/dto/pair-transaction.dto.ts | 29 ++++ src/dex/entities/pair-transaction.entity.ts | 7 +- src/dex/entities/pair.entity.ts | 9 +- src/dex/services/dex-sync.service.ts | 37 ++++- src/dex/services/pair-transaction.service.ts | 67 +++++++++ src/dex/services/pair.service.ts | 10 +- 12 files changed, 300 insertions(+), 24 deletions(-) create mode 100644 src/dex/controllers/pair-transactions.controller.ts create mode 100644 src/dex/dto/pair-transaction.dto.ts create mode 100644 src/dex/services/pair-transaction.service.ts diff --git a/src/configs/constants.ts b/src/configs/constants.ts index 9bb90b3..2089091 100644 --- a/src/configs/constants.ts +++ b/src/configs/constants.ts @@ -117,9 +117,9 @@ export const UPDATE_TRENDING_TOKENS_ENABLED = false; export const PULL_INVITATIONS_ENABLED = false; export const PULL_ACCOUNTS_ENABLED = false; export const PULL_TRENDING_TAGS_ENABLED = false; -export const PULL_SOCIAL_POSTS_ENABLED = false; -export const PULL_DEX_TOKENS_ENABLED = false; -export const PULL_DEX_PAIRS_ENABLED = false; +export const PULL_SOCIAL_POSTS_ENABLED = true; +export const PULL_DEX_TOKENS_ENABLED = true; +export const PULL_DEX_PAIRS_ENABLED = true; /** * API Keys and Security diff --git a/src/dex/controllers/dex-tokens.controller.ts b/src/dex/controllers/dex-tokens.controller.ts index 953f460..73e7366 100644 --- a/src/dex/controllers/dex-tokens.controller.ts +++ b/src/dex/controllers/dex-tokens.controller.ts @@ -18,8 +18,8 @@ import { DexTokenService } from '../services/dex-token.service'; import { DexTokenDto } from '../dto'; import { ApiOkResponsePaginated } from '@/utils/api-type'; -@Controller('dex-tokens') -@ApiTags('DEX Tokens') +@Controller('dex/tokens') +@ApiTags('DEX') export class DexTokensController { constructor(private readonly dexTokenService: DexTokenService) {} diff --git a/src/dex/controllers/pair-transactions.controller.ts b/src/dex/controllers/pair-transactions.controller.ts new file mode 100644 index 0000000..579f1e4 --- /dev/null +++ b/src/dex/controllers/pair-transactions.controller.ts @@ -0,0 +1,130 @@ +import { + Controller, + DefaultValuePipe, + Get, + NotFoundException, + Param, + ParseIntPipe, + Query, +} from '@nestjs/common'; +import { + ApiOperation, + ApiParam, + ApiQuery, + ApiTags, + ApiOkResponse, +} from '@nestjs/swagger'; +import { PairTransactionService } from '../services/pair-transaction.service'; +import { PairTransactionDto } from '../dto'; +import { ApiOkResponsePaginated } from '@/utils/api-type'; + +@Controller('dex/transactions') +@ApiTags('DEX') +export class PairTransactionsController { + constructor( + private readonly pairTransactionService: PairTransactionService, + ) {} + + @ApiQuery({ name: 'page', type: 'number', required: false }) + @ApiQuery({ name: 'limit', type: 'number', required: false }) + @ApiQuery({ + name: 'order_by', + enum: ['created_at', 'tx_type'], + required: false, + }) + @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) + @ApiQuery({ + name: 'pair_address', + type: 'string', + required: false, + description: 'Filter by specific pair address', + }) + @ApiQuery({ + name: 'tx_type', + type: 'string', + required: false, + description: 'Filter by transaction type', + }) + @ApiOperation({ + operationId: 'listAllPairTransactions', + summary: 'Get all pair transactions', + description: + 'Retrieve a paginated list of all DEX pair transactions with optional filtering and sorting', + }) + @ApiOkResponsePaginated(PairTransactionDto) + @Get() + async listAll( + @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, + @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, + @Query('order_by') orderBy: string = 'created_at', + @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', + @Query('pair_address') pairAddress?: string, + @Query('tx_type') txType?: string, + ) { + return this.pairTransactionService.findAll( + { page, limit }, + orderBy, + orderDirection, + pairAddress, + txType, + ); + } + + @ApiParam({ + name: 'txHash', + type: 'string', + description: 'Transaction hash', + }) + @ApiOperation({ + operationId: 'getPairTransactionByTxHash', + summary: 'Get pair transaction by transaction hash', + description: 'Retrieve a specific pair transaction by its transaction hash', + }) + @ApiOkResponse({ type: PairTransactionDto }) + @Get(':txHash') + async getByTxHash(@Param('txHash') txHash: string) { + const pairTransaction = + await this.pairTransactionService.findByTxHash(txHash); + if (!pairTransaction) { + throw new NotFoundException( + `Pair transaction with hash ${txHash} not found`, + ); + } + return pairTransaction; + } + + @ApiParam({ + name: 'pairAddress', + type: 'string', + description: 'Pair contract address', + }) + @ApiQuery({ name: 'page', type: 'number', required: false }) + @ApiQuery({ name: 'limit', type: 'number', required: false }) + @ApiQuery({ + name: 'order_by', + enum: ['created_at', 'tx_type'], + required: false, + }) + @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) + @ApiOperation({ + operationId: 'getPairTransactionsByPairAddress', + summary: 'Get pair transactions by pair address', + description: 'Retrieve paginated transactions for a specific pair', + }) + @ApiOkResponsePaginated(PairTransactionDto) + @Get('pair/:pairAddress') + async getByPairAddress( + @Param('pairAddress') pairAddress: string, + @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, + @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, + @Query('order_by') orderBy: string = 'created_at', + @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', + ) { + return this.pairTransactionService.findByPairAddress( + pairAddress, + { page, limit }, + orderBy, + orderDirection, + ); + } +} diff --git a/src/dex/controllers/pairs.controller.ts b/src/dex/controllers/pairs.controller.ts index ef29252..c35a908 100644 --- a/src/dex/controllers/pairs.controller.ts +++ b/src/dex/controllers/pairs.controller.ts @@ -18,8 +18,8 @@ import { PairService } from '../services/pair.service'; import { PairDto } from '../dto'; import { ApiOkResponsePaginated } from '@/utils/api-type'; -@Controller('pairs') -@ApiTags('DEX Pairs') +@Controller('dex/pairs') +@ApiTags('DEX') export class PairsController { constructor(private readonly pairService: PairService) {} diff --git a/src/dex/dex.module.ts b/src/dex/dex.module.ts index 169daf3..16e7fde 100644 --- a/src/dex/dex.module.ts +++ b/src/dex/dex.module.ts @@ -4,21 +4,33 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { DexTokensController } from './controllers/dex-tokens.controller'; import { PairsController } from './controllers/pairs.controller'; +import { PairTransactionsController } from './controllers/pair-transactions.controller'; import { DexToken } from './entities/dex-token.entity'; import { Pair } from './entities/pair.entity'; +import { PairTransaction } from './entities/pair-transaction.entity'; import { DexSyncService } from './services/dex-sync.service'; import { DexTokenService } from './services/dex-token.service'; import { PairService } from './services/pair.service'; +import { PairTransactionService } from './services/pair-transaction.service'; @Module({ imports: [ AeModule, TransactionsModule, - TypeOrmModule.forFeature([Pair, DexToken]), + TypeOrmModule.forFeature([Pair, DexToken, PairTransaction]), + ], + providers: [ + PairService, + DexTokenService, + PairTransactionService, + DexSyncService, + ], + exports: [PairService, DexTokenService, PairTransactionService], + controllers: [ + PairsController, + DexTokensController, + PairTransactionsController, ], - providers: [PairService, DexTokenService, DexSyncService], - exports: [PairService, DexTokenService], - controllers: [PairsController, DexTokensController], }) export class DexModule { // diff --git a/src/dex/dto/index.ts b/src/dex/dto/index.ts index 21304d2..5c9a969 100644 --- a/src/dex/dto/index.ts +++ b/src/dex/dto/index.ts @@ -1,2 +1,3 @@ export * from './pair.dto'; export * from './dex-token.dto'; +export * from './pair-transaction.dto'; diff --git a/src/dex/dto/pair-transaction.dto.ts b/src/dex/dto/pair-transaction.dto.ts new file mode 100644 index 0000000..ceab38b --- /dev/null +++ b/src/dex/dto/pair-transaction.dto.ts @@ -0,0 +1,29 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { PairDto } from './pair.dto'; + +export class PairTransactionDto { + @ApiProperty({ + description: 'Transaction hash', + example: 'th_2AfnEfCSPx4A6VjMBfDfqHNYcqDJjuJjGV1qhqP5qNKNBvYfE2', + }) + tx_hash: string; + + @ApiProperty({ + description: 'Associated pair', + type: () => PairDto, + }) + pair: PairDto; + + @ApiProperty({ + description: + 'Transaction type (e.g., swap, add_liquidity, remove_liquidity)', + example: 'swap_exact_tokens_for_tokens', + }) + tx_type: string; + + @ApiProperty({ + description: 'Transaction creation timestamp', + example: '2024-01-01T00:00:00.000Z', + }) + created_at: Date; +} diff --git a/src/dex/entities/pair-transaction.entity.ts b/src/dex/entities/pair-transaction.entity.ts index 09b26ca..dfc8430 100644 --- a/src/dex/entities/pair-transaction.entity.ts +++ b/src/dex/entities/pair-transaction.entity.ts @@ -1,11 +1,10 @@ -import { DexToken } from './dex-token.entity'; import { + Column, CreateDateColumn, Entity, - PrimaryColumn, - ManyToOne, JoinColumn, - Column, + ManyToOne, + PrimaryColumn, } from 'typeorm'; import { Pair } from './pair.entity'; diff --git a/src/dex/entities/pair.entity.ts b/src/dex/entities/pair.entity.ts index ecb4ec8..124af26 100644 --- a/src/dex/entities/pair.entity.ts +++ b/src/dex/entities/pair.entity.ts @@ -1,11 +1,12 @@ import { DexToken } from './dex-token.entity'; +import { PairTransaction } from './pair-transaction.entity'; import { CreateDateColumn, Entity, PrimaryColumn, ManyToOne, JoinColumn, - Column, + OneToMany, } from 'typeorm'; @Entity({ @@ -23,10 +24,8 @@ export class Pair { @JoinColumn({ name: 'token1_address' }) token1: DexToken; - @Column({ - default: 0, - }) - transactions_count: number; + @OneToMany(() => PairTransaction, (transaction) => transaction.pair) + transactions: PairTransaction[]; @CreateDateColumn({ type: 'timestamp', diff --git a/src/dex/services/dex-sync.service.ts b/src/dex/services/dex-sync.service.ts index 78bd87e..46e972c 100644 --- a/src/dex/services/dex-sync.service.ts +++ b/src/dex/services/dex-sync.service.ts @@ -17,6 +17,8 @@ import { Repository } from 'typeorm'; import { DEX_CONTRACTS } from '../config/dex-contracts.config'; import { DexToken } from '../entities/dex-token.entity'; import { Pair } from '../entities/pair.entity'; +import { PairTransaction } from '../entities/pair-transaction.entity'; +import moment from 'moment'; @Injectable() export class DexSyncService { @@ -27,6 +29,8 @@ export class DexSyncService { private readonly dexTokenRepository: Repository, @InjectRepository(Pair) private readonly dexPairRepository: Repository, + @InjectRepository(PairTransaction) + private readonly dexPairTransactionRepository: Repository, private aeSdkService: AeSdkService, ) { @@ -38,6 +42,7 @@ export class DexSyncService { console.log('==== DexSyncService ===='); console.log('========================'); // + return; this.routerContract = await this.aeSdkService.sdk.initializeContract({ aci: routerInterface, @@ -87,7 +92,8 @@ export class DexSyncService { // console.log('pairInfo', pairInfo.pairAddress); // console.log('--------------------------------'); - await this.saveDexPair(pairInfo); + const pair = await this.saveDexPair(pairInfo); + await this.saveDexPairTransaction(pair, item); } if (result.next) { return await this.pullDexPairsFromMdw( @@ -116,7 +122,7 @@ export class DexSyncService { if (!decodedEvents) { return null; } - let pairAddress = decodedEvents.find( + const pairAddress = decodedEvents.find( (event) => event.contract?.name === 'IAedexV2Pair', )?.contract?.address; let token0Address = null; @@ -216,4 +222,31 @@ export class DexSyncService { pairs_count: pairsCount, }); } + + private async saveDexPairTransaction(pair: Pair, item: ITransaction) { + const existingTransaction = await this.dexPairTransactionRepository + .createQueryBuilder('pairTransaction') + .where('pairTransaction.tx_hash = :tx_hash', { + tx_hash: item.hash, + }) + .getOne(); + if (existingTransaction) { + await this.dexPairTransactionRepository.update( + existingTransaction.tx_hash, + { + pair: pair, + tx_type: item.tx.function, + tx_hash: item.hash, + created_at: moment(item.microTime).toDate(), + }, + ); + return existingTransaction; + } + await this.dexPairTransactionRepository.save({ + pair: pair, + tx_type: item.tx.function, + tx_hash: item.hash, + created_at: moment(item.microTime).toDate(), + }); + } } diff --git a/src/dex/services/pair-transaction.service.ts b/src/dex/services/pair-transaction.service.ts new file mode 100644 index 0000000..acef19e --- /dev/null +++ b/src/dex/services/pair-transaction.service.ts @@ -0,0 +1,67 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { PairTransaction } from '../entities/pair-transaction.entity'; +import { + IPaginationOptions, + paginate, + Pagination, +} from 'nestjs-typeorm-paginate'; + +@Injectable() +export class PairTransactionService { + constructor( + @InjectRepository(PairTransaction) + private readonly pairTransactionRepository: Repository, + ) {} + + async findAll( + options: IPaginationOptions, + orderBy: string = 'created_at', + orderDirection: 'ASC' | 'DESC' = 'DESC', + pairAddress?: string, + txType?: string, + ): Promise> { + const query = this.pairTransactionRepository + .createQueryBuilder('pairTransaction') + .leftJoinAndSelect('pairTransaction.pair', 'pair') + .leftJoinAndSelect('pair.token0', 'token0') + .leftJoinAndSelect('pair.token1', 'token1'); + + // Filter by pair address if provided + if (pairAddress) { + query.andWhere('pair.address = :pairAddress', { pairAddress }); + } + + // Filter by transaction type if provided + if (txType) { + query.andWhere('pairTransaction.tx_type = :txType', { txType }); + } + + // Add ordering + if (orderBy) { + query.orderBy(`pairTransaction.${orderBy}`, orderDirection); + } + + return paginate(query, options); + } + + async findByTxHash(txHash: string): Promise { + return this.pairTransactionRepository + .createQueryBuilder('pairTransaction') + .leftJoinAndSelect('pairTransaction.pair', 'pair') + .leftJoinAndSelect('pair.token0', 'token0') + .leftJoinAndSelect('pair.token1', 'token1') + .where('pairTransaction.tx_hash = :txHash', { txHash }) + .getOne(); + } + + async findByPairAddress( + pairAddress: string, + options: IPaginationOptions, + orderBy: string = 'created_at', + orderDirection: 'ASC' | 'DESC' = 'DESC', + ): Promise> { + return this.findAll(options, orderBy, orderDirection, pairAddress); + } +} diff --git a/src/dex/services/pair.service.ts b/src/dex/services/pair.service.ts index 182eeeb..28863f1 100644 --- a/src/dex/services/pair.service.ts +++ b/src/dex/services/pair.service.ts @@ -23,10 +23,15 @@ export class PairService { const query = this.pairRepository .createQueryBuilder('pair') .leftJoinAndSelect('pair.token0', 'token0') - .leftJoinAndSelect('pair.token1', 'token1'); + .leftJoinAndSelect('pair.token1', 'token1') + .loadRelationCountAndMap('pair.transactions_count', 'pair.transactions'); if (orderBy) { - query.orderBy(`pair.${orderBy}`, orderDirection); + if (orderBy === 'transactions_count') { + query.orderBy('pair.transactions_count', orderDirection); + } else { + query.orderBy(`pair.${orderBy}`, orderDirection); + } } return paginate(query, options); @@ -37,6 +42,7 @@ export class PairService { .createQueryBuilder('pair') .leftJoinAndSelect('pair.token0', 'token0') .leftJoinAndSelect('pair.token1', 'token1') + .loadRelationCountAndMap('pair.transactions_count', 'pair.transactions') .where('pair.address = :address', { address }) .getOne(); } From ef2f08ac53c6478177b015c5096c13cc53473ad5 Mon Sep 17 00:00:00 2001 From: Badi Ifaoui Date: Thu, 4 Sep 2025 14:53:51 +0100 Subject: [PATCH 14/14] Revert "Features/dex" --- .github/workflows/deploy_develop.yaml | 28 +- .github/workflows/deploy_main.yaml | 28 +- .github/workflows/deploy_staging.yaml | 27 +- .github/workflows/ssh_deploy.yaml | 2 +- package-lock.json | 16 - package.json | 1 - src/app.module.ts | 4 - src/app.service.ts | 31 +- src/bcl/bcl.module.ts | 2 +- src/bcl/services/sync-transactions.service.ts | 31 +- src/configs/constants.ts | 28 +- src/dex/config/dex-contracts.config.ts | 9 - src/dex/controllers/dex-tokens.controller.ts | 76 -- .../pair-transactions.controller.ts | 130 ---- src/dex/controllers/pairs.controller.ts | 70 -- src/dex/dex.module.ts | 37 - src/dex/dto/dex-token.dto.ts | 39 - src/dex/dto/index.ts | 3 - src/dex/dto/pair-transaction.dto.ts | 29 - src/dex/dto/pair.dto.ts | 34 - src/dex/entities/dex-token.entity.ts | 31 - src/dex/entities/pair-transaction.entity.ts | 30 - src/dex/entities/pair.entity.ts | 35 - src/dex/services/dex-sync.service.ts | 252 ------ src/dex/services/dex-token.service.ts | 38 - src/dex/services/pair-transaction.service.ts | 67 -- src/dex/services/pair.service.ts | 49 -- src/social/README.md | 157 ---- src/social/config/post-contracts.config.ts | 46 -- src/social/controllers/posts.controller.ts | 137 ---- src/social/dto/index.ts | 1 - src/social/dto/post.dto.ts | 75 -- src/social/entities/post.entity.ts | 61 -- src/social/interfaces/post.interfaces.ts | 94 --- src/social/post.module.ts | 17 - src/social/services/post.service.spec.ts | 332 -------- src/social/services/post.service.ts | 724 ------------------ src/social/utils/content-parser.util.spec.ts | 283 ------- src/social/utils/content-parser.util.ts | 147 ---- .../transaction-history.service.spec.ts | 4 +- 40 files changed, 108 insertions(+), 3097 deletions(-) delete mode 100644 src/dex/config/dex-contracts.config.ts delete mode 100644 src/dex/controllers/dex-tokens.controller.ts delete mode 100644 src/dex/controllers/pair-transactions.controller.ts delete mode 100644 src/dex/controllers/pairs.controller.ts delete mode 100644 src/dex/dex.module.ts delete mode 100644 src/dex/dto/dex-token.dto.ts delete mode 100644 src/dex/dto/index.ts delete mode 100644 src/dex/dto/pair-transaction.dto.ts delete mode 100644 src/dex/dto/pair.dto.ts delete mode 100644 src/dex/entities/dex-token.entity.ts delete mode 100644 src/dex/entities/pair-transaction.entity.ts delete mode 100644 src/dex/entities/pair.entity.ts delete mode 100644 src/dex/services/dex-sync.service.ts delete mode 100644 src/dex/services/dex-token.service.ts delete mode 100644 src/dex/services/pair-transaction.service.ts delete mode 100644 src/dex/services/pair.service.ts delete mode 100644 src/social/README.md delete mode 100644 src/social/config/post-contracts.config.ts delete mode 100644 src/social/controllers/posts.controller.ts delete mode 100644 src/social/dto/index.ts delete mode 100644 src/social/dto/post.dto.ts delete mode 100644 src/social/entities/post.entity.ts delete mode 100644 src/social/interfaces/post.interfaces.ts delete mode 100644 src/social/post.module.ts delete mode 100644 src/social/services/post.service.spec.ts delete mode 100644 src/social/services/post.service.ts delete mode 100644 src/social/utils/content-parser.util.spec.ts delete mode 100644 src/social/utils/content-parser.util.ts diff --git a/.github/workflows/deploy_develop.yaml b/.github/workflows/deploy_develop.yaml index 10c6ea0..694c954 100644 --- a/.github/workflows/deploy_develop.yaml +++ b/.github/workflows/deploy_develop.yaml @@ -15,17 +15,17 @@ on: required: false type: string jobs: - deploy: - name: superhero + deploy_wordcraft: + name: wordcraft uses: ./.github/workflows/ssh_deploy.yaml with: VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "develop-superhero-api-mainnet" + CONTAINER_NAME: "develop-wordcraftfun-api-mainnet" secrets: AE_NETWORK_ID: "ae_mainnet" API_HOST_PORT: "3043" DB_DATABASE: "api" - DEPLOY_HOST: "api.dev.tokensale.org" + DEPLOY_HOST: "api.dev.wordcraft.fun" DEPLOY_KEY: ${{ secrets.DEV_DEPLOY_KEY }} DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} DB_USER: ${{ secrets.DEV_DB_USER }} @@ -34,4 +34,22 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDING_TAGS_API_KEY }} - + deploy_trendminer: + name: trendminer + uses: ./.github/workflows/ssh_deploy.yaml + with: + VERSION: ${{ inputs.VERSION }} + CONTAINER_NAME: "develop-trendminerfun-api-mainnet" + secrets: + AE_NETWORK_ID: "ae_mainnet" + API_HOST_PORT: "3043" + DB_DATABASE: "api" + DEPLOY_HOST: "api.dev.trendminer.fun" + DEPLOY_KEY: ${{ secrets.DEV_TRENDMINERFUN_DEPLOY_KEY }} + DEPLOY_USERNAME: ${{ secrets.DEV_DEPLOY_USERNAME }} + DB_USER: ${{ secrets.DEV_DB_USER }} + DB_PASSWORD: ${{ secrets.DEV_DB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} + TRENDING_TAGS_API_KEY: ${{ secrets.DEV_TRENDMINERFUN_TRENDING_TAGS_API_KEY }} diff --git a/.github/workflows/deploy_main.yaml b/.github/workflows/deploy_main.yaml index 65c52c1..9d43edb 100644 --- a/.github/workflows/deploy_main.yaml +++ b/.github/workflows/deploy_main.yaml @@ -15,17 +15,17 @@ on: required: false type: string jobs: - deploy: - name: superhero + deploy_wordcraft: + name: wordcraft uses: ./.github/workflows/ssh_deploy.yaml with: VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "production-superhero-api-mainnet" + CONTAINER_NAME: "production-wordcraftfun-api-mainnet" secrets: AE_NETWORK_ID: "ae_mainnet" API_HOST_PORT: "3033" DB_DATABASE: "api" - DEPLOY_HOST: "api.superhero.com" + DEPLOY_HOST: "api.wordcraft.fun" DEPLOY_KEY: ${{ secrets.PROD_DEPLOY_KEY }} DEPLOY_USERNAME: ${{ secrets.PROD_DEPLOY_USERNAME }} DB_USER: ${{ secrets.PROD_DB_USER }} @@ -34,4 +34,22 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.TRENDING_TAGS_API_KEY }} - \ No newline at end of file + # Uncomment when ready to deploy to trendminer production + # deploy_trendminer: + # name: trendminer + # uses: ./.github/workflows/ssh_deploy.yaml + # with: + # VERSION: ${{ inputs.VERSION }} + # CONTAINER_NAME: "production-trendminerfun-api-mainnet" + # secrets: + # AE_NETWORK_ID: "ae_mainnet" + # API_HOST_PORT: "3033" + # DB_DATABASE: "api" + # DEPLOY_HOST: "api.trendminer.fun" + # DEPLOY_KEY: ${{ secrets.PROD_TRENDMINERFUN_DEPLOY_KEY }} + # DEPLOY_USERNAME: ${{ secrets.PROD_DEPLOY_USERNAME }} + # DB_USER: ${{ secrets.PROD_DB_USER }} + # DB_PASSWORD: ${{ secrets.PROD_DB_PASSWORD }} + # DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} diff --git a/.github/workflows/deploy_staging.yaml b/.github/workflows/deploy_staging.yaml index 70c57b9..53b8aeb 100644 --- a/.github/workflows/deploy_staging.yaml +++ b/.github/workflows/deploy_staging.yaml @@ -15,17 +15,17 @@ on: required: false type: string jobs: - deploy: - name: superhero + deploy_wordcraft: + name: wordcraft uses: ./.github/workflows/ssh_deploy.yaml with: VERSION: ${{ inputs.VERSION }} - CONTAINER_NAME: "staging-superhero-api-mainnet" + CONTAINER_NAME: "staging-wordcraftfun-api-mainnet" secrets: AE_NETWORK_ID: "ae_mainnet" API_HOST_PORT: "3033" DB_DATABASE: "api" - DEPLOY_HOST: "api.stag.superhero.com" + DEPLOY_HOST: "api.stag.wordcraft.fun" DEPLOY_KEY: ${{ secrets.STAG_DEPLOY_KEY }} DEPLOY_USERNAME: ${{ secrets.STAG_DEPLOY_USERNAME }} DB_USER: ${{ secrets.STAG_DB_USER }} @@ -34,3 +34,22 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} TRENDING_TAGS_API_KEY: ${{ secrets.STAG_TRENDING_TAGS_API_KEY }} + # Uncomment when ready to deploy to trendminer staging + # deploy_trendminer: + # name: trendminer + # uses: ./.github/workflows/ssh_deploy.yaml + # with: + # VERSION: ${{ inputs.VERSION }} + # CONTAINER_NAME: "staging-trendminerfun-api-mainnet" + # secrets: + # AE_NETWORK_ID: "ae_mainnet" + # API_HOST_PORT: "3033" + # DB_DATABASE: "api" + # DEPLOY_HOST: "api.stag.trendminer.fun" + # DEPLOY_KEY: ${{ secrets.STAG_TRENDMINERFUN_DEPLOY_KEY }} + # DEPLOY_USERNAME: ${{ secrets.STAG_DEPLOY_USERNAME }} + # DB_USER: ${{ secrets.STAG_DB_USER }} + # DB_PASSWORD: ${{ secrets.STAG_DB_PASSWORD }} + # DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} diff --git a/.github/workflows/ssh_deploy.yaml b/.github/workflows/ssh_deploy.yaml index 78f62fb..18be025 100644 --- a/.github/workflows/ssh_deploy.yaml +++ b/.github/workflows/ssh_deploy.yaml @@ -10,7 +10,7 @@ on: type: string CONTAINER_NAME: description: "Container name" - default: "superhero-api" + default: "wordcraftfun-api" required: false type: string secrets: diff --git a/package-lock.json b/package-lock.json index 93b4a05..63b5591 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,6 @@ "camelcase-keys-deep": "^0.1.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", - "dex-contracts-v2": "github:aeternity/dex-contracts-v2", "hbs": "^4.2.0", "keyv": "^5.2.3", "moment": "^2.30.1", @@ -4654,15 +4653,6 @@ "node": ">=8" } }, - "node_modules/dex-contracts-v2": { - "version": "1.0.0", - "resolved": "git+ssh://git@github.com/aeternity/dex-contracts-v2.git#9992835ac3e2d0074e5aac03612aea1baea07839", - "hasInstallScript": true, - "license": "ISC", - "dependencies": { - "fs": "^0.0.1-security" - } - }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -5744,12 +5734,6 @@ "node": ">= 0.6" } }, - "node_modules/fs": { - "version": "0.0.1-security", - "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", - "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==", - "license": "ISC" - }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", diff --git a/package.json b/package.json index 309f117..ca00bf9 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,6 @@ "@nestjs/typeorm": "^10.0.2", "@nestjs/websockets": "^10.3.9", "bctsl-sdk": "git+ssh://git@github.com/bctsl/bctsl-sdk#v1.0.0", - "dex-contracts-v2": "github:aeternity/dex-contracts-v2", "bignumber.js": "^9.1.2", "bull": "^4.15.0", "cache-manager": "^6.4.0", diff --git a/src/app.module.ts b/src/app.module.ts index b2c3bb1..80b1c16 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -23,8 +23,6 @@ import { AnalyticsModule } from './analytics/analytics.module'; import { AffiliationModule } from './affiliation/affiliation.module'; import { AccountModule } from './account/account.module'; import { TrendingTagsModule } from './trending-tags/trending-tags.module'; -import { PostModule } from './social/post.module'; -import { DexModule } from './dex/dex.module'; @Module({ imports: [ @@ -61,8 +59,6 @@ import { DexModule } from './dex/dex.module'; AffiliationModule, AccountModule, TrendingTagsModule, - PostModule, - DexModule, ], controllers: [AppController], providers: [AppService], diff --git a/src/app.service.ts b/src/app.service.ts index c8293ae..d19a556 100644 --- a/src/app.service.ts +++ b/src/app.service.ts @@ -2,31 +2,22 @@ import { InjectQueue } from '@nestjs/bull'; import { Injectable } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { Queue } from 'bull'; -import moment, { Moment } from 'moment'; import { AePricingService } from './ae-pricing/ae-pricing.service'; import { CommunityFactoryService } from './ae/community-factory.service'; -import { WebSocketService } from './ae/websocket.service'; -import { SyncTransactionsService } from './bcl/services/sync-transactions.service'; -import { PostService } from './social/services/post.service'; import { DELETE_OLD_TOKENS_QUEUE } from './tokens/queues/constants'; -import { ITransaction } from './utils/types'; - +import moment, { Moment } from 'moment'; @Injectable() export class AppService { startedAt: Moment; constructor( private communityFactoryService: CommunityFactoryService, private aePricingService: AePricingService, - private websocketService: WebSocketService, - private syncTransactionsService: SyncTransactionsService, - private postService: PostService, @InjectQueue(DELETE_OLD_TOKENS_QUEUE) private readonly deleteOldTokensQueue: Queue, ) { this.init(); this.startedAt = moment(); - this.setupLiveSync(); } async init() { @@ -39,26 +30,6 @@ export class AppService { }); } - setupLiveSync() { - let syncedTransactions = []; - - this.websocketService.subscribeForTransactionsUpdates( - (transaction: ITransaction) => { - // Prevent duplicate transactions - if (!syncedTransactions.includes(transaction.hash)) { - this.syncTransactionsService.handleLiveTransaction(transaction); - this.postService.handleLiveTransaction(transaction); - } - syncedTransactions.push(transaction.hash); - - // Reset synced transactions after 100 transactions - if (syncedTransactions.length > 100) { - syncedTransactions = []; - } - }, - ); - } - @Cron(CronExpression.EVERY_HOUR) syncAeCoinPricing() { this.aePricingService.pullAndSaveCoinCurrencyRates(); diff --git a/src/bcl/bcl.module.ts b/src/bcl/bcl.module.ts index cdf5ed6..ba04445 100644 --- a/src/bcl/bcl.module.ts +++ b/src/bcl/bcl.module.ts @@ -43,7 +43,7 @@ import { VerifyTransactionsService } from './services/verify-transactions.servic FixHoldersService, VerifyTransactionsService, ], - exports: [SyncBlocksService, SyncTransactionsService], + exports: [SyncBlocksService], controllers: [DebugFailedTransactionsController], }) export class BclModule { diff --git a/src/bcl/services/sync-transactions.service.ts b/src/bcl/services/sync-transactions.service.ts index 6c7f6b2..585d784 100644 --- a/src/bcl/services/sync-transactions.service.ts +++ b/src/bcl/services/sync-transactions.service.ts @@ -1,4 +1,5 @@ -import { ACTIVE_NETWORK, TX_FUNCTIONS } from '@/configs'; +import { WebSocketService } from '@/ae/websocket.service'; +import { ACTIVE_NETWORK, LIVE_SYNCING_ENABLED, TX_FUNCTIONS } from '@/configs'; import { TransactionService } from '@/transactions/services/transaction.service'; import { fetchJson } from '@/utils/common'; import { ITransaction } from '@/utils/types'; @@ -13,6 +14,7 @@ export class SyncTransactionsService { private readonly logger = new Logger(SyncTransactionsService.name); constructor( + private websocketService: WebSocketService, private readonly transactionService: TransactionService, @InjectRepository(FailedTransaction) @@ -21,10 +23,31 @@ export class SyncTransactionsService { // } - async handleLiveTransaction(transaction: ITransaction) { - if (Object.values(TX_FUNCTIONS).includes(transaction.tx.function)) { - this.transactionService.saveTransaction(transaction, null, true); + onModuleInit() { + this.setupLiveSync(); + } + + setupLiveSync() { + if (!LIVE_SYNCING_ENABLED) { + return; } + let syncedTransactions = []; + + this.websocketService.subscribeForTransactionsUpdates( + (transaction: ITransaction) => { + if (Object.values(TX_FUNCTIONS).includes(transaction.tx.function)) { + // Prevent duplicate transactions + if (!syncedTransactions.includes(transaction.hash)) { + syncedTransactions.push(transaction.hash); + this.transactionService.saveTransaction(transaction, null, true); + } + } + // Reset synced transactions after 100 transactions + if (syncedTransactions.length > 100) { + syncedTransactions = []; + } + }, + ); } async fetchAndSyncTransactions( diff --git a/src/configs/constants.ts b/src/configs/constants.ts index 2089091..2b4551b 100644 --- a/src/configs/constants.ts +++ b/src/configs/constants.ts @@ -81,17 +81,6 @@ export const TX_FUNCTIONS = { buy: 'buy', sell: 'sell', create_community: 'create_community', - - // dex(swap) - swap_exact_tokens_for_tokens: 'swap_exact_tokens_for_tokens', - swap_tokens_for_exact_tokens: 'swap_tokens_for_exact_tokens', - swap_exact_ae_for_tokens: 'swap_exact_ae_for_tokens', - swap_exact_tokens_for_ae: 'swap_exact_tokens_for_ae', - swap_tokens_for_exact_ae: 'swap_tokens_for_exact_ae', - swap_ae_for_exact_tokens: 'swap_ae_for_exact_tokens', - - add_liquidity: 'add_liquidity', - add_liquidity_ae: 'add_liquidity_ae', } as const; export const WAIT_TIME_WHEN_REQUEST_FAILED = 3000; // 3 seconds @@ -110,16 +99,13 @@ export const MAX_RETRIES_FOR_FAILED_TRANSACTIONS = 10; export const MAX_TOKENS_TO_CHECK_WITHOUT_HOLDERS = 20; -export const SYNCING_ENABLED = false; -export const LIVE_SYNCING_ENABLED = false; -export const PERIODIC_SYNCING_ENABLED = false; -export const UPDATE_TRENDING_TOKENS_ENABLED = false; -export const PULL_INVITATIONS_ENABLED = false; -export const PULL_ACCOUNTS_ENABLED = false; -export const PULL_TRENDING_TAGS_ENABLED = false; -export const PULL_SOCIAL_POSTS_ENABLED = true; -export const PULL_DEX_TOKENS_ENABLED = true; -export const PULL_DEX_PAIRS_ENABLED = true; +export const SYNCING_ENABLED = true; +export const LIVE_SYNCING_ENABLED = true; +export const PERIODIC_SYNCING_ENABLED = true; +export const UPDATE_TRENDING_TOKENS_ENABLED = true; +export const PULL_INVITATIONS_ENABLED = true; +export const PULL_ACCOUNTS_ENABLED = true; +export const PULL_TRENDING_TAGS_ENABLED = true; /** * API Keys and Security diff --git a/src/dex/config/dex-contracts.config.ts b/src/dex/config/dex-contracts.config.ts deleted file mode 100644 index bad223a..0000000 --- a/src/dex/config/dex-contracts.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Configuration for supported dex contracts - */ -export const DEX_CONTRACTS = { - factory: 'ct_2mfj3FoZxnhkSw5RZMcP8BfPoB1QR4QiYGNCdkAvLZ1zfF6paW', - router: 'ct_azbNZ1XrPjXfqBqbAh1ffLNTQ1sbnuUDFvJrXjYz7JQA1saQ3', - wae: 'ct_J3zBY8xxjsRr3QojETNw48Eb38fjvEuJKkQ6KzECvubvEcvCa', - aeeth: 'ct_ryTY1mxqjCjq1yBn9i6HDaCSdA6thXUFZTA84EMzbWd1SLKdh', -}; diff --git a/src/dex/controllers/dex-tokens.controller.ts b/src/dex/controllers/dex-tokens.controller.ts deleted file mode 100644 index 73e7366..0000000 --- a/src/dex/controllers/dex-tokens.controller.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { - Controller, - DefaultValuePipe, - Get, - NotFoundException, - Param, - ParseIntPipe, - Query, -} from '@nestjs/common'; -import { - ApiOperation, - ApiParam, - ApiQuery, - ApiTags, - ApiOkResponse, -} from '@nestjs/swagger'; -import { DexTokenService } from '../services/dex-token.service'; -import { DexTokenDto } from '../dto'; -import { ApiOkResponsePaginated } from '@/utils/api-type'; - -@Controller('dex/tokens') -@ApiTags('DEX') -export class DexTokensController { - constructor(private readonly dexTokenService: DexTokenService) {} - - @ApiQuery({ name: 'page', type: 'number', required: false }) - @ApiQuery({ name: 'limit', type: 'number', required: false }) - @ApiQuery({ - name: 'order_by', - enum: ['pairs_count', 'name', 'symbol', 'created_at'], - required: false, - }) - @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) - @ApiOperation({ - operationId: 'listAllDexTokens', - summary: 'Get all DEX tokens', - description: - 'Retrieve a paginated list of all DEX tokens with optional sorting', - }) - @ApiOkResponsePaginated(DexTokenDto) - @Get() - async listAll( - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, - @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, - @Query('order_by') orderBy: string = 'created_at', - @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', - ) { - return this.dexTokenService.findAll( - { page, limit }, - orderBy, - orderDirection, - ); - } - - @ApiParam({ - name: 'address', - type: 'string', - description: 'Token contract address', - }) - @ApiOperation({ - operationId: 'getDexTokenByAddress', - summary: 'Get DEX token by address', - description: 'Retrieve a specific DEX token by its contract address', - }) - @ApiOkResponse({ type: DexTokenDto }) - @Get(':address') - async getByAddress(@Param('address') address: string) { - const token = await this.dexTokenService.findByAddress(address); - if (!token) { - throw new NotFoundException( - `DEX token with address ${address} not found`, - ); - } - return token; - } -} diff --git a/src/dex/controllers/pair-transactions.controller.ts b/src/dex/controllers/pair-transactions.controller.ts deleted file mode 100644 index 579f1e4..0000000 --- a/src/dex/controllers/pair-transactions.controller.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - Controller, - DefaultValuePipe, - Get, - NotFoundException, - Param, - ParseIntPipe, - Query, -} from '@nestjs/common'; -import { - ApiOperation, - ApiParam, - ApiQuery, - ApiTags, - ApiOkResponse, -} from '@nestjs/swagger'; -import { PairTransactionService } from '../services/pair-transaction.service'; -import { PairTransactionDto } from '../dto'; -import { ApiOkResponsePaginated } from '@/utils/api-type'; - -@Controller('dex/transactions') -@ApiTags('DEX') -export class PairTransactionsController { - constructor( - private readonly pairTransactionService: PairTransactionService, - ) {} - - @ApiQuery({ name: 'page', type: 'number', required: false }) - @ApiQuery({ name: 'limit', type: 'number', required: false }) - @ApiQuery({ - name: 'order_by', - enum: ['created_at', 'tx_type'], - required: false, - }) - @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) - @ApiQuery({ - name: 'pair_address', - type: 'string', - required: false, - description: 'Filter by specific pair address', - }) - @ApiQuery({ - name: 'tx_type', - type: 'string', - required: false, - description: 'Filter by transaction type', - }) - @ApiOperation({ - operationId: 'listAllPairTransactions', - summary: 'Get all pair transactions', - description: - 'Retrieve a paginated list of all DEX pair transactions with optional filtering and sorting', - }) - @ApiOkResponsePaginated(PairTransactionDto) - @Get() - async listAll( - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, - @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, - @Query('order_by') orderBy: string = 'created_at', - @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', - @Query('pair_address') pairAddress?: string, - @Query('tx_type') txType?: string, - ) { - return this.pairTransactionService.findAll( - { page, limit }, - orderBy, - orderDirection, - pairAddress, - txType, - ); - } - - @ApiParam({ - name: 'txHash', - type: 'string', - description: 'Transaction hash', - }) - @ApiOperation({ - operationId: 'getPairTransactionByTxHash', - summary: 'Get pair transaction by transaction hash', - description: 'Retrieve a specific pair transaction by its transaction hash', - }) - @ApiOkResponse({ type: PairTransactionDto }) - @Get(':txHash') - async getByTxHash(@Param('txHash') txHash: string) { - const pairTransaction = - await this.pairTransactionService.findByTxHash(txHash); - if (!pairTransaction) { - throw new NotFoundException( - `Pair transaction with hash ${txHash} not found`, - ); - } - return pairTransaction; - } - - @ApiParam({ - name: 'pairAddress', - type: 'string', - description: 'Pair contract address', - }) - @ApiQuery({ name: 'page', type: 'number', required: false }) - @ApiQuery({ name: 'limit', type: 'number', required: false }) - @ApiQuery({ - name: 'order_by', - enum: ['created_at', 'tx_type'], - required: false, - }) - @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) - @ApiOperation({ - operationId: 'getPairTransactionsByPairAddress', - summary: 'Get pair transactions by pair address', - description: 'Retrieve paginated transactions for a specific pair', - }) - @ApiOkResponsePaginated(PairTransactionDto) - @Get('pair/:pairAddress') - async getByPairAddress( - @Param('pairAddress') pairAddress: string, - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, - @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, - @Query('order_by') orderBy: string = 'created_at', - @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', - ) { - return this.pairTransactionService.findByPairAddress( - pairAddress, - { page, limit }, - orderBy, - orderDirection, - ); - } -} diff --git a/src/dex/controllers/pairs.controller.ts b/src/dex/controllers/pairs.controller.ts deleted file mode 100644 index c35a908..0000000 --- a/src/dex/controllers/pairs.controller.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { - Controller, - DefaultValuePipe, - Get, - NotFoundException, - Param, - ParseIntPipe, - Query, -} from '@nestjs/common'; -import { - ApiOperation, - ApiParam, - ApiQuery, - ApiTags, - ApiOkResponse, -} from '@nestjs/swagger'; -import { PairService } from '../services/pair.service'; -import { PairDto } from '../dto'; -import { ApiOkResponsePaginated } from '@/utils/api-type'; - -@Controller('dex/pairs') -@ApiTags('DEX') -export class PairsController { - constructor(private readonly pairService: PairService) {} - - @ApiQuery({ name: 'page', type: 'number', required: false }) - @ApiQuery({ name: 'limit', type: 'number', required: false }) - @ApiQuery({ - name: 'order_by', - enum: ['transactions_count', 'created_at'], - required: false, - }) - @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) - @ApiOperation({ - operationId: 'listAllPairs', - summary: 'Get all pairs', - description: - 'Retrieve a paginated list of all DEX pairs with optional sorting', - }) - @ApiOkResponsePaginated(PairDto) - @Get() - async listAll( - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, - @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, - @Query('order_by') orderBy: string = 'created_at', - @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', - ) { - return this.pairService.findAll({ page, limit }, orderBy, orderDirection); - } - - @ApiParam({ - name: 'address', - type: 'string', - description: 'Pair contract address', - }) - @ApiOperation({ - operationId: 'getPairByAddress', - summary: 'Get pair by address', - description: 'Retrieve a specific pair by its contract address', - }) - @ApiOkResponse({ type: PairDto }) - @Get(':address') - async getByAddress(@Param('address') address: string) { - const pair = await this.pairService.findByAddress(address); - if (!pair) { - throw new NotFoundException(`Pair with address ${address} not found`); - } - return pair; - } -} diff --git a/src/dex/dex.module.ts b/src/dex/dex.module.ts deleted file mode 100644 index 16e7fde..0000000 --- a/src/dex/dex.module.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { AeModule } from '@/ae/ae.module'; -import { TransactionsModule } from '@/transactions/transactions.module'; -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { DexTokensController } from './controllers/dex-tokens.controller'; -import { PairsController } from './controllers/pairs.controller'; -import { PairTransactionsController } from './controllers/pair-transactions.controller'; -import { DexToken } from './entities/dex-token.entity'; -import { Pair } from './entities/pair.entity'; -import { PairTransaction } from './entities/pair-transaction.entity'; -import { DexSyncService } from './services/dex-sync.service'; -import { DexTokenService } from './services/dex-token.service'; -import { PairService } from './services/pair.service'; -import { PairTransactionService } from './services/pair-transaction.service'; - -@Module({ - imports: [ - AeModule, - TransactionsModule, - TypeOrmModule.forFeature([Pair, DexToken, PairTransaction]), - ], - providers: [ - PairService, - DexTokenService, - PairTransactionService, - DexSyncService, - ], - exports: [PairService, DexTokenService, PairTransactionService], - controllers: [ - PairsController, - DexTokensController, - PairTransactionsController, - ], -}) -export class DexModule { - // -} diff --git a/src/dex/dto/dex-token.dto.ts b/src/dex/dto/dex-token.dto.ts deleted file mode 100644 index 1a9121b..0000000 --- a/src/dex/dto/dex-token.dto.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { ApiProperty } from '@nestjs/swagger'; - -export class DexTokenDto { - @ApiProperty({ - description: 'Token contract address', - example: 'ct_2AfnEfCSPx4A6VjMBfDfqHNYcqDJjuJjGV1qhqP5qNKNBvYfE2', - }) - address: string; - - @ApiProperty({ - description: 'Token name', - example: 'Wrapped Aeternity', - }) - name: string; - - @ApiProperty({ - description: 'Token symbol', - example: 'WAE', - }) - symbol: string; - - @ApiProperty({ - description: 'Token decimals', - example: 18, - }) - decimals: number; - - @ApiProperty({ - description: 'Number of pairs this token is part of', - example: 5, - }) - pairs_count: number; - - @ApiProperty({ - description: 'Token creation timestamp', - example: '2024-01-01T00:00:00.000Z', - }) - created_at: Date; -} diff --git a/src/dex/dto/index.ts b/src/dex/dto/index.ts deleted file mode 100644 index 5c9a969..0000000 --- a/src/dex/dto/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './pair.dto'; -export * from './dex-token.dto'; -export * from './pair-transaction.dto'; diff --git a/src/dex/dto/pair-transaction.dto.ts b/src/dex/dto/pair-transaction.dto.ts deleted file mode 100644 index ceab38b..0000000 --- a/src/dex/dto/pair-transaction.dto.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { PairDto } from './pair.dto'; - -export class PairTransactionDto { - @ApiProperty({ - description: 'Transaction hash', - example: 'th_2AfnEfCSPx4A6VjMBfDfqHNYcqDJjuJjGV1qhqP5qNKNBvYfE2', - }) - tx_hash: string; - - @ApiProperty({ - description: 'Associated pair', - type: () => PairDto, - }) - pair: PairDto; - - @ApiProperty({ - description: - 'Transaction type (e.g., swap, add_liquidity, remove_liquidity)', - example: 'swap_exact_tokens_for_tokens', - }) - tx_type: string; - - @ApiProperty({ - description: 'Transaction creation timestamp', - example: '2024-01-01T00:00:00.000Z', - }) - created_at: Date; -} diff --git a/src/dex/dto/pair.dto.ts b/src/dex/dto/pair.dto.ts deleted file mode 100644 index 21fc036..0000000 --- a/src/dex/dto/pair.dto.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { DexTokenDto } from './dex-token.dto'; - -export class PairDto { - @ApiProperty({ - description: 'Pair contract address', - example: 'ct_2AfnEfCSPx4A6VjMBfDfqHNYcqDJjuJjGV1qhqP5qNKNBvYfE2', - }) - address: string; - - @ApiProperty({ - description: 'First token in the pair', - type: () => DexTokenDto, - }) - token0: DexTokenDto; - - @ApiProperty({ - description: 'Second token in the pair', - type: () => DexTokenDto, - }) - token1: DexTokenDto; - - @ApiProperty({ - description: 'Number of transactions for this pair', - example: 150, - }) - transactions_count: number; - - @ApiProperty({ - description: 'Pair creation timestamp', - example: '2024-01-01T00:00:00.000Z', - }) - created_at: Date; -} diff --git a/src/dex/entities/dex-token.entity.ts b/src/dex/entities/dex-token.entity.ts deleted file mode 100644 index abe0f41..0000000 --- a/src/dex/entities/dex-token.entity.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Column, CreateDateColumn, Entity, PrimaryColumn } from 'typeorm'; - -@Entity({ - name: 'dex_tokens', -}) -export class DexToken { - @PrimaryColumn() - address: string; - - @Column() - name: string; - - @Column() - symbol: string; - - @Column({ - default: 18, - }) - decimals: number; - - @Column({ - default: 0, - }) - pairs_count: number; - - @CreateDateColumn({ - type: 'timestamp', - default: () => 'CURRENT_TIMESTAMP(6)', - }) - public created_at: Date; -} diff --git a/src/dex/entities/pair-transaction.entity.ts b/src/dex/entities/pair-transaction.entity.ts deleted file mode 100644 index dfc8430..0000000 --- a/src/dex/entities/pair-transaction.entity.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - JoinColumn, - ManyToOne, - PrimaryColumn, -} from 'typeorm'; -import { Pair } from './pair.entity'; - -@Entity({ - name: 'pair_transactions', -}) -export class PairTransaction { - @PrimaryColumn() - tx_hash: string; - - @ManyToOne(() => Pair, (pair) => pair.address) - @JoinColumn({ name: 'pair_address' }) - pair: Pair; - - @Column() - tx_type: string; - - @CreateDateColumn({ - type: 'timestamp', - default: () => 'CURRENT_TIMESTAMP(6)', - }) - public created_at: Date; -} diff --git a/src/dex/entities/pair.entity.ts b/src/dex/entities/pair.entity.ts deleted file mode 100644 index 124af26..0000000 --- a/src/dex/entities/pair.entity.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { DexToken } from './dex-token.entity'; -import { PairTransaction } from './pair-transaction.entity'; -import { - CreateDateColumn, - Entity, - PrimaryColumn, - ManyToOne, - JoinColumn, - OneToMany, -} from 'typeorm'; - -@Entity({ - name: 'pairs', -}) -export class Pair { - @PrimaryColumn() - address: string; - - @ManyToOne(() => DexToken, (dexToken) => dexToken.address) - @JoinColumn({ name: 'token0_address' }) - token0: DexToken; - - @ManyToOne(() => DexToken, (dexToken) => dexToken.address) - @JoinColumn({ name: 'token1_address' }) - token1: DexToken; - - @OneToMany(() => PairTransaction, (transaction) => transaction.pair) - transactions: PairTransaction[]; - - @CreateDateColumn({ - type: 'timestamp', - default: () => 'CURRENT_TIMESTAMP(6)', - }) - public created_at: Date; -} diff --git a/src/dex/services/dex-sync.service.ts b/src/dex/services/dex-sync.service.ts deleted file mode 100644 index 46e972c..0000000 --- a/src/dex/services/dex-sync.service.ts +++ /dev/null @@ -1,252 +0,0 @@ -import camelcaseKeysDeep from 'camelcase-keys-deep'; - -import { AeSdkService } from '@/ae/ae-sdk.service'; -import { ACTIVE_NETWORK, TX_FUNCTIONS } from '@/configs'; -import { IMiddlewareRequestConfig } from '@/social/interfaces/post.interfaces'; -import { fetchJson } from '@/utils/common'; -import { ITransaction } from '@/utils/types'; -import { Encoded } from '@aeternity/aepp-sdk'; -import ContractWithMethods, { - ContractMethodsBase, -} from '@aeternity/aepp-sdk/es/contract/Contract'; -import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import factoryInterface from 'dex-contracts-v2/build/AedexV2Factory.aci.json'; -import routerInterface from 'dex-contracts-v2/build/AedexV2Router.aci.json'; -import { Repository } from 'typeorm'; -import { DEX_CONTRACTS } from '../config/dex-contracts.config'; -import { DexToken } from '../entities/dex-token.entity'; -import { Pair } from '../entities/pair.entity'; -import { PairTransaction } from '../entities/pair-transaction.entity'; -import moment from 'moment'; - -@Injectable() -export class DexSyncService { - routerContract: ContractWithMethods; - factoryContract: ContractWithMethods; - constructor( - @InjectRepository(DexToken) - private readonly dexTokenRepository: Repository, - @InjectRepository(Pair) - private readonly dexPairRepository: Repository, - @InjectRepository(PairTransaction) - private readonly dexPairTransactionRepository: Repository, - - private aeSdkService: AeSdkService, - ) { - // - } - - async onModuleInit(): Promise { - console.log('========================'); - console.log('==== DexSyncService ===='); - console.log('========================'); - // - return; - - this.routerContract = await this.aeSdkService.sdk.initializeContract({ - aci: routerInterface, - address: DEX_CONTRACTS.router as Encoded.ContractAddress, - }); - this.factoryContract = await this.aeSdkService.sdk.initializeContract({ - aci: factoryInterface, - address: DEX_CONTRACTS.factory as Encoded.ContractAddress, - }); - this.syncDexTokens(); - } - - async syncDexTokens() { - const config: IMiddlewareRequestConfig = { - direction: 'forward', - limit: 100, - type: 'contract_call', - contract: DEX_CONTRACTS.router, - }; - const queryString = new URLSearchParams({ - direction: config.direction, - limit: config.limit.toString(), - type: config.type, - contract: config.contract, - }).toString(); - const url = `${ACTIVE_NETWORK.middlewareUrl}/v3/transactions?${queryString}`; - await this.pullDexPairsFromMdw(url); - } - - async pullDexPairsFromMdw(url: string) { - const result = await fetchJson(url); - const data = result?.data ?? []; - for (const item of camelcaseKeysDeep(data)) { - if ( - item.tx.result !== 'ok' || - item.tx.return == 'invalid' || - !item.tx.function - ) { - continue; - } - // console.log('--------------------------------'); - const pairInfo = await this.extractPairInfoFromTransaction(item); - if (!pairInfo) { - // console.log('pairInfo', pairInfo); - continue; - } - // console.log('pairInfo', pairInfo.pairAddress); - // console.log('--------------------------------'); - - const pair = await this.saveDexPair(pairInfo); - await this.saveDexPairTransaction(pair, item); - } - if (result.next) { - return await this.pullDexPairsFromMdw( - `${ACTIVE_NETWORK.middlewareUrl}${result.next}`, - ); - } - return result; - } - - async extractPairInfoFromTransaction(item: ITransaction) { - console.log('item.tx.function:', item.tx.function); - let decodedEvents = null; - try { - decodedEvents = this.routerContract.$decodeEvents(item.tx.log); - } catch (error: any) { - console.log('routerContract.$decodeEvents error', error?.message); - } - if (!decodedEvents) { - try { - decodedEvents = this.factoryContract.$decodeEvents(item.tx.log); - // console.log('factoryContract.$decodeEvents decodedEvents', decodedEvents); - } catch (error: any) { - console.log('factoryContract.$decodeEvents error', error?.message); - } - } - if (!decodedEvents) { - return null; - } - const pairAddress = decodedEvents.find( - (event) => event.contract?.name === 'IAedexV2Pair', - )?.contract?.address; - let token0Address = null; - let token1Address = null; - const args = item.tx.arguments; - - if ( - item.tx.function === TX_FUNCTIONS.swap_exact_tokens_for_tokens || - item.tx.function === TX_FUNCTIONS.swap_tokens_for_exact_tokens || - item.tx.function === TX_FUNCTIONS.swap_exact_tokens_for_ae || - item.tx.function === TX_FUNCTIONS.swap_tokens_for_exact_ae - ) { - token0Address = args[2].value[0]?.value; - token1Address = args[2].value[1]?.value; - } else if ( - item.tx.function === TX_FUNCTIONS.swap_exact_ae_for_tokens || - item.tx.function === TX_FUNCTIONS.swap_ae_for_exact_tokens - ) { - token0Address = args[1].value[0]?.value; - token1Address = args[1].value[1]?.value; - } else if (item.tx.function === TX_FUNCTIONS.add_liquidity) { - token0Address = args[0].value; - token1Address = args[1].value; - // console.log('args::', JSON.stringify(args, null, 2)); - } else if (item.tx.function === TX_FUNCTIONS.add_liquidity_ae) { - // this mean add new pair WAE -> Token - token0Address = DEX_CONTRACTS.wae; - token1Address = args[0].value; - } else { - // if (item.tx.function?.includes('liquidity')) { - // return null; - // } - // console.log('item.tx.function:', item.tx.function); - // console.log('args::', JSON.stringify(args, null, 2)); - } - if (!token0Address || !token1Address) { - return null; - } - - const token0 = await this.getOrCreateToken(token0Address); - const token1 = await this.getOrCreateToken(token1Address); - - return { pairAddress, token0, token1 }; - } - - private async getOrCreateToken(address: string) { - const token = await this.dexTokenRepository.findOne({ - where: { address }, - }); - if (token) { - return token; - } - const tokenData = await fetchJson( - `${ACTIVE_NETWORK.middlewareUrl}/v3/aex9/${address}`, - ); - return this.dexTokenRepository.save({ - address, - name: tokenData.name, - symbol: tokenData.symbol, - decimals: tokenData.decimals, - }); - } - - private async saveDexPair(pairInfo: { - pairAddress: string; - token0: DexToken; - token1: DexToken; - }) { - let pair = await this.dexPairRepository.findOne({ - where: { address: pairInfo.pairAddress }, - }); - if (pair) { - return pair; - } - - pair = await this.dexPairRepository.save({ - address: pairInfo.pairAddress, - token0: pairInfo.token0, - token1: pairInfo.token1, - }); - await this.updateTokenPairsCount(pairInfo.token0); - await this.updateTokenPairsCount(pairInfo.token1); - return pair; - } - - private async updateTokenPairsCount(token: DexToken) { - const pairsCount = await this.dexPairRepository - .createQueryBuilder('pair') - .where('pair.token0_address = :token0_address', { - token0_address: token.address, - }) - .orWhere('pair.token1_address = :token1_address', { - token1_address: token.address, - }) - .getCount(); - await this.dexTokenRepository.update(token.address, { - pairs_count: pairsCount, - }); - } - - private async saveDexPairTransaction(pair: Pair, item: ITransaction) { - const existingTransaction = await this.dexPairTransactionRepository - .createQueryBuilder('pairTransaction') - .where('pairTransaction.tx_hash = :tx_hash', { - tx_hash: item.hash, - }) - .getOne(); - if (existingTransaction) { - await this.dexPairTransactionRepository.update( - existingTransaction.tx_hash, - { - pair: pair, - tx_type: item.tx.function, - tx_hash: item.hash, - created_at: moment(item.microTime).toDate(), - }, - ); - return existingTransaction; - } - await this.dexPairTransactionRepository.save({ - pair: pair, - tx_type: item.tx.function, - tx_hash: item.hash, - created_at: moment(item.microTime).toDate(), - }); - } -} diff --git a/src/dex/services/dex-token.service.ts b/src/dex/services/dex-token.service.ts deleted file mode 100644 index de287a9..0000000 --- a/src/dex/services/dex-token.service.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; -import { DexToken } from '../entities/dex-token.entity'; -import { - IPaginationOptions, - paginate, - Pagination, -} from 'nestjs-typeorm-paginate'; - -@Injectable() -export class DexTokenService { - constructor( - @InjectRepository(DexToken) - private readonly dexTokenRepository: Repository, - ) {} - - async findAll( - options: IPaginationOptions, - orderBy: string = 'created_at', - orderDirection: 'ASC' | 'DESC' = 'DESC', - ): Promise> { - const query = this.dexTokenRepository.createQueryBuilder('dexToken'); - - if (orderBy) { - query.orderBy(`dexToken.${orderBy}`, orderDirection); - } - - return paginate(query, options); - } - - async findByAddress(address: string): Promise { - return this.dexTokenRepository - .createQueryBuilder('dexToken') - .where('dexToken.address = :address', { address }) - .getOne(); - } -} diff --git a/src/dex/services/pair-transaction.service.ts b/src/dex/services/pair-transaction.service.ts deleted file mode 100644 index acef19e..0000000 --- a/src/dex/services/pair-transaction.service.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; -import { PairTransaction } from '../entities/pair-transaction.entity'; -import { - IPaginationOptions, - paginate, - Pagination, -} from 'nestjs-typeorm-paginate'; - -@Injectable() -export class PairTransactionService { - constructor( - @InjectRepository(PairTransaction) - private readonly pairTransactionRepository: Repository, - ) {} - - async findAll( - options: IPaginationOptions, - orderBy: string = 'created_at', - orderDirection: 'ASC' | 'DESC' = 'DESC', - pairAddress?: string, - txType?: string, - ): Promise> { - const query = this.pairTransactionRepository - .createQueryBuilder('pairTransaction') - .leftJoinAndSelect('pairTransaction.pair', 'pair') - .leftJoinAndSelect('pair.token0', 'token0') - .leftJoinAndSelect('pair.token1', 'token1'); - - // Filter by pair address if provided - if (pairAddress) { - query.andWhere('pair.address = :pairAddress', { pairAddress }); - } - - // Filter by transaction type if provided - if (txType) { - query.andWhere('pairTransaction.tx_type = :txType', { txType }); - } - - // Add ordering - if (orderBy) { - query.orderBy(`pairTransaction.${orderBy}`, orderDirection); - } - - return paginate(query, options); - } - - async findByTxHash(txHash: string): Promise { - return this.pairTransactionRepository - .createQueryBuilder('pairTransaction') - .leftJoinAndSelect('pairTransaction.pair', 'pair') - .leftJoinAndSelect('pair.token0', 'token0') - .leftJoinAndSelect('pair.token1', 'token1') - .where('pairTransaction.tx_hash = :txHash', { txHash }) - .getOne(); - } - - async findByPairAddress( - pairAddress: string, - options: IPaginationOptions, - orderBy: string = 'created_at', - orderDirection: 'ASC' | 'DESC' = 'DESC', - ): Promise> { - return this.findAll(options, orderBy, orderDirection, pairAddress); - } -} diff --git a/src/dex/services/pair.service.ts b/src/dex/services/pair.service.ts deleted file mode 100644 index 28863f1..0000000 --- a/src/dex/services/pair.service.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; -import { Pair } from '../entities/pair.entity'; -import { - IPaginationOptions, - paginate, - Pagination, -} from 'nestjs-typeorm-paginate'; - -@Injectable() -export class PairService { - constructor( - @InjectRepository(Pair) - private readonly pairRepository: Repository, - ) {} - - async findAll( - options: IPaginationOptions, - orderBy: string = 'created_at', - orderDirection: 'ASC' | 'DESC' = 'DESC', - ): Promise> { - const query = this.pairRepository - .createQueryBuilder('pair') - .leftJoinAndSelect('pair.token0', 'token0') - .leftJoinAndSelect('pair.token1', 'token1') - .loadRelationCountAndMap('pair.transactions_count', 'pair.transactions'); - - if (orderBy) { - if (orderBy === 'transactions_count') { - query.orderBy('pair.transactions_count', orderDirection); - } else { - query.orderBy(`pair.${orderBy}`, orderDirection); - } - } - - return paginate(query, options); - } - - async findByAddress(address: string): Promise { - return this.pairRepository - .createQueryBuilder('pair') - .leftJoinAndSelect('pair.token0', 'token0') - .leftJoinAndSelect('pair.token1', 'token1') - .loadRelationCountAndMap('pair.transactions_count', 'pair.transactions') - .where('pair.address = :address', { address }) - .getOne(); - } -} diff --git a/src/social/README.md b/src/social/README.md deleted file mode 100644 index e4e102f..0000000 --- a/src/social/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Social Posts Module - -This module handles social media posts on the Aeternity blockchain, including post creation, content parsing, and data retrieval. - -## Overview - -The Social Posts module processes blockchain transactions to extract and store social media posts. It supports multiple contract versions and provides comprehensive content parsing, media extraction, and topic analysis. - -## Architecture - -### Core Components - -- **PostService**: Main service handling post processing and storage -- **Post Entity**: Database entity representing social posts -- **PostsController**: REST API endpoints for post retrieval -- **Content Parser**: Utility for parsing post content and extracting metadata -- **Contract Configuration**: Centralized contract management - -### Key Features - -- ✅ Multi-contract support with versioning -- ✅ Real-time transaction processing -- ✅ Content sanitization and validation -- ✅ Topic extraction (hashtags) -- ✅ Media URL validation and extraction -- ✅ Comprehensive error handling and logging -- ✅ Database transaction safety -- ✅ Retry mechanisms for API failures -- ✅ Concurrent processing with locks - -## Configuration - -### Contract Configuration - -Contracts are configured in `src/social/config/post-contracts.config.ts`: - -```typescript -export const POST_CONTRACTS: IPostContract[] = [ - { - contractAddress: 'ct_2Hyt9ZxzXra5NAzhePkRsDPDWppoatVD7CtHnUoHVbuehwR8Nb', - version: 3, - description: 'Current social posting contract' - }, -]; -``` - -### Content Parsing Options - -Content parsing can be customized via `IContentParsingOptions`: - -- `maxTopics`: Maximum number of hashtags to extract (default: 10) -- `maxMediaItems`: Maximum number of media URLs to extract (default: 5) -- `sanitizeContent`: Whether to sanitize content (default: true) - -## API Endpoints - -### GET /posts - -Retrieve paginated list of posts with optional sorting. - -**Query Parameters:** -- `page`: Page number (default: 1) -- `limit`: Items per page (default: 100) -- `order_by`: Sort field ('total_comments' | 'created_at') -- `order_direction`: Sort direction ('ASC' | 'DESC') - -### GET /posts/:id - -Retrieve a specific post by ID. - -## Data Processing Flow - -1. **Transaction Reception**: Live transactions or batch processing from middleware -2. **Contract Validation**: Check if transaction is from supported contract -3. **Content Extraction**: Extract content and metadata from transaction arguments -4. **Content Parsing**: Parse content for topics and media URLs -5. **Validation**: Validate content structure and data integrity -6. **Storage**: Save to database with transaction safety -7. **Error Handling**: Log errors and handle failures gracefully - -## Content Processing - -### Topic Extraction - -- Extracts hashtags starting with '#' -- Converts to lowercase -- Filters invalid characters -- Removes duplicates -- Limits to reasonable length (1-50 characters) - -### Media Validation - -- Validates URL format and protocol (http/https) -- Checks for common media file extensions -- Supports major media hosting platforms -- Limits number of media items per post - -### Content Sanitization - -- Trims whitespace -- Normalizes line endings -- Limits consecutive line breaks -- Enforces maximum content length (5000 chars) - -## Error Handling - -The module implements comprehensive error handling: - -- **Validation Errors**: Invalid transaction data or content -- **Network Errors**: Middleware API failures with retry logic -- **Database Errors**: Transaction rollback and error logging -- **Processing Errors**: Individual transaction failures don't stop batch processing - -## Monitoring and Logging - -Structured logging provides visibility into: - -- Transaction processing status -- Error details with stack traces -- Performance metrics -- Contract processing statistics -- Retry attempts and failures - -## Development - -### Running Tests - -```bash -npm test src/social -``` - -### Adding New Contracts - -1. Add contract configuration to `post-contracts.config.ts` -2. Update contract version handling if needed -3. Test with sample transactions - -### Extending Content Parsing - -1. Modify `content-parser.util.ts` -2. Add new parsing options to interfaces -3. Update tests and documentation - -## Performance Considerations - -- **Concurrent Processing**: Uses processing locks to prevent duplicate work -- **Batch Processing**: Processes multiple transactions efficiently -- **Database Transactions**: Ensures data consistency -- **Error Isolation**: Individual failures don't affect batch processing -- **Retry Logic**: Handles temporary network failures gracefully - -## Security - -- **Input Validation**: All transaction data is validated -- **Content Sanitization**: User content is sanitized before storage -- **URL Validation**: Media URLs are validated for security -- **Error Information**: Sensitive data is not logged in errors diff --git a/src/social/config/post-contracts.config.ts b/src/social/config/post-contracts.config.ts deleted file mode 100644 index 3c5ec10..0000000 --- a/src/social/config/post-contracts.config.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { IPostContract } from '../interfaces/post.interfaces'; - -/** - * Configuration for supported post contracts - * Each contract represents a different version or type of social posting functionality - */ -export const POST_CONTRACTS: IPostContract[] = [ - // Commented out older contract - kept for reference - // { - // contractAddress: 'ct_2AfnEfCSZCTEkxL5Yoi4Yfq6fF7YapHRaFKDJK3THMXMBspp5z', - // version: 1, - // description: 'Legacy tip/retip contract' - // }, - { - contractAddress: 'ct_2Hyt9ZxzXra5NAzhePkRsDPDWppoatVD7CtHnUoHVbuehwR8Nb', - version: 3, - description: 'Current social posting contract', - }, -]; - -/** - * Get contract configuration by address - */ -export function getContractByAddress( - address: string, -): IPostContract | undefined { - return POST_CONTRACTS.find( - (contract) => contract.contractAddress === address, - ); -} - -/** - * Get all active contract addresses - */ -export function getActiveContractAddresses(): string[] { - return POST_CONTRACTS.map((contract) => contract.contractAddress); -} - -/** - * Check if a contract address is supported - */ -export function isContractSupported(address: string): boolean { - return POST_CONTRACTS.some( - (contract) => contract.contractAddress === address, - ); -} diff --git a/src/social/controllers/posts.controller.ts b/src/social/controllers/posts.controller.ts deleted file mode 100644 index ebc81d2..0000000 --- a/src/social/controllers/posts.controller.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { - Controller, - DefaultValuePipe, - Get, - NotFoundException, - Param, - ParseIntPipe, - Query, -} from '@nestjs/common'; -import { - ApiOperation, - ApiParam, - ApiQuery, - ApiTags, - ApiOkResponse, -} from '@nestjs/swagger'; -import { InjectRepository } from '@nestjs/typeorm'; -import { paginate } from 'nestjs-typeorm-paginate'; -import { Repository } from 'typeorm'; -import { Post } from '../entities/post.entity'; -import { PostDto } from '../dto'; -import { ApiOkResponsePaginated } from '@/utils/api-type'; - -@Controller('posts') -@ApiTags('Posts') -export class PostsController { - constructor( - @InjectRepository(Post) - private readonly postRepository: Repository, - ) { - // - } - - @ApiQuery({ name: 'page', type: 'number', required: false }) - @ApiQuery({ name: 'limit', type: 'number', required: false }) - @ApiQuery({ - name: 'order_by', - enum: ['total_comments', 'created_at'], - required: false, - }) - @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) - @ApiQuery({ - name: 'search', - type: 'string', - required: false, - description: 'Search term to filter posts by content or topics', - }) - @ApiOperation({ - operationId: 'listAll', - summary: 'Get all posts', - description: - 'Retrieve a paginated list of all posts with optional sorting and search functionality', - }) - @ApiOkResponsePaginated(PostDto) - @Get() - async listAll( - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, - @Query('limit', new DefaultValuePipe(100), ParseIntPipe) limit = 100, - @Query('order_by') orderBy: string = 'created_at', - @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'DESC', - @Query('search') search?: string, - ) { - const query = this.postRepository - .createQueryBuilder('post') - .where('post.post_id IS NULL'); - - // Add search functionality - if (search) { - const searchTerm = `%${search}%`; - query.where( - '(post.content ILIKE :searchTerm OR CAST(post.topics AS TEXT) ILIKE :searchTerm)', - { searchTerm }, - ); - } - - // Add ordering - if (orderBy) { - query.orderBy(`post.${orderBy}`, orderDirection); - } - - return paginate(query, { page, limit }); - } - - @ApiParam({ name: 'id', type: 'string', description: 'Post ID' }) - @ApiOperation({ - operationId: 'getById', - summary: 'Get post by ID', - description: 'Retrieve a specific post by its unique identifier', - }) - @ApiOkResponse({ - type: PostDto, - description: 'Post retrieved successfully', - }) - @Get(':id') - async getById(@Param('id') id: string) { - const post = await this.postRepository.findOne({ - where: { id }, - }); - if (!post) { - throw new NotFoundException(`Post with ID ${id} not found`); - } - return post; - } - - @ApiParam({ name: 'id', type: 'string', description: 'Post ID' }) - @ApiQuery({ name: 'page', type: 'number', required: false }) - @ApiQuery({ name: 'limit', type: 'number', required: false }) - @ApiQuery({ name: 'order_direction', enum: ['ASC', 'DESC'], required: false }) - @ApiOperation({ - operationId: 'getComments', - summary: 'Get comments for a post', - description: 'Retrieve paginated comments for a specific post', - }) - @ApiOkResponsePaginated(PostDto) - @Get(':id/comments') - async getComments( - @Param('id') id: string, - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page = 1, - @Query('limit', new DefaultValuePipe(50), ParseIntPipe) limit = 50, - @Query('order_direction') orderDirection: 'ASC' | 'DESC' = 'ASC', - ) { - // First check if the parent post exists - const parentPost = await this.postRepository.findOne({ - where: { id }, - }); - if (!parentPost) { - throw new NotFoundException(`Post with ID ${id} not found`); - } - - const query = this.postRepository - .createQueryBuilder('post') - .where('post.post_id = :parentPostId', { parentPostId: id }) - .orderBy('post.created_at', orderDirection); - - return paginate(query, { page, limit }); - } -} diff --git a/src/social/dto/index.ts b/src/social/dto/index.ts deleted file mode 100644 index 4171984..0000000 --- a/src/social/dto/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { PostDto } from './post.dto'; diff --git a/src/social/dto/post.dto.ts b/src/social/dto/post.dto.ts deleted file mode 100644 index 983ab08..0000000 --- a/src/social/dto/post.dto.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { ApiProperty } from '@nestjs/swagger'; - -export class PostDto { - @ApiProperty({ - description: 'Unique identifier for the post', - example: '12345_v1', - }) - id: string; - - @ApiProperty({ - description: 'Transaction hash associated with the post', - example: 'th_1234567890abcdef...', - }) - tx_hash: string; - - @ApiProperty({ - description: 'Transaction arguments as JSON array', - type: 'array', - items: { type: 'object' }, - example: [{ type: 'string', value: 'Hello world!' }], - }) - tx_args: any[]; - - @ApiProperty({ - description: 'Address of the post sender/creator', - example: 'ak_2a1j2Mk9YSmC1gioUq4PWRm3bsv887MbuRVwyv4KaUGoR1eiKi', - }) - sender_address: string; - - @ApiProperty({ - description: 'Address of the smart contract', - example: 'ct_2AfnEfCSPx4A6UYXj2XHDqHXcC7EF2bgbp8UN1KPAJDysPJT32', - }) - contract_address: string; - - @ApiProperty({ - description: 'Type of the post/transaction', - example: 'post', - }) - type: string; - - @ApiProperty({ - description: 'Main content of the post', - example: 'Hello world! This is my first post on the blockchain.', - }) - content: string; - - @ApiProperty({ - description: 'Array of topics/hashtags associated with the post', - type: [String], - example: ['#blockchain', '#hello', '#firstpost'], - }) - topics: string[]; - - @ApiProperty({ - description: 'Array of media URLs associated with the post', - type: [String], - example: ['https://example.com/image.jpg', 'https://example.com/video.mp4'], - }) - media: string[]; - - @ApiProperty({ - description: 'Total number of comments on this post', - example: 5, - }) - total_comments: number; - - @ApiProperty({ - description: 'Timestamp when the post was created', - type: 'string', - format: 'date-time', - example: '2023-12-01T10:30:00.000Z', - }) - created_at: Date; -} diff --git a/src/social/entities/post.entity.ts b/src/social/entities/post.entity.ts deleted file mode 100644 index bac5bcc..0000000 --- a/src/social/entities/post.entity.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - JoinColumn, - ManyToOne, - PrimaryColumn, -} from 'typeorm'; - -@Entity({ - name: 'posts', -}) -export class Post { - @PrimaryColumn() - id: string; - - @Column({ nullable: true }) - post_id: string; - - @ManyToOne(() => Post, (post) => post.id, { - nullable: true, - onDelete: 'CASCADE', - }) - @JoinColumn({ name: 'post_id' }) - parent_post: Post; - - @Column({ - unique: true, - }) - tx_hash: string; - - @Column('json') - tx_args: any[]; - - @Column() - sender_address: string; - - @Column() - contract_address: string; - - @Column() - type: string; - - @Column() - content: string; - - @Column('json', { default: [] }) - topics: string[]; - - @Column('json', { default: [] }) - media: string[]; - - @Column() - total_comments: number; - - @CreateDateColumn({ - type: 'timestamp', - default: () => 'CURRENT_TIMESTAMP(6)', - }) - public created_at: Date; -} diff --git a/src/social/interfaces/post.interfaces.ts b/src/social/interfaces/post.interfaces.ts deleted file mode 100644 index dfbdc34..0000000 --- a/src/social/interfaces/post.interfaces.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { ITransaction } from '@/utils/types'; - -/** - * Configuration for a post contract - */ -export interface IPostContract { - contractAddress: string; - version: number; - description?: string; -} - -/** - * Parsed post content with extracted metadata - */ -export interface IParsedPostContent { - content: string; - topics: string[]; - media: string[]; -} - -/** - * Data structure for creating a new post - */ -export interface ICreatePostData { - id: string; - type: string; - tx_hash: string; - sender_address: string; - contract_address: string; - content: string; - topics: string[]; - media: string[]; - total_comments: number; - tx_args: any[]; - created_at: Date; - post_id?: string; -} - -/** - * Result of post processing operation - */ -export interface IPostProcessingResult { - success: boolean; - post?: any; - error?: string; - skipped?: boolean; - reason?: string; -} - -/** - * Configuration for middleware API requests - */ -export interface IMiddlewareRequestConfig { - direction: 'forward' | 'backward'; - limit: number; - type: string; - contract: string; -} - -/** - * Response structure from middleware API - */ -export interface IMiddlewareResponse { - data: ITransaction[]; - next?: string; - prev?: string; -} - -/** - * Options for content parsing - */ -export interface IContentParsingOptions { - maxTopics?: number; - maxMediaItems?: number; - sanitizeContent?: boolean; -} - -/** - * Comment detection result - */ -export interface ICommentInfo { - isComment: boolean; - parentPostId?: string; - commentArgument?: any; -} - -/** - * Comment processing result - */ -export interface ICommentProcessingResult { - success: boolean; - parentPostExists?: boolean; - error?: string; -} diff --git a/src/social/post.module.ts b/src/social/post.module.ts deleted file mode 100644 index 4429bb7..0000000 --- a/src/social/post.module.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { AeModule } from '@/ae/ae.module'; -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { Post } from './entities/post.entity'; -import { PostService } from './services/post.service'; -import { TransactionsModule } from '@/transactions/transactions.module'; -import { PostsController } from './controllers/posts.controller'; - -@Module({ - imports: [AeModule, TransactionsModule, TypeOrmModule.forFeature([Post])], - providers: [PostService], - exports: [PostService], - controllers: [PostsController], -}) -export class PostModule { - // -} diff --git a/src/social/services/post.service.spec.ts b/src/social/services/post.service.spec.ts deleted file mode 100644 index 7f3eff9..0000000 --- a/src/social/services/post.service.spec.ts +++ /dev/null @@ -1,332 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { getRepositoryToken } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; -import { PostService } from './post.service'; -import { Post } from '../entities/post.entity'; -import { ITransaction } from '@/utils/types'; -import { Logger } from '@nestjs/common'; - -// Mock the external dependencies -jest.mock('@/utils/common'); -jest.mock('../config/post-contracts.config'); -jest.mock('../utils/content-parser.util'); - -describe('PostService', () => { - let service: PostService; - let repository: jest.Mocked>; - let logger: jest.Mocked; - - const mockRepository = { - findOne: jest.fn(), - create: jest.fn(), - save: jest.fn(), - manager: { - transaction: jest.fn(), - }, - }; - - const createMockTransaction = ( - overrides: Partial = {}, - ): ITransaction => ({ - blockHeight: 123456, - claim: null, - hash: 'th_testHash123', - microIndex: 1, - microTime: Date.now(), - pending: false, - tx: { - abiVersion: 1, - amount: 0, - microTime: Date.now(), - arguments: [], - callerId: 'ak_testCaller123', - code: '', - commitmentId: null, - contractId: 'ct_testContract123', - fee: 1000, - gas: 5000, - gasPrice: 1000000000, - gasUsed: 3000, - name: null, - nameFee: 0, - nameId: null, - nameSalt: '', - nonce: 1, - pointers: null, - result: 'ok', - return: { type: 'tuple', value: 'test-return-value' }, - returnType: 'ok', - type: 'ContractCallTx' as const, - VSN: '1', - }, - ...overrides, - }); - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PostService, - { - provide: getRepositoryToken(Post), - useValue: mockRepository, - }, - ], - }).compile(); - - service = module.get(PostService); - repository = module.get(getRepositoryToken(Post)); - logger = service['logger'] as jest.Mocked; - - // Mock logger methods - logger.log = jest.fn(); - logger.error = jest.fn(); - logger.warn = jest.fn(); - logger.debug = jest.fn(); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('validateTransaction', () => { - it('should return false for null transaction', () => { - const result = service['validateTransaction'](null as any); - expect(result).toBe(false); - }); - - it('should return false for transaction without required fields', () => { - const transaction = {} as ITransaction; - const result = service['validateTransaction'](transaction); - expect(result).toBe(false); - }); - - it('should return true for valid transaction', () => { - const transaction = createMockTransaction({ - tx: { - ...createMockTransaction().tx, - callerId: 'ak_testCaller', - contractId: 'ct_testContract', - arguments: [], - }, - }); - - const result = service['validateTransaction'](transaction); - expect(result).toBe(true); - }); - }); - - describe('generatePostId', () => { - it('should generate ID with return value when available', () => { - const transaction = createMockTransaction({ - tx: { - ...createMockTransaction().tx, - return: { - type: 'tuple', - value: 'return-value', - }, - }, - }); - - const contract = { version: 3, contractAddress: 'test' }; - const result = service['generatePostId'](transaction, contract); - - expect(result).toBe('return-value_v3'); - }); - - it('should generate fallback ID when return value is not available', () => { - const transaction = createMockTransaction({ - hash: 'th_testHash12345678', - tx: { - ...createMockTransaction().tx, - return: undefined as any, - }, - }); - - const contract = { version: 3, contractAddress: 'test' }; - const result = service['generatePostId'](transaction, contract); - - expect(result).toBe('12345678_v3'); - }); - }); - - describe('handleLiveTransaction', () => { - it('should return error for transaction without contract ID', async () => { - const transaction = createMockTransaction({ - tx: { - ...createMockTransaction().tx, - contractId: undefined as any, - }, - }); - - const result = await service.handleLiveTransaction(transaction); - - expect(result.success).toBe(false); - expect(result.error).toBe('Missing contract ID or unsupported contract'); - expect(result.skipped).toBe(true); - }); - - it('should return error for unsupported contract', async () => { - // Mock the contract support check - const { - isContractSupported, - } = require('../config/post-contracts.config'); - isContractSupported.mockReturnValue(false); - - const transaction = createMockTransaction({ - tx: { - ...createMockTransaction().tx, - contractId: 'ct_unsupportedContract', - }, - }); - - const result = await service.handleLiveTransaction(transaction); - - expect(result.success).toBe(false); - expect(result.error).toBe('Missing contract ID or unsupported contract'); - expect(result.skipped).toBe(true); - }); - - it('should process supported contract successfully', async () => { - // Mock the contract support and configuration - const { - isContractSupported, - getContractByAddress, - } = require('../config/post-contracts.config'); - isContractSupported.mockReturnValue(true); - getContractByAddress.mockReturnValue({ - contractAddress: 'ct_testContract', - version: 3, - }); - - const mockPost = { id: 'test-post-id' }; - jest - .spyOn(service, 'savePostFromTransaction') - .mockResolvedValue(mockPost as Post); - - const transaction = createMockTransaction({ - tx: { - ...createMockTransaction().tx, - contractId: 'ct_testContract', - }, - }); - - const result = await service.handleLiveTransaction(transaction); - - expect(result.success).toBe(true); - expect(result.post).toBe(mockPost); - }); - }); - - describe('savePostFromTransaction', () => { - it('should return null for invalid transaction', async () => { - jest.spyOn(service as any, 'validateTransaction').mockReturnValue(false); - - const result = await service.savePostFromTransaction({} as ITransaction, { - contractAddress: 'test', - version: 3, - }); - - expect(result).toBeNull(); - }); - - it('should return existing post if already exists', async () => { - jest.spyOn(service as any, 'validateTransaction').mockReturnValue(true); - - const existingPost = { id: 'existing-post', tx_hash: 'th_testHash' }; - repository.findOne.mockResolvedValue(existingPost as Post); - - const transaction = createMockTransaction({ - hash: 'th_testHash', - tx: { - ...createMockTransaction().tx, - arguments: [{ type: 'tuple', value: 'test content' }], - }, - }); - - const result = await service.savePostFromTransaction(transaction, { - contractAddress: 'test', - version: 3, - }); - - expect(result).toBe(existingPost); - }); - - it('should create new post for valid transaction', async () => { - jest.spyOn(service as any, 'validateTransaction').mockReturnValue(true); - jest - .spyOn(service as any, 'generatePostId') - .mockReturnValue('new-post-id'); - - // Mock content parser - const { parsePostContent } = require('../utils/content-parser.util'); - parsePostContent.mockReturnValue({ - content: 'parsed content', - topics: ['#test'], - media: [], - }); - - repository.findOne.mockResolvedValue(null); - - const newPost = { id: 'new-post-id', tx_hash: 'th_testHash' }; - const mockManager = { - create: jest.fn().mockReturnValue(newPost), - save: jest.fn().mockResolvedValue(newPost), - }; - (repository.manager.transaction as jest.Mock).mockImplementation( - (callback) => callback(mockManager), - ); - - const transaction = createMockTransaction({ - hash: 'th_testHash', - tx: { - ...createMockTransaction().tx, - callerId: 'ak_testCaller', - contractId: 'ct_testContract', - function: 'create_community', - arguments: [ - { type: 'tuple', value: 'test content' }, - { type: 'list', value: [] }, - ], - }, - }); - - const result = await service.savePostFromTransaction(transaction, { - contractAddress: 'ct_testContract', - version: 3, - }); - - expect(result).toBe(newPost); - expect(parsePostContent).toHaveBeenCalledWith('test content', []); - }); - }); -}); - -/** - * Additional test cases to implement: - * - * 1. loadPostsFromMdw tests: - * - Successful data loading - * - Retry mechanism on failures - * - Pagination handling - * - Empty response handling - * - * 2. pullLatestPosts tests: - * - Processing lock behavior - * - Error handling and recovery - * - URL construction - * - * 3. pullLatestPostsForContracts tests: - * - Multiple contract processing - * - Parallel execution - * - Error aggregation - * - * 4. Integration tests: - * - End-to-end transaction processing - * - Database interaction testing - * - Error scenarios - * - * 5. Performance tests: - * - Large batch processing - * - Memory usage - * - Concurrent request handling - */ diff --git a/src/social/services/post.service.ts b/src/social/services/post.service.ts deleted file mode 100644 index aa160df..0000000 --- a/src/social/services/post.service.ts +++ /dev/null @@ -1,724 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; -import { Post } from '../entities/post.entity'; -import { - ACTIVE_NETWORK, - MAX_RETRIES_WHEN_REQUEST_FAILED, - PULL_SOCIAL_POSTS_ENABLED, - WAIT_TIME_WHEN_REQUEST_FAILED, -} from '@/configs'; -import { fetchJson } from '@/utils/common'; -import moment from 'moment'; -import { ITransaction } from '@/utils/types'; -import camelcaseKeysDeep from 'camelcase-keys-deep'; -import { - POST_CONTRACTS, - getContractByAddress, - isContractSupported, -} from '../config/post-contracts.config'; -import { - IPostContract, - ICreatePostData, - IPostProcessingResult, - IMiddlewareResponse, - IMiddlewareRequestConfig, - ICommentInfo, - ICommentProcessingResult, -} from '../interfaces/post.interfaces'; -import { parsePostContent } from '../utils/content-parser.util'; - -@Injectable() -export class PostService { - private readonly logger = new Logger(PostService.name); - private readonly isProcessing = new Map(); - - constructor( - @InjectRepository(Post) - private readonly postRepository: Repository, - ) { - this.logger.log('PostService initialized'); - } - - async onModuleInit(): Promise { - if (PULL_SOCIAL_POSTS_ENABLED) { - try { - await this.pullLatestPostsForContracts(); - // Run cleanup for any orphaned comments from previous runs - await this.fixOrphanedComments(); - } catch (error) { - this.logger.error('Failed to initialize PostService module', error); - // Don't throw - allow the service to start even if initial sync fails - } - this.logger.log('PostService module initialized successfully'); - } - } - - async handleLiveTransaction( - transaction: ITransaction, - ): Promise { - const contractAddress = transaction?.tx?.contractId; - - if (!contractAddress || !isContractSupported(contractAddress)) { - return { - success: false, - error: 'Missing contract ID or unsupported contract', - skipped: true, - }; - } - - const contract = getContractByAddress(contractAddress); - if (!contract) { - this.logger.error('Contract configuration not found', { - contractAddress, - }); - return { success: false, error: 'Contract configuration missing' }; - } - - try { - const post = await this.savePostFromTransaction(transaction, contract); - this.logger.log('Live transaction processed successfully', { - hash: transaction.hash, - contractAddress, - postId: post?.id, - }); - return { success: true, post }; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - - this.logger.error('Failed to process live transaction', { - hash: transaction.hash, - contractAddress, - error: errorMessage, - stack: errorStack, - }); - return { success: false, error: errorMessage }; - } - } - - async pullLatestPostsForContracts(): Promise { - const contractsProcessingKey = 'all_contracts'; - - if (this.isProcessing.get(contractsProcessingKey)) { - this.logger.warn('Contract processing already in progress, skipping...'); - return; - } - - this.isProcessing.set(contractsProcessingKey, true); - - try { - this.logger.log( - `Starting to pull posts for ${POST_CONTRACTS.length} contracts`, - ); - - const results = await Promise.allSettled( - POST_CONTRACTS.map((contract) => this.pullLatestPosts(contract)), - ); - - const successful = results.filter((r) => r.status === 'fulfilled').length; - const failed = results.filter((r) => r.status === 'rejected').length; - - this.logger.log( - `Contract processing completed: ${successful} successful, ${failed} failed`, - ); - - // Log any failures - results.forEach((result, index) => { - if (result.status === 'rejected') { - const error = - result.reason instanceof Error - ? result.reason - : new Error(String(result.reason)); - this.logger.error( - `Failed to process contract ${POST_CONTRACTS[index].contractAddress}`, - { - error: error.message, - stack: error.stack, - }, - ); - } - }); - } finally { - this.isProcessing.delete(contractsProcessingKey); - } - } - - async pullLatestPosts(contract: IPostContract): Promise { - const processingKey = `contract_${contract.contractAddress}`; - - if (this.isProcessing.get(processingKey)) { - this.logger.warn( - `Contract ${contract.contractAddress} already being processed, skipping...`, - ); - return []; - } - - this.isProcessing.set(processingKey, true); - - try { - const config: IMiddlewareRequestConfig = { - direction: 'forward', - limit: 100, - type: 'contract_call', - contract: contract.contractAddress, - }; - - const queryString = new URLSearchParams({ - direction: config.direction, - limit: config.limit.toString(), - type: config.type, - contract: config.contract, - }).toString(); - - const url = `${ACTIVE_NETWORK.middlewareUrl}/v3/transactions?${queryString}`; - - const posts = await this.loadPostsFromMdw(url, contract); - - this.logger.log( - `Successfully pulled ${posts.length} posts for contract ${contract.contractAddress}`, - ); - - return posts; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - - this.logger.error( - `Failed to pull posts for contract ${contract.contractAddress}`, - { - error: errorMessage, - stack: errorStack, - }, - ); - throw error; - } finally { - this.isProcessing.delete(processingKey); - } - } - - async loadPostsFromMdw( - url: string, - contract: IPostContract, - posts: any[] = [], - totalRetries = 0, - ): Promise { - let result: IMiddlewareResponse; - - try { - result = await fetchJson(url); - } catch (error) { - if (totalRetries < MAX_RETRIES_WHEN_REQUEST_FAILED) { - const nextRetry = totalRetries + 1; - const errorMessage = - error instanceof Error ? error.message : String(error); - - this.logger.warn( - `Middleware request failed, retrying (${nextRetry}/${MAX_RETRIES_WHEN_REQUEST_FAILED})`, - { - url, - error: errorMessage, - retryIn: WAIT_TIME_WHEN_REQUEST_FAILED, - }, - ); - - await new Promise((resolve) => - setTimeout(resolve, WAIT_TIME_WHEN_REQUEST_FAILED), - ); - return this.loadPostsFromMdw(url, contract, posts, nextRetry); - } - - const errorMessage = - error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - - this.logger.error( - 'Failed to load posts from middleware after all retries', - { - url, - error: errorMessage, - stack: errorStack, - totalRetries, - }, - ); - return posts; - } - - if (!result?.data?.length) { - this.logger.debug('No data received from middleware', { url }); - return posts; - } - - // Process transactions sequentially to handle parent-child dependencies - // This ensures parent posts are created before their comments - const successfulPosts: any[] = []; - let failedCount = 0; - - for (const transaction of result.data) { - try { - const camelCasedTransaction = camelcaseKeysDeep( - transaction, - ) as ITransaction; - const post = await this.savePostFromTransaction( - camelCasedTransaction, - contract, - ); - if (post) { - successfulPosts.push(post); - } - } catch (error) { - failedCount++; - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.warn('Failed to process individual transaction', { - txHash: transaction?.hash, - error: errorMessage, - }); - } - } - - posts.push(...successfulPosts); - - if (failedCount > 0) { - this.logger.warn( - `${failedCount} transactions failed to process in this batch`, - ); - } - - // Continue with pagination if available - if (result.next) { - const nextUrl = `${ACTIVE_NETWORK.middlewareUrl}${result.next}`; - return this.loadPostsFromMdw(nextUrl, contract, posts, 0); - } - - return posts; - } - - async savePostFromTransaction( - transaction: ITransaction, - contract: IPostContract, - ): Promise { - if (!this.validateTransaction(transaction)) { - this.logger.warn('Invalid transaction data', { - hash: transaction?.hash, - }); - return null; - } - - const txHash = transaction.hash; - const commentInfo = this.detectComment(transaction); - - try { - // Check if post already exists - const existingPost = await this.postRepository.findOne({ - where: { tx_hash: txHash }, - }); - - // Handle existing post that needs to be converted to comment - if (existingPost && commentInfo.isComment && !existingPost.post_id) { - const result = await this.processExistingPostAsComment( - existingPost, - commentInfo, - txHash, - ); - - if (result.success) { - return existingPost; - } else { - this.logger.warn('Failed to process existing post as comment', { - txHash, - error: result.error, - parentPostExists: result.parentPostExists, - }); - // Continue with regular flow if comment processing fails - } - } - - if (existingPost) { - return existingPost; - } - - // Validate required transaction data - if (!transaction.tx?.arguments?.[0]?.value) { - this.logger.warn('Transaction missing content argument', { txHash }); - return null; - } - - const content = transaction.tx.arguments[0].value; - if (typeof content !== 'string' || content.trim().length === 0) { - this.logger.warn('Invalid or empty content', { txHash }); - return null; - } - - // For new comments, validate parent post exists with retry logic - if (commentInfo.isComment && commentInfo.parentPostId) { - const parentPostExists = await this.validateParentPost( - commentInfo.parentPostId, - ); - if (!parentPostExists) { - this.logger.warn( - 'Cannot create comment: parent post not found after retries', - { - txHash, - parentPostId: commentInfo.parentPostId, - }, - ); - // Still create the comment but mark it as orphaned for later processing - // This prevents data loss in case of timing issues - } - } - - // Parse content and extract metadata - const parsedContent = parsePostContent( - content, - transaction.tx.arguments[1]?.value || [], - ); - - // Create post data with proper validation - const postData: ICreatePostData = { - id: this.generatePostId(transaction, contract), - type: transaction.tx.function || 'unknown', - tx_hash: txHash, - sender_address: transaction.tx.callerId, - contract_address: transaction.tx.contractId, - content: parsedContent.content, - topics: parsedContent.topics, - media: parsedContent.media, - total_comments: 0, - tx_args: transaction.tx.arguments, - created_at: moment(transaction.microTime).toDate(), - post_id: commentInfo.isComment ? commentInfo.parentPostId : null, - }; - - this.logger.debug('Creating new post', { - txHash, - postId: postData.id, - isComment: commentInfo.isComment, - parentPostId: postData.post_id, - topicsCount: postData.topics.length, - mediaCount: postData.media.length, - }); - - // Use database transaction for consistency - const post = await this.postRepository.manager.transaction( - async (manager) => { - const newPost = manager.create(Post, postData); - return await manager.save(newPost); - }, - ); - - // Update parent post comment count if this is a comment - if (commentInfo.isComment && commentInfo.parentPostId) { - await this.updatePostCommentCount(commentInfo.parentPostId); - } - - this.logger.log('Post saved successfully', { - txHash, - postId: post.id, - isComment: commentInfo.isComment, - parentPostId: postData.post_id, - contractAddress: contract.contractAddress, - }); - - return post; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - - this.logger.error('Failed to save post from transaction', { - txHash, - contractAddress: contract.contractAddress, - error: errorMessage, - stack: errorStack, - }); - throw error; - } - } - - /** - * Validates transaction data structure - */ - private validateTransaction(transaction: ITransaction): boolean { - if (!transaction) { - return false; - } - - const requiredFields = ['hash', 'microTime']; - for (const field of requiredFields) { - if (!transaction[field]) { - this.logger.warn(`Transaction missing required field: ${field}`); - return false; - } - } - - if (!transaction.tx) { - this.logger.warn('Transaction missing tx data'); - return false; - } - - const requiredTxFields = ['callerId', 'contractId', 'arguments']; - for (const field of requiredTxFields) { - if (!transaction.tx[field]) { - this.logger.warn(`Transaction.tx missing required field: ${field}`); - return false; - } - } - - return true; - } - - /** - * Generates a unique post ID - */ - private generatePostId( - transaction: ITransaction, - contract: IPostContract, - ): string { - const returnValue = transaction.tx?.return?.value; - if (returnValue) { - return `${returnValue}_v${contract.version}`; - } - - // Fallback to hash-based ID if return value is not available - return `${transaction.hash.slice(-8)}_v${contract.version}`; - } - - /** - * Detects if a transaction represents a comment and extracts parent post information - */ - private detectComment(transaction: ITransaction): ICommentInfo { - if (!transaction?.tx?.arguments?.[1]?.value) { - return { isComment: false }; - } - - const commentArgument = transaction.tx.arguments[1].value.find((arg) => - arg.value?.includes('comment:'), - ); - - if (!commentArgument?.value) { - return { isComment: false }; - } - - const parentPostId = commentArgument.value.split('comment:')[1]; - if (!parentPostId || parentPostId.trim().length === 0) { - this.logger.warn('Invalid comment format: missing parent post ID', { - txHash: transaction.hash, - commentValue: commentArgument.value, - }); - return { isComment: false }; - } - - return { - isComment: true, - parentPostId: parentPostId.trim(), - commentArgument, - }; - } - - /** - * Validates that a parent post exists for a comment with retry logic - * This handles timing issues in parallel processing where parent posts - * might be processed concurrently - */ - private async validateParentPost( - parentPostId: string, - maxRetries: number = 3, - retryDelay: number = 100, - ): Promise { - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - const parentPost = await this.postRepository - .createQueryBuilder('post') - .where('post.id = :parentPostId', { parentPostId }) - .getOne(); - if (parentPost) { - this.logger.debug('Parent post found for comment validation', { - parentPostId, - attempt, - }); - return true; - } - - // If not found and we have retries left, wait and try again - if (attempt < maxRetries) { - this.logger.debug('Parent post not found, retrying...', { - parentPostId, - attempt, - nextRetryIn: retryDelay, - }); - await new Promise((resolve) => setTimeout(resolve, retryDelay)); - retryDelay *= 2; // Exponential backoff - } - } catch (error) { - this.logger.error('Error during parent post validation', { - parentPostId, - attempt, - error: error instanceof Error ? error.message : String(error), - }); - - if (attempt === maxRetries) { - return false; - } - } - } - - this.logger.warn('Parent post not found after all retries', { - parentPostId, - maxRetries, - }); - return false; - } - - /** - * Processes comment-specific logic for existing posts - */ - private async processExistingPostAsComment( - existingPost: Post, - commentInfo: ICommentInfo, - txHash: string, - ): Promise { - if (!commentInfo.isComment || !commentInfo.parentPostId) { - return { success: false, error: 'Invalid comment information' }; - } - - // Check if parent post exists - const parentPostExists = await this.validateParentPost( - commentInfo.parentPostId, - ); - if (!parentPostExists) { - this.logger.warn('Parent post does not exist for comment', { - txHash, - parentPostId: commentInfo.parentPostId, - }); - return { - success: false, - parentPostExists: false, - error: 'Parent post not found', - }; - } - - try { - // Update existing post to be a comment - await this.postRepository.update(existingPost.id, { - post_id: commentInfo.parentPostId, - }); - - // Update comment count for parent post - await this.updatePostCommentCount(commentInfo.parentPostId); - - this.logger.log('Successfully updated existing post as comment', { - txHash, - postId: existingPost.id, - parentPostId: commentInfo.parentPostId, - }); - - return { success: true, parentPostExists: true }; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.error('Failed to process existing post as comment', { - txHash, - postId: existingPost.id, - parentPostId: commentInfo.parentPostId, - error: errorMessage, - }); - return { success: false, error: errorMessage }; - } - } - - /** - * Updates the comment count for a parent post - */ - private async updatePostCommentCount(parentPostId: string): Promise { - try { - const count = await this.postRepository - .createQueryBuilder('post') - .where('post.post_id = :parentPostId', { parentPostId }) - .getCount(); - - await this.postRepository.update( - { id: parentPostId }, - { total_comments: count }, - ); - - this.logger.debug('Updated comment count for parent post', { - parentPostId, - commentCount: count, - }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.error('Failed to update comment count', { - parentPostId, - error: errorMessage, - }); - // Don't throw - comment count update failure shouldn't break the main flow - } - } - - /** - * Fixes orphaned comments by linking them to their parent posts - * This is a cleanup method for comments that were created before their parent posts - */ - async fixOrphanedComments(): Promise { - try { - this.logger.log('Starting orphaned comments cleanup...'); - - // Find comments that have a post_id but the parent post doesn't exist - const orphanedComments = await this.postRepository - .createQueryBuilder('comment') - .leftJoin('posts', 'parent', 'parent.id = comment.post_id') - .where('comment.post_id IS NOT NULL') - .andWhere('parent.id IS NULL') - .getMany(); - - if (orphanedComments.length === 0) { - this.logger.log('No orphaned comments found'); - return; - } - - this.logger.log(`Found ${orphanedComments.length} orphaned comments`); - - let fixedCount = 0; - for (const comment of orphanedComments) { - try { - // Check if parent post now exists - const parentExists = await this.validateParentPost( - comment.post_id, - 1, - 0, - ); - if (parentExists) { - // Update comment count for the parent - await this.updatePostCommentCount(comment.post_id); - fixedCount++; - } else { - // If parent still doesn't exist, remove the post_id to make it a regular post - await this.postRepository.update(comment.id, { post_id: null }); - this.logger.warn('Converted orphaned comment to regular post', { - commentId: comment.id, - originalParentId: comment.post_id, - }); - } - } catch (error) { - this.logger.error('Failed to fix orphaned comment', { - commentId: comment.id, - parentId: comment.post_id, - error: error instanceof Error ? error.message : String(error), - }); - } - } - - this.logger.log( - `Orphaned comments cleanup completed: ${fixedCount} fixed`, - ); - } catch (error) { - this.logger.error('Failed to run orphaned comments cleanup', { - error: error instanceof Error ? error.message : String(error), - }); - } - } -} diff --git a/src/social/utils/content-parser.util.spec.ts b/src/social/utils/content-parser.util.spec.ts deleted file mode 100644 index 382b10a..0000000 --- a/src/social/utils/content-parser.util.spec.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { - parsePostContent, - extractTopics, - extractMedia, - sanitizeContent, - isValidMediaUrl, -} from './content-parser.util'; - -describe('Content Parser Utilities', () => { - describe('parsePostContent', () => { - it('should parse content with topics and media', () => { - const content = 'Hello #world #test this is a post'; - const mediaArguments = [ - { value: 'https://example.com/image.jpg' }, - { value: 'https://example.com/video.mp4' }, - ]; - - const result = parsePostContent(content, mediaArguments); - - expect(result.content).toBe(content); - expect(result.topics).toEqual(['#world', '#test']); - expect(result.media).toEqual([ - 'https://example.com/image.jpg', - 'https://example.com/video.mp4', - ]); - }); - - it('should handle empty content and media', () => { - const result = parsePostContent('', []); - - expect(result.content).toBe(''); - expect(result.topics).toEqual([]); - expect(result.media).toEqual([]); - }); - - it('should apply custom options', () => { - const content = '#one #two #three #four #five'; - const options = { maxTopics: 2, sanitizeContent: false }; - - const result = parsePostContent(content, [], options); - - expect(result.topics).toHaveLength(2); - }); - }); - - describe('extractTopics', () => { - it('should extract hashtags from content', () => { - const content = 'Check out this #awesome #blockchain #dapp'; - const topics = extractTopics(content); - - expect(topics).toEqual(['#awesome', '#blockchain', '#dapp']); - }); - - it('should handle mixed case and convert to lowercase', () => { - const content = 'Testing #CamelCase #UPPERCASE #lowercase'; - const topics = extractTopics(content); - - expect(topics).toEqual(['#camelcase', '#uppercase', '#lowercase']); - }); - - it('should filter out invalid hashtags', () => { - const content = '# #valid_tag #123 #'; - const topics = extractTopics(content); - - expect(topics).toEqual(['#valid_tag', '#123']); - }); - - it('should remove duplicates while preserving order', () => { - const content = '#first #second #first #third #second'; - const topics = extractTopics(content); - - expect(topics).toEqual(['#first', '#second', '#third']); - }); - - it('should respect maxTopics limit', () => { - const content = '#one #two #three #four #five'; - const topics = extractTopics(content, 3); - - expect(topics).toHaveLength(3); - expect(topics).toEqual(['#one', '#two', '#three']); - }); - - it('should handle empty or invalid input', () => { - expect(extractTopics('')).toEqual([]); - expect(extractTopics(null as any)).toEqual([]); - expect(extractTopics(undefined as any)).toEqual([]); - expect(extractTopics(123 as any)).toEqual([]); - }); - - it('should filter out very long hashtags', () => { - const longTag = '#' + 'a'.repeat(60); // 61 characters total - const validTag = '#valid'; - const content = `${longTag} ${validTag}`; - - const topics = extractTopics(content); - - expect(topics).toEqual(['#valid']); - }); - }); - - describe('extractMedia', () => { - it('should extract valid media URLs', () => { - const mediaArguments = [ - { value: 'https://example.com/image.jpg' }, - { value: 'https://example.com/video.mp4' }, - { value: 'not-a-url' }, - { value: null }, - ]; - - const media = extractMedia(mediaArguments); - - expect(media).toEqual([ - 'https://example.com/image.jpg', - 'https://example.com/video.mp4', - ]); - }); - - it('should respect maxMediaItems limit', () => { - const mediaArguments = [ - { value: 'https://example.com/1.jpg' }, - { value: 'https://example.com/2.jpg' }, - { value: 'https://example.com/3.jpg' }, - ]; - - const media = extractMedia(mediaArguments, 2); - - expect(media).toHaveLength(2); - }); - - it('should handle invalid input gracefully', () => { - expect(extractMedia(null as any)).toEqual([]); - expect(extractMedia(undefined as any)).toEqual([]); - expect(extractMedia('not-array' as any)).toEqual([]); - }); - - it('should handle extraction errors', () => { - const mediaArguments = [ - { - value: { - toString: () => { - throw new Error('Test error'); - }, - }, - }, - { value: 'https://example.com/valid.jpg' }, - ]; - - const media = extractMedia(mediaArguments); - - expect(media).toEqual(['https://example.com/valid.jpg']); - }); - }); - - describe('sanitizeContent', () => { - it('should trim whitespace', () => { - const content = ' Hello world '; - const sanitized = sanitizeContent(content); - - expect(sanitized).toBe('Hello world'); - }); - - it('should normalize line endings', () => { - const content = 'Line 1\r\nLine 2\r\nLine 3'; - const sanitized = sanitizeContent(content); - - expect(sanitized).toBe('Line 1\nLine 2\nLine 3'); - }); - - it('should limit consecutive line breaks', () => { - const content = 'Line 1\n\n\n\n\nLine 2'; - const sanitized = sanitizeContent(content); - - expect(sanitized).toBe('Line 1\n\nLine 2'); - }); - - it('should enforce maximum length', () => { - const content = 'a'.repeat(6000); - const sanitized = sanitizeContent(content); - - expect(sanitized).toHaveLength(5000); - }); - - it('should handle invalid input', () => { - expect(sanitizeContent(null as any)).toBe(''); - expect(sanitizeContent(undefined as any)).toBe(''); - expect(sanitizeContent(123 as any)).toBe(''); - }); - }); - - describe('isValidMediaUrl', () => { - it('should validate URLs with media extensions', () => { - const validUrls = [ - 'https://example.com/image.jpg', - 'http://example.com/image.jpeg', - 'https://example.com/image.png', - 'https://example.com/image.gif', - 'https://example.com/image.webp', - 'https://example.com/video.mp4', - 'https://example.com/video.webm', - 'https://example.com/video.mov', - ]; - - validUrls.forEach((url) => { - expect(isValidMediaUrl(url)).toBe(true); - }); - }); - - it('should validate URLs from known media hosts', () => { - const mediaHostUrls = [ - 'https://imgur.com/gallery/abc123', - 'https://giphy.com/gifs/abc123', - 'https://youtube.com/watch?v=abc123', - 'https://vimeo.com/123456789', - 'https://i.imgur.com/abc123', - ]; - - mediaHostUrls.forEach((url) => { - expect(isValidMediaUrl(url)).toBe(true); - }); - }); - - it('should reject invalid URLs', () => { - const invalidUrls = [ - 'not-a-url', - 'ftp://example.com/file.jpg', - 'https://example.com/document.pdf', - 'https://example.com/page.html', - '', - null, - undefined, - ]; - - invalidUrls.forEach((url) => { - expect(isValidMediaUrl(url as any)).toBe(false); - }); - }); - - it('should handle malformed URLs gracefully', () => { - const malformedUrls = [ - 'https://', - 'https://.', - 'https://example', - 'https://example.', - 'javascript:alert(1)', - ]; - - malformedUrls.forEach((url) => { - expect(isValidMediaUrl(url)).toBe(false); - }); - }); - - it('should require valid protocols', () => { - expect(isValidMediaUrl('ftp://example.com/image.jpg')).toBe(false); - expect(isValidMediaUrl('file:///local/image.jpg')).toBe(false); - expect(isValidMediaUrl('data:image/png;base64,abc')).toBe(false); - }); - }); -}); - -/** - * Additional test cases to consider: - * - * 1. Edge cases: - * - Very large content strings - * - Unicode characters in hashtags - * - International domain names - * - URL encoding in media URLs - * - * 2. Security tests: - * - XSS prevention in content - * - Malicious URL detection - * - Script injection attempts - * - * 3. Performance tests: - * - Large number of hashtags - * - Large media arrays - * - Complex regex patterns - * - * 4. Integration tests: - * - Real-world content examples - * - Multi-language content - * - Mixed content types - */ diff --git a/src/social/utils/content-parser.util.ts b/src/social/utils/content-parser.util.ts deleted file mode 100644 index 661febe..0000000 --- a/src/social/utils/content-parser.util.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { - IParsedPostContent, - IContentParsingOptions, -} from '../interfaces/post.interfaces'; - -/** - * Default options for content parsing - */ -const DEFAULT_PARSING_OPTIONS: Required = { - maxTopics: 10, - maxMediaItems: 5, - sanitizeContent: true, -}; - -/** - * Parses post content and extracts topics and media - */ -export function parsePostContent( - content: string, - mediaArguments: any[] = [], - options: IContentParsingOptions = {}, -): IParsedPostContent { - const config = { ...DEFAULT_PARSING_OPTIONS, ...options }; - - // Sanitize content if requested - const sanitizedContent = config.sanitizeContent - ? sanitizeContent(content) - : content; - - // Extract topics (hashtags) - const topics = extractTopics(sanitizedContent, config.maxTopics); - - // Extract media URLs - const media = extractMedia(mediaArguments, config.maxMediaItems); - - return { - content: sanitizedContent, - topics, - media, - }; -} - -/** - * Extracts hashtags from content - */ -export function extractTopics( - content: string, - maxTopics: number = 10, -): string[] { - if (!content || typeof content !== 'string') { - return []; - } - - const topics = content - .split(/\s+/) - .filter((word) => word.startsWith('#') && word.length > 1) - .map((topic) => topic.toLowerCase().replace(/[^a-z0-9#_]/g, '')) - .filter((topic) => topic.length > 1 && topic.length <= 50) // Reasonable length limits - .slice(0, maxTopics); - - // Remove duplicates while preserving order - return [...new Set(topics)]; -} - -/** - * Extracts media URLs from transaction arguments - */ -export function extractMedia( - mediaArguments: any[] = [], - maxMediaItems: number = 5, -): string[] { - if (!Array.isArray(mediaArguments)) { - return []; - } - - try { - const media = mediaArguments - .map((item) => item?.value) - .filter((value) => value && typeof value === 'string') - .filter((url) => isValidMediaUrl(url)) - .slice(0, maxMediaItems); - - return media; - } catch (error) { - console.warn('Error extracting media from arguments:', error); - return []; - } -} - -/** - * Basic content sanitization - */ -export function sanitizeContent(content: string): string { - if (!content || typeof content !== 'string') { - return ''; - } - - return content - .trim() - .replace(/\r\n/g, '\n') // Normalize line endings - .replace(/\n{3,}/g, '\n\n') // Limit consecutive line breaks - .slice(0, 5000); // Reasonable length limit -} - -/** - * Validates if a URL appears to be a valid media URL - */ -export function isValidMediaUrl(url: string): boolean { - if (!url || typeof url !== 'string') { - return false; - } - - try { - const parsedUrl = new URL(url); - const validProtocols = ['http:', 'https:']; - const mediaExtensions = [ - '.jpg', - '.jpeg', - '.png', - '.gif', - '.webp', - '.mp4', - '.webm', - '.mov', - ]; - - const hasValidProtocol = validProtocols.includes(parsedUrl.protocol); - const hasMediaExtension = mediaExtensions.some((ext) => - parsedUrl.pathname.toLowerCase().endsWith(ext), - ); - - // Allow URLs from common media hosting services even without explicit extensions - const commonMediaHosts = [ - 'imgur.com', - 'giphy.com', - 'youtube.com', - 'vimeo.com', - ]; - const isFromMediaHost = commonMediaHosts.some((host) => - parsedUrl.hostname.includes(host), - ); - - return hasValidProtocol && (hasMediaExtension || isFromMediaHost); - } catch { - return false; - } -} diff --git a/src/transactions/services/transaction-history.service.spec.ts b/src/transactions/services/transaction-history.service.spec.ts index 5a183bf..03a81c1 100644 --- a/src/transactions/services/transaction-history.service.spec.ts +++ b/src/transactions/services/transaction-history.service.spec.ts @@ -5,8 +5,8 @@ import { DataSource, Repository } from 'typeorm'; import { Transaction } from '../entities/transaction.entity'; import { TokensService } from '@/tokens/tokens.service'; -import { TransactionHistoryService } from './transaction-history.service'; -import { TransactionService } from './transaction.service'; +import { TransactionHistoryService } from '../services/transaction-history.service'; +import { TransactionService } from '../services/transaction.service'; describe('TransactionHistoryService', () => { let service: TransactionHistoryService;