Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ doc/api/
*.js_
*.js.deps
*.js.map
.flutter-plugins-dependencies
54 changes: 54 additions & 0 deletions bin/config/web_scraping_config.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/// Configuration for web scraping operations
class WebScrapingConfig {
/// Maximum number of retry attempts for web scraping operations
static const int maxRetries = 3;

/// Base delay between retries (will be exponentially increased)
static const Duration baseRetryDelay = Duration(seconds: 2);

/// Rate limiting delay between requests to avoid being blocked
static const Duration rateLimitDelay = Duration(seconds: 10);

/// Timeout for web scraping operations
static const Duration timeout = Duration(seconds: 30);

/// User agent string to use for requests - looks like a regular browser
static const String userAgent =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

/// Maximum number of concurrent scraping operations
static const int maxConcurrentOperations = 2;

/// Whether to enable detailed logging
static const bool enableDetailedLogging = true;

/// Retry delays for different types of failures
static const Map<String, Duration> retryDelays = {
'network': Duration(seconds: 5),
'rate_limit': Duration(seconds: 30),
'server_error': Duration(seconds: 10),
'timeout': Duration(seconds: 15),
};

/// URLs that should be treated as rate-limited
static const List<String> rateLimitedDomains = [
'untappd.com',
'beerizer.com',
];

/// Custom delays for specific domains
static const Map<String, Duration> domainDelays = {
'untappd.com': Duration(seconds: 15),
'beerizer.com': Duration(seconds: 10),
};

/// Get delay for a specific domain
static Duration getDelayForDomain(String domain) {
return domainDelays[domain] ?? rateLimitDelay;
}

/// Check if a domain should be rate-limited
static bool isRateLimitedDomain(String domain) {
return rateLimitedDomains.contains(domain);
}
}
13 changes: 10 additions & 3 deletions bin/main.dart
Original file line number Diff line number Diff line change
@@ -1,10 +1,17 @@
import 'dart:io';

import 'package:sentry/sentry.dart';

import 'beer_bot.dart';

String BOT_TOKEN = Platform.environment['DISCORD_TOKEN'] ?? '';
String SENTRY_DSN = Platform.environment['SENTRY_DSN'] ?? '';

void main(List<String> arguments) async {
// Initialize the bot
BeerBot().init(BOT_TOKEN);
Future<void> main(List<String> arguments) async {
await Sentry.init(
(options) {
options.dsn = SENTRY_DSN;
},
appRunner: () => BeerBot().init(BOT_TOKEN),
);
}
Loading