diff --git a/CLAUDE.md b/CLAUDE.md index e54da94f1..b8950ce56 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -214,6 +214,7 @@ Fully deployed. 1010+ tests passing. Facilitator queue (3-panel admin-like UI fo - `readme/Privacy.md` — Privacy Policy source content with GDPR legal basis (Vue component renders this) - `readme/Twitter.md` — Automated Twitter/X commands (schedule, data sources, tweet format, Browsershot config) - `readme/TagSuggestions.md` — Quick tags sync API (mobile presets, bulk-replace, cross-device sync) +- `readme/ExportData.md` — CSV data export system (user/team/location exports, column layout, date filters, S3 pipeline) ## Daily Changelog After every change in a session, append a one-line entry to `readme/changelog/YYYY-MM-DD.md` (create the file if it doesn't exist for today's date). Group entries by session. This is the running record of all work done each day. diff --git a/app/Actions/Teams/DownloadTeamDataAction.php b/app/Actions/Teams/DownloadTeamDataAction.php index 5c38aa291..d1ba17164 100644 --- a/app/Actions/Teams/DownloadTeamDataAction.php +++ b/app/Actions/Teams/DownloadTeamDataAction.php @@ -10,16 +10,21 @@ class DownloadTeamDataAction { - public function run(User $user, Team $team) + public function run(User $user, Team $team, array $dateFilter = []) { $path = now()->format('Y') . "/" . now()->format('m') . "/" . now()->format('d') . - "/" . now()->getTimestamp() . - '/_Team_OpenLitterMap.csv'; // 2020/10/25/unix/ + "/" . now()->getTimestamp(); + + if (!empty($dateFilter)) { + $path .= '_from_' . $dateFilter['fromDate'] . '_to_' . $dateFilter['toDate']; + } + + $path .= '/_Team_OpenLitterMap.csv'; /* Dispatch job to create CSV file for export */ - (new CreateCSVExport(null, null, $team->id)) + (new CreateCSVExport(null, null, $team->id, null, $dateFilter)) ->queue($path, 's3', null, ['visibility' => 'public']) ->chain([ // These jobs are executed when above is finished. diff --git a/app/Console/Commands/SendEmailToSubscribed.php b/app/Console/Commands/SendEmailToSubscribed.php index 576d82a58..ce344dbc6 100644 --- a/app/Console/Commands/SendEmailToSubscribed.php +++ b/app/Console/Commands/SendEmailToSubscribed.php @@ -37,9 +37,9 @@ public function handle(): int $userCount = User::where('emailsub', 1)->count(); - // Subscribers not already in users table (avoid duplicates) - $userEmails = User::where('emailsub', 1)->pluck('email')->toArray(); - $subscriberCount = Subscriber::whereNotIn('email', $userEmails)->count(); + // Exclude ALL user emails from subscribers (users control their own sub via emailsub flag) + $userEmailSubquery = User::withoutGlobalScopes()->select('email'); + $subscriberCount = Subscriber::whereNotIn('email', $userEmailSubquery)->count(); $totalCount = $userCount + $subscriberCount; @@ -78,7 +78,7 @@ public function handle(): int }); // ─── Subscribers (deduplicated) ────────────────────────────── - Subscriber::whereNotIn('email', $userEmails) + Subscriber::whereNotIn('email', $userEmailSubquery) ->orderBy('id') ->chunk($chunkSize, function ($subscribers) use (&$dispatched, $bar) { foreach ($subscribers as $subscriber) { diff --git a/app/Exports/CreateCSVExport.php b/app/Exports/CreateCSVExport.php index 195b7d24d..ac85b18d8 100644 --- a/app/Exports/CreateCSVExport.php +++ b/app/Exports/CreateCSVExport.php @@ -2,7 +2,12 @@ namespace App\Exports; +use App\Enums\VerificationStatus; use App\Models\Litter\Tags\Category; +use App\Models\Litter\Tags\LitterObjectType; +use App\Models\Litter\Tags\Materials; +use App\Models\Litter\Tags\PhotoTag; +use App\Models\Litter\Tags\PhotoTagExtraTags; use App\Models\Photo; use Illuminate\Bus\Queueable; @@ -29,6 +34,15 @@ class CreateCSVExport implements FromQuery, WithMapping, WithHeadings */ private array $categoryObjects = []; + /** @var array */ + private array $materials = []; + + /** @var array */ + private array $types = []; + + private bool $hasBrands = false; + private bool $hasCustomTags = false; + public $timeout = 240; public function __construct($location_type, $location_id, $team_id = null, $user_id = null, array $dateFilter = []) @@ -39,18 +53,69 @@ public function __construct($location_type, $location_id, $team_id = null, $user $this->user_id = $user_id; $this->dateFilter = $dateFilter; + // Pre-scan: find which columns actually have data for this export scope. + // Use subqueries (not pluck) so MySQL optimizes internally for large exports. + $photoIdQuery = $this->scopeQuery(Photo::query())->select('id'); + $tagIdQuery = PhotoTag::whereIn('photo_id', $photoIdQuery)->select('id'); + + $activeObjectIds = PhotoTag::whereIn('photo_id', $photoIdQuery) + ->whereNotNull('category_id') + ->whereNotNull('litter_object_id') + ->select('category_id', 'litter_object_id') + ->distinct() + ->get(); + + $activeCatIds = $activeObjectIds->pluck('category_id')->unique()->all(); + $activeObjMap = $activeObjectIds->groupBy('category_id') + ->map(fn ($rows) => $rows->pluck('litter_object_id')->all()) + ->all(); + + // Single query for all extra tag types + $extraTagTypes = PhotoTagExtraTags::whereIn('photo_tag_id', $tagIdQuery) + ->select('tag_type', 'tag_type_id') + ->distinct() + ->get(); + + $activeMaterialIds = $extraTagTypes->where('tag_type', 'material')->pluck('tag_type_id')->all(); + $this->hasBrands = $extraTagTypes->where('tag_type', 'brand')->isNotEmpty(); + $this->hasCustomTags = $extraTagTypes->where('tag_type', 'custom_tag')->isNotEmpty(); + + $activeTypeIds = PhotoTag::whereIn('photo_id', $photoIdQuery) + ->whereNotNull('litter_object_type_id') + ->distinct() + ->pluck('litter_object_type_id') + ->all(); + + // Load and filter category/object columns to only those with data $this->categoryObjects = Category::with(['litterObjects' => fn ($q) => $q->orderBy('litter_objects.id')]) + ->whereIn('id', $activeCatIds) ->orderBy('id') ->get() ->map(fn ($cat) => [ 'id' => $cat->id, 'key' => $cat->key, - 'objects' => $cat->litterObjects->map(fn ($obj) => [ - 'id' => $obj->id, - 'key' => $obj->key, - ])->values()->toArray(), + 'objects' => $cat->litterObjects + ->filter(fn ($obj) => in_array($obj->id, $activeObjMap[$cat->id] ?? [])) + ->map(fn ($obj) => ['id' => $obj->id, 'key' => $obj->key]) + ->values() + ->toArray(), ]) + ->filter(fn ($cat) => !empty($cat['objects'])) + ->values() ->toArray(); + + // Filter materials and types to only those with data + $this->materials = !empty($activeMaterialIds) + ? Materials::whereIn('id', $activeMaterialIds)->orderBy('id')->get() + ->map(fn ($m) => ['id' => $m->id, 'key' => $m->key]) + ->toArray() + : []; + + $this->types = !empty($activeTypeIds) + ? LitterObjectType::whereIn('id', $activeTypeIds)->orderBy('id')->get() + ->map(fn ($t) => ['id' => $t->id, 'key' => $t->key]) + ->toArray() + : []; } /** @@ -81,7 +146,29 @@ public function headings(): array } } - return array_merge($result, ['custom_tag_1', 'custom_tag_2', 'custom_tag_3']); + if (!empty($this->materials)) { + $result[] = 'MATERIALS'; + foreach ($this->materials as $material) { + $result[] = $material['key']; + } + } + + if (!empty($this->types)) { + $result[] = 'TYPES'; + foreach ($this->types as $type) { + $result[] = $type['key']; + } + } + + if ($this->hasBrands) { + $result[] = 'brands'; + } + + if ($this->hasCustomTags) { + $result = array_merge($result, ['custom_tag_1', 'custom_tag_2', 'custom_tag_3']); + } + + return $result; } /** @@ -93,27 +180,53 @@ public function map($row): array { $result = [ $row->id, - $row->verified, + $row->verified->value, $row->model, $row->datetime, $row->created_at, $row->lat, $row->lon, - $row->remaining ? 'No' : 'Yes', + $row->picked_up ? 'Yes' : 'No', $row->display_name, $row->summary['totals']['litter'] ?? $row->total_tags, ]; $tags = $row->summary['tags'] ?? []; + $brandKeys = $row->summary['keys']['brands'] ?? []; - // Build lookup: category_id → object_id → quantity (from flat tags array) + // Single pass: iterate flat tags array + // Structure: [ { clo_id, category_id, object_id, type_id, quantity, materials: [id, ...], brands: {id: qty}, custom_tags: [id, ...] } ] $tagLookup = []; + $materialLookup = []; + $typeLookup = []; + $brandParts = []; + foreach ($tags as $tag) { $catId = $tag['category_id'] ?? 0; $objId = $tag['object_id'] ?? 0; - $tagLookup[$catId][$objId] = ($tagLookup[$catId][$objId] ?? 0) + ($tag['quantity'] ?? 0); + $qty = $tag['quantity'] ?? 0; + + $tagLookup[$catId][$objId] = ($tagLookup[$catId][$objId] ?? 0) + $qty; + + // Materials: array of IDs — each gets the parent tag's quantity + foreach ($tag['materials'] ?? [] as $materialId) { + $materialLookup[$materialId] = ($materialLookup[$materialId] ?? 0) + $qty; + } + + // Types: type_id is in the summary + $typeId = $tag['type_id'] ?? null; + if ($typeId) { + $typeLookup[$typeId] = ($typeLookup[$typeId] ?? 0) + $qty; + } + + // Brands: {id: qty} objects with independent quantities + foreach ($tag['brands'] ?? [] as $brandId => $brandQty) { + $brandName = $brandKeys[$brandId] ?? "brand_{$brandId}"; + $brandParts[$brandName] = ($brandParts[$brandName] ?? 0) + $brandQty; + } } + // Category/object columns foreach ($this->categoryObjects as $category) { $result[] = null; // category separator column @@ -122,15 +235,42 @@ public function map($row): array } } - // Custom tags from extra_tags (eager-loaded in query()) - $customTagNames = $row->photoTags - ->flatMap(fn ($pt) => $pt->extraTags->where('tag_type', 'custom_tag')) - ->take(3) - ->map(fn ($extra) => $extra->extraTag?->key) - ->values() - ->toArray(); + // Materials columns (only if any exist in export scope) + if (!empty($this->materials)) { + $result[] = null; // MATERIALS separator + foreach ($this->materials as $material) { + $result[] = $materialLookup[$material['id']] ?? null; + } + } + + // Types columns (only if any exist in export scope) + if (!empty($this->types)) { + $result[] = null; // TYPES separator + foreach ($this->types as $type) { + $result[] = $typeLookup[$type['id']] ?? null; + } + } - return array_merge($result, array_pad($customTagNames, 3, null)); + // Brands: single delimited column (only if any exist in export scope) + if ($this->hasBrands) { + $result[] = !empty($brandParts) + ? implode(';', array_map(fn ($name, $qty) => "{$name}:{$qty}", array_keys($brandParts), array_values($brandParts))) + : null; + } + + // Custom tags (only if any exist in export scope) + if ($this->hasCustomTags) { + $customTagNames = $row->photoTags + ->flatMap(fn ($pt) => $pt->extraTags->where('tag_type', 'custom_tag')) + ->take(3) + ->map(fn ($extra) => $extra->extraTag?->key) + ->values() + ->toArray(); + + $result = array_merge($result, array_pad($customTagNames, 3, null)); + } + + return $result; } /** @@ -138,25 +278,43 @@ public function map($row): array */ public function query() { - $query = Photo::with(['photoTags.extraTags.extraTag']); + return $this->scopeQuery( + Photo::with(['photoTags.extraTags.extraTag']) + ); + } + /** + * Apply the export scope (user/team/location + date filter + verification) to a query. + */ + private function scopeQuery($query) + { if (!empty($this->dateFilter)) { - $query->whereBetween( - $this->dateFilter['column'], - [$this->dateFilter['fromDate'], $this->dateFilter['toDate']] - ); + $allowedColumns = ['created_at', 'datetime', 'updated_at']; + $column = in_array($this->dateFilter['column'], $allowedColumns, true) + ? $this->dateFilter['column'] + : 'datetime'; + + $query->whereBetween($column, [ + $this->dateFilter['fromDate'], + $this->dateFilter['toDate'], + ]); } if ($this->user_id) { - return $query->where(['user_id' => $this->user_id]); - } elseif ($this->team_id) { - return $query->where(['team_id' => $this->team_id, 'verified' => 2]); + return $query->where('user_id', $this->user_id); + } + + // Team/location exports: only approved photos (ADMIN_APPROVED and above) + $query->where('verified', '>=', VerificationStatus::ADMIN_APPROVED->value); + + if ($this->team_id) { + return $query->where('team_id', $this->team_id); } elseif ($this->location_type === 'city') { - return $query->where(['city_id' => $this->location_id, 'verified' => 2]); + return $query->where('city_id', $this->location_id); } elseif ($this->location_type === 'state') { - return $query->where(['state_id' => $this->location_id, 'verified' => 2]); + return $query->where('state_id', $this->location_id); } else { - return $query->where(['country_id' => $this->location_id, 'verified' => 2]); + return $query->where('country_id', $this->location_id); } } } diff --git a/app/Http/Controllers/API/TeamsController.php b/app/Http/Controllers/API/TeamsController.php index 685f6c6ad..6d63792c6 100644 --- a/app/Http/Controllers/API/TeamsController.php +++ b/app/Http/Controllers/API/TeamsController.php @@ -4,6 +4,7 @@ use App\Actions\Teams\CreateTeamAction; use App\Actions\Teams\DownloadTeamDataAction; +use Carbon\Carbon; use App\Actions\Teams\JoinTeamAction; use App\Actions\Teams\LeaveTeamAction; use App\Actions\Teams\ListTeamMembersAction; @@ -232,7 +233,30 @@ public function download (Request $request, DownloadTeamDataAction $action): arr return $this->fail('not-a-member'); } - $action->run($user, $team); + if (!$team->isLeader($user->id) && !$user->hasRole('school_manager')) { + return $this->fail('not-authorized'); + } + + $dateFilter = []; + if ($request->dateField && ($request->fromDate || $request->toDate)) { + try { + $dateFilter = [ + 'column' => in_array($request->dateField, ['created_at', 'datetime', 'updated_at'], true) + ? $request->dateField + : 'datetime', + 'fromDate' => $request->fromDate + ? Carbon::parse($request->fromDate)->toDateString() + : '2017-01-01', + 'toDate' => $request->toDate + ? Carbon::parse($request->toDate)->toDateString() + : now()->toDateString(), + ]; + } catch (\Exception) { + // Invalid date format — ignore filter and export all data + } + } + + $action->run($user, $team, $dateFilter); return $this->success(); } diff --git a/app/Http/Controllers/Teams/TeamPhotosController.php b/app/Http/Controllers/Teams/TeamPhotosController.php index 0e512e167..2a5501934 100644 --- a/app/Http/Controllers/Teams/TeamPhotosController.php +++ b/app/Http/Controllers/Teams/TeamPhotosController.php @@ -446,19 +446,41 @@ public function mapPoints(Request $request): JsonResponse $points = Photo::where('team_id', $team->id) ->whereNotNull('lat') ->whereNotNull('lon') - ->select(['id', 'lat', 'lon', 'verified', 'is_public', 'total_tags', 'created_at']) + ->with(['user:id,name,username,show_username_maps,show_name_maps,global_flag']) + ->select([ + 'id', 'user_id', 'lat', 'lon', 'verified', 'is_public', + 'total_tags', 'remaining', 'filename', 'datetime', 'summary', + 'created_at', + ]) ->orderByDesc('created_at') ->limit(5000) ->get() - ->map(fn ($photo) => [ - 'id' => $photo->id, - 'lat' => $photo->lat, - 'lng' => $photo->lon, - 'tags' => $photo->total_tags, - 'verified' => $photo->verified->value, - 'is_public' => $photo->is_public, - 'date' => $photo->created_at->toDateString(), - ]); + ->map(function ($photo) use ($team) { + $applySafeguarding = $team->safeguarding; + + return [ + 'id' => $photo->id, + 'lat' => $photo->lat, + 'lng' => $photo->lon, + 'tags' => $photo->total_tags, + 'verified' => $photo->verified->value, + 'is_public' => $photo->is_public, + 'date' => $photo->created_at->toDateString(), + // Popup fields — same shape as global map PointsController::show + 'filename' => $photo->filename, + 'datetime' => $photo->datetime, + 'picked_up' => $photo->picked_up, + 'summary' => $photo->summary, + 'team' => $team->name, + 'name' => $applySafeguarding ? null : ( + $photo->user && $photo->user->show_name_maps ? $photo->user->name : null + ), + 'username' => $applySafeguarding ? null : ( + $photo->user && $photo->user->show_username_maps ? $photo->user->username : null + ), + 'flag' => $applySafeguarding ? null : $photo->user?->global_flag, + ]; + }); return response()->json([ 'success' => true, diff --git a/app/Http/Controllers/User/Photos/UsersUploadsController.php b/app/Http/Controllers/User/Photos/UsersUploadsController.php index 9540e0046..6ce2c6d36 100644 --- a/app/Http/Controllers/User/Photos/UsersUploadsController.php +++ b/app/Http/Controllers/User/Photos/UsersUploadsController.php @@ -24,79 +24,7 @@ public function index(Request $request): JsonResponse $query = Photo::where('user_id', $user->id) ->where('filename', '!=', '/assets/verified.jpg'); - // Tagged/Untagged filter — uses summary column (set by GeneratePhotoSummaryService on tagging) - if ($request->has('tagged')) { - $request->boolean('tagged') - ? $query->whereNotNull('summary') - : $query->whereNull('summary'); - } - - // ID filter - if ($request->filled('id')) { - $id = $request->integer('id'); - $operator = $request->input('id_operator', '='); - if (! in_array($operator, ['=', '>', '<'], true)) { - $operator = '='; - } - $query->where('id', $operator, $id); - } - - // Tag filter (search by litter object key) - if ($request->filled('tag')) { - $tag = $request->input('tag'); - $query->whereHas('photoTags.object', function($q) use ($tag) { - $q->where('key', 'like', "%{$tag}%"); - }); - } - - // Custom tag filter (search through extra tags) - if ($request->filled('custom_tag')) { - $customTag = $request->input('custom_tag'); - $query->whereHas('photoTags.extraTags', function($q) use ($customTag) { - $q->where('tag_type', 'custom_tag') - ->whereHas('extraTag', function($q2) use ($customTag) { - $q2->where('key', 'like', "%{$customTag}%"); - }); - }); - } - - // Picked up filter (per-tag level: true, false, or null=no info) - if ($request->has('picked_up')) { - $pickedUp = $request->input('picked_up'); - if ($pickedUp === 'true') { - $query->whereHas('photoTags', fn($q) => $q->where('picked_up', true)); - } elseif ($pickedUp === 'false') { - $query->whereHas('photoTags', fn($q) => $q->where('picked_up', false)); - } - } - - // Country filter - if ($request->filled('country')) { - $query->whereHas('countryRelation', fn($q) => $q->where('country', $request->input('country'))); - } - - // State filter - if ($request->filled('state')) { - $query->whereHas('stateRelation', fn($q) => $q->where('state', $request->input('state'))); - } - - // City filter - if ($request->filled('city')) { - $query->whereHas('cityRelation', fn($q) => $q->where('city', $request->input('city'))); - } - - // Verified status filter - if ($request->has('verified') && $request->input('verified') !== null) { - $query->where('verified', (int) $request->input('verified')); - } - - // Date range filter - if ($request->filled('date_from')) { - $query->where('datetime', '>=', $request->input('date_from')); - } - if ($request->filled('date_to')) { - $query->where('datetime', '<=', $request->input('date_to')); - } + $this->applyFilters($query, $request); $photos = $query ->with([ @@ -204,7 +132,13 @@ public function stats(Request $request): JsonResponse $user = $request->user(); $userId = $user->id; - // Use photo count in cache key for auto-invalidation on upload + $hasFilters = $request->hasAny(['tagged', 'id', 'tag', 'custom_tag', 'picked_up', 'country', 'state', 'city', 'verified', 'date_from', 'date_to']); + + if ($hasFilters) { + return $this->filteredStats($request, $userId); + } + + // Unfiltered: use cached global stats $baseQuery = Photo::where('user_id', $userId) ->where('filename', '!=', '/assets/verified.jpg'); $totalPhotos = $baseQuery->count(); @@ -241,6 +175,30 @@ public function stats(Request $request): JsonResponse return response()->json($data); } + private function filteredStats(Request $request, int $userId): JsonResponse + { + $query = Photo::where('user_id', $userId) + ->where('filename', '!=', '/assets/verified.jpg'); + + $this->applyFilters($query, $request); + + $totalPhotos = (clone $query)->count(); + $leftToTag = (clone $query)->whereNull('summary')->count(); + $totalTags = (int) (clone $query)->sum('total_tags'); + + $taggedPhotos = max(0, $totalPhotos - $leftToTag); + $taggedPercentage = $totalPhotos > 0 + ? (int) round(($taggedPhotos / $totalPhotos) * 100) + : 0; + + return response()->json([ + 'totalPhotos' => $totalPhotos, + 'totalTags' => $totalTags, + 'leftToTag' => $leftToTag, + 'taggedPercentage' => $taggedPercentage, + ]); + } + /** * Get hierarchical location data for the user's photos */ @@ -412,4 +370,75 @@ private function getNewTags($photo): array return $newTags; } + + /** + * Apply shared filters to a user photos query. + */ + private function applyFilters($query, Request $request): void + { + if ($request->has('tagged')) { + $request->boolean('tagged') + ? $query->whereNotNull('summary') + : $query->whereNull('summary'); + } + + if ($request->filled('id')) { + $id = $request->integer('id'); + $operator = $request->input('id_operator', '='); + if (! in_array($operator, ['=', '>', '<'], true)) { + $operator = '='; + } + $query->where('id', $operator, $id); + } + + if ($request->filled('tag')) { + $tag = $request->input('tag'); + $query->whereHas('photoTags.object', function ($q) use ($tag) { + $q->where('key', 'like', "%{$tag}%"); + }); + } + + if ($request->filled('custom_tag')) { + $customTag = $request->input('custom_tag'); + $query->whereHas('photoTags.extraTags', function ($q) use ($customTag) { + $q->where('tag_type', 'custom_tag') + ->whereHas('extraTag', function ($q2) use ($customTag) { + $q2->where('key', 'like', "%{$customTag}%"); + }); + }); + } + + if ($request->has('picked_up')) { + $pickedUp = $request->input('picked_up'); + if ($pickedUp === 'true') { + $query->whereHas('photoTags', fn ($q) => $q->where('picked_up', true)); + } elseif ($pickedUp === 'false') { + $query->whereHas('photoTags', fn ($q) => $q->where('picked_up', false)); + } + } + + if ($request->filled('country')) { + $query->whereHas('countryRelation', fn ($q) => $q->where('country', $request->input('country'))); + } + + if ($request->filled('state')) { + $query->whereHas('stateRelation', fn ($q) => $q->where('state', $request->input('state'))); + } + + if ($request->filled('city')) { + $query->whereHas('cityRelation', fn ($q) => $q->where('city', $request->input('city'))); + } + + if ($request->has('verified') && $request->input('verified') !== null) { + $query->where('verified', (int) $request->input('verified')); + } + + if ($request->filled('date_from')) { + $query->where('datetime', '>=', $request->input('date_from')); + } + + if ($request->filled('date_to')) { + $query->where('datetime', '<=', $request->input('date_to')); + } + } } diff --git a/app/Jobs/Emails/DispatchEmail.php b/app/Jobs/Emails/DispatchEmail.php index 2a38946e9..c4eab4e58 100644 --- a/app/Jobs/Emails/DispatchEmail.php +++ b/app/Jobs/Emails/DispatchEmail.php @@ -9,6 +9,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Mail; +use Illuminate\Queue\Middleware\RateLimited; class DispatchEmail implements ShouldQueue { @@ -25,6 +26,11 @@ public function __construct($user) $this->user = $user; } + public function middleware(): array + { + return [new RateLimited('ses-emails')]; + } + public function handle(): void { Mail::to($this->user->email)->send(new EmailUpdate($this->user)); diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 9d48f4803..7a90f8747 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -9,11 +9,12 @@ use App\Models\Users\User; use App\Observers\PhotoObserver; use App\Services\Clustering\ClusteringService; -use Illuminate\Auth\Notifications\ResetPassword; use Illuminate\Database\Eloquent\Relations\Relation; use Illuminate\Support\Facades\Gate; use Illuminate\Support\ServiceProvider; use Laravel\Cashier\Cashier; +use Illuminate\Cache\RateLimiting\Limit; +use Illuminate\Support\Facades\RateLimiter; class AppServiceProvider extends ServiceProvider { @@ -47,5 +48,9 @@ public function boot() Gate::define('viewPulse', function (User $user) { return $user->hasRole('superadmin'); }); + + RateLimiter::for('ses-emails', function () { + return Limit::perSecond(12); + }); } } diff --git a/package.json b/package.json index 2ff21cc96..0f371bed5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openlittermap-web", - "version": "5.7.5", + "version": "5.8.2", "type": "module", "author": "Seán Lynch", "license": "GPL v3", diff --git a/readme/API.md b/readme/API.md index 3d60719bb..975bdf27c 100644 --- a/readme/API.md +++ b/readme/API.md @@ -1990,13 +1990,32 @@ Returns max 5000 points with `id`, `lat`, `lng`, `tags`, `verified`, `is_public` ### POST /api/teams/download — Download Team Data -**Auth:** Required (Sanctum) +**Auth:** Required (Sanctum). Must be team leader or have `school_manager` role. + +**Request:** +```json +{ + "team_id": 1, + "dateField": "datetime", + "fromDate": "2025-01-01", + "toDate": "2025-12-31" +} +``` + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `team_id` | int | Yes | Team ID | +| `dateField` | string | No | Column to filter: `created_at`, `datetime`, or `updated_at` | +| `fromDate` | string | No | Start date (YYYY-MM-DD). Default: `2017-01-01` | +| `toDate` | string | No | End date (YYYY-MM-DD). Default: today | -**Request:** `{ "team_id": 1 }` **Response:** `{ "success": true }` -**Error:** `{ "success": false, "message": "not-a-member" }` +**Errors:** +- `{ "success": false, "message": "team-not-found" }` — invalid team_id +- `{ "success": false, "message": "not-a-member" }` — user not on team +- `{ "success": false, "message": "not-authorized" }` — member but not leader/school_manager -Queues background export job. +Queues background CSV export (only `verified >= ADMIN_APPROVED` photos). Emails S3 download link when ready. See `readme/ExportData.md` for full CSV column layout. --- diff --git a/readme/ExportData.md b/readme/ExportData.md new file mode 100644 index 000000000..0a85601fd --- /dev/null +++ b/readme/ExportData.md @@ -0,0 +1,256 @@ +# Data Export System + +CSV export for users, teams, and locations. Queued via Maatwebsite/Excel to S3, then an email with a download link is sent to the user. + +## Architecture + +``` +User/Team/Location triggers export + ↓ +Controller validates auth + builds date filter + ↓ +CreateCSVExport queued to S3 (Maatwebsite/Excel FromQuery) + ↓ +EmailUserExportCompleted job chains → sends ExportWithLink mailable + ↓ +User receives email with S3 download URL +``` + +## Key Files + +| File | Purpose | +|------|---------| +| `app/Exports/CreateCSVExport.php` | Core CSV export class (headings, mapping, query) | +| `app/Actions/Teams/DownloadTeamDataAction.php` | Orchestrates team export dispatch | +| `app/Http/Controllers/User/ProfileController.php` | User export endpoint (`download()`) | +| `app/Http/Controllers/API/TeamsController.php` | Team export endpoint (`download()`) | +| `app/Http/Controllers/DownloadControllerNew.php` | Location-based export endpoint | +| `app/Jobs/EmailUserExportCompleted.php` | Queued job — sends download email | +| `app/Mail/ExportWithLink.php` | Mailable with S3 URL | +| `config/excel.php` | Maatwebsite/Excel config (chunk size: 1000) | + +### Frontend + +| File | Purpose | +|------|---------| +| `resources/js/views/User/Uploads/components/UploadsHeader.vue` | Export CSV button on Uploads page | +| `resources/js/views/Teams/TeamsHub.vue` | Export CSV button on Teams page | +| `resources/js/stores/teams/index.js` | `downloadTeamData()` store action | + +### Tests + +| File | Coverage | +|------|----------| +| `tests/Unit/Exports/CreateCSVExportTest.php` | Headings, mapping, materials aggregation, brands format, types from DB, null values | +| `tests/Feature/Teams/DownloadTeamDataTest.php` | Member auth, non-member rejection, date filter | + +## API Endpoints + +### GET /api/user/profile/download — User Data Export + +**Auth:** Required (Sanctum) + +**Query params (all optional):** + +| Param | Type | Description | +|-------|------|-------------| +| `dateField` | string | Column to filter: `created_at`, `datetime`, or `updated_at` | +| `fromDate` | string | Start date (YYYY-MM-DD). Default: `2017-01-01` | +| `toDate` | string | End date (YYYY-MM-DD). Default: today | + +**Response:** `{ "success": true }` + +Exports **all** user photos (any verification status). Email sent when ready. + +### POST /api/teams/download — Team Data Export + +**Auth:** Required (Sanctum). Must be a team member. + +**Request body:** + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `team_id` | int | Yes | Team ID | +| `dateField` | string | No | Column to filter: `created_at`, `datetime`, or `updated_at` | +| `fromDate` | string | No | Start date (YYYY-MM-DD). Default: `2017-01-01` | +| `toDate` | string | No | End date (YYYY-MM-DD). Default: today | + +**Response:** `{ "success": true }` +**Error:** `{ "success": false, "message": "not-a-member" }` + +Only team leaders and `school_manager` role can export. Returns `{ "success": false, "message": "not-authorized" }` for other members. + +Exports only `verified >= ADMIN_APPROVED` team photos (includes BBOX_APPLIED, BBOX_VERIFIED, AI_READY). Email sent when ready. + +### POST /api/download — Location Data Export + +**Auth:** Optional (uses `email` param if unauthenticated) + +**Request body:** + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `locationType` | string | Yes | `country`, `state`, or `city` | +| `locationId` | int | Yes | Location ID | +| `email` | string | No | Required if unauthenticated | + +**Response:** `{ "success": true }` + +Exports only `verified >= ADMIN_APPROVED` photos for the location. + +## CSV Column Layout + +The CSV has four sections: + +### 1. Fixed columns (10) + +| Column | Source | +|--------|--------| +| `id` | `photos.id` | +| `verification` | `photos.verified` (VerificationStatus enum value) | +| `phone` | `photos.model` (device model) | +| `date_taken` | `photos.datetime` | +| `date_uploaded` | `photos.created_at` | +| `lat` | `photos.lat` | +| `lon` | `photos.lon` | +| `picked up` | Inverted from `photos.remaining` (`Yes`/`No`) | +| `address` | `photos.display_name` accessor (derived from `address_array` JSON) | +| `total_tags` | `summary.totals.litter` (fallback: `photos.total_tags`) | + +### 2. Category/object columns (~180 dynamic) + +One separator column per category (uppercase key, always null in data rows), then one column per litter object in that category. Value = aggregated quantity from `summary.tags.{catId}.{objId}.quantity`. + +``` +ALCOHOL, can, bottle, wrapper, ... +SMOKING, butts, lighters, ... +FOOD, sweetwrappers, ... +... +``` + +### 3. Materials, types, and brands columns (~76) + +| Section | Columns | Source | Format | +|---------|---------|--------|--------| +| Materials | `MATERIALS` separator + 40 material keys | `summary.tags[*].materials` (array of material IDs) | Integer quantity per material (inherits parent tag quantity), aggregated across all tags | +| Types | `TYPES` separator + 33 type keys | `summary.tags[*].type_id` | Integer quantity per type, aggregated across all tags | +| Brands | Single `brands` column | `summary.tags[*].brands` (`{brandId: qty}`) + `summary.keys.brands` for name resolution | Semicolon-delimited string: `brandname:qty;brandname:qty` | + +**Why brands are a single column:** There are 2,600+ brands in the database. One column per brand would make the CSV unusable. The delimited format is parseable with Excel `TEXTSPLIT()` or Python `str.split(';')`. + +### 4. Custom tags (3 fixed) + +| Column | Source | +|--------|--------| +| `custom_tag_1` | First custom tag key from `photo_tags.extraTags` where `tag_type = 'custom_tag'` | +| `custom_tag_2` | Second custom tag key | +| `custom_tag_3` | Third custom tag key | + +Extracted from the eager-loaded `photoTags.extraTags.extraTag` relationship. Limited to 3 columns for backward compatibility. + +## Summary JSON Structure + +The `map()` method reads from the `photos.summary` JSON (v5.1 flat array format): + +```json +{ + "tags": [ + { + "clo_id": 152, + "category_id": 16, + "object_id": 5, + "type_id": 24, + "quantity": 1, + "picked_up": true, + "materials": [5], + "brands": {"77": 1}, + "custom_tags": [321] + } + ], + "totals": { + "litter": 1, + "materials": 1, + "brands": 1, + "custom_tags": 1 + }, + "keys": { + "categories": { "16": "softdrinks" }, + "objects": { "5": "can" }, + "types": { "24": "soda" }, + "materials": { "5": "aluminium" }, + "brands": { "77": "pepsi" }, + "custom_tags": { "321": "bn:Alani Nu" } + } +} +``` + +- **Tags**: Flat array — each entry has `category_id`, `object_id`, `type_id`, `quantity` +- **Materials**: Array of material IDs `[5]` — quantity is inherited from parent tag +- **Brands**: `{ brandId: quantity }` objects with independent quantities +- **Custom tags**: Array of custom tag IDs `[321]` — quantity inherited from parent tag +- **type_id**: Present in the summary (can be null) — used for type column mapping +- **Keys**: Human-readable name lookups by ID (used for brand name resolution in CSV) + +## Date Filter Plumbing + +Both user and team exports support the same date filter contract: + +```php +$dateFilter = [ + 'column' => 'datetime', // or 'created_at' or 'updated_at' + 'fromDate' => '2025-01-01', // YYYY-MM-DD + 'toDate' => '2025-12-31', // YYYY-MM-DD +]; +``` + +- **User exports** (`ProfileController::download()`): Parses from query params. Defaults: `fromDate = 2017-01-01`, `toDate = now()`. +- **Team exports** (`TeamsController::download()`): Parses from request body. Same defaults. Whitelists `dateField` to `created_at`, `datetime`, `updated_at`. +- **`CreateCSVExport`**: Applies via `whereBetween($column, [$fromDate, $toDate])` in `query()`. + +## S3 Path Patterns + +| Scope | Path | +|-------|------| +| User (no filter) | `YYYY/MM/DD/UNIX_MyData_OpenLitterMap.csv` | +| User (date filter) | `YYYY/MM/DD/UNIX_from_DATE_to_DATE_MyData_OpenLitterMap.csv` | +| Team (no filter) | `YYYY/MM/DD/UNIX/_Team_OpenLitterMap.csv` | +| Team (date filter) | `YYYY/MM/DD/UNIX_from_DATE_to_DATE/_Team_OpenLitterMap.csv` | +| Location | `YYYY/MM/DD/UNIX/{LocationName}_OpenLitterMap.csv` | + +## Frontend Integration + +### Uploads Page (`UploadsHeader.vue`) + +- "Export CSV" button next to the "Apply" filter button +- Sends `GET /api/user/profile/download` with `dateField: 'datetime'` and the current `dateFrom`/`dateTo` filter values +- Shows inline success message: "Export started — check your email for the download link." + +### Teams Page (`TeamsHub.vue`) + +- "Export CSV" button next to the period selector in the team header +- Converts the period selector value (`today`, `week`, `month`, `year`, `all`) to `fromDate`/`toDate` date strings +- Sends `POST /api/teams/download` with `team_id` and date filter +- Uses toast notification for success/error feedback +- Only visible/functional for team leaders and school managers (server returns `not-authorized` for other members) + +### Teams Store (`stores/teams/index.js`) + +```js +async downloadTeamData(teamId, dateFilter = {}) { + await axios.post('/api/teams/download', { team_id: teamId, ...dateFilter }); +} +``` + +## Query Scoping Rules + +| Export Type | Verification Filter | Auth | Photos Included | +|-------------|-------------------|------|-----------------| +| User | None | Any authenticated user (own data) | All photos (any status) | +| Team | `verified >= ADMIN_APPROVED` | Team leader or `school_manager` only | Approved photos (ADMIN_APPROVED, BBOX_APPLIED, BBOX_VERIFIED, AI_READY) | +| Location | `verified >= ADMIN_APPROVED` | Optional (email param for guests) | Approved photos (same as above) | + +School team photos with `is_public = false` are excluded because teacher approval is required to reach `ADMIN_APPROVED`, and approval also sets `is_public = true`. + +## Timeout + +Export job timeout: 240 seconds. The `FromQuery` concern chunks automatically (1000 rows per chunk via `config/excel.php`). diff --git a/readme/changelog/2026-04-05.md b/readme/changelog/2026-04-05.md index 7d5eab4a9..9a445d917 100644 --- a/readme/changelog/2026-04-05.md +++ b/readme/changelog/2026-04-05.md @@ -1,5 +1,11 @@ # 2026-04-05 +## v5.8.1 +- fix(export): hide empty CSV columns via pre-scan, subquery-based pre-scan (no `pluck` memory bloat), single extra-tags query, DRY `scopeQuery()`, SQL injection whitelist, `picked_up` accessor, leader/school_manager auth test, filtered stats on Uploads page, error message color fix + +## v5.8.0 +- feat(export): CSV data export for users and teams — full v5 tag data (materials, types, brands), date filters, export buttons on Uploads + Teams pages, leader/school_manager auth, `verified >= ADMIN_APPROVED`, filtered stats, new `readme/ExportData.md` + ## v5.7.5 - fix(tagging): remove `.other` object extra-tag requirement — `other.other` is a valid standalone tag for "I don't know what this is" fallback; fixes mobile quickTag submission error diff --git a/readme/changelog/2026-04-06.md b/readme/changelog/2026-04-06.md new file mode 100644 index 000000000..063daeb87 --- /dev/null +++ b/readme/changelog/2026-04-06.md @@ -0,0 +1,6 @@ +# 2026-04-06 + +## v5.8.2 +- fix(photos): add shared `resolvePhotoUrl()` composable for S3/MinIO URL handling — fixes broken images in TeamPhotoList, TeamPhotoEdit, TeamApprovalQueue, ParticipantWorkspace; refactors Uploads.vue, PhotoPreview.vue, and PhotoViewer.vue to use same helper (covers FacilitatorQueue, AdminQueue, AddTags automatically) +- feat(teams): team map popup now matches global map — shows photo image, tag summary, user attribution, picked-up status, and date via `popupHelper`; images load directly via `resolvePhotoUrl` (bypasses signed-url endpoint for pending private photos) +- fix(teams): improve text readability across 14 team Vue components — bump `text-white/30`→`/50`, `text-white/40`→`/60`, `text-white/50`→`/60` (section headers), `placeholder-white/30`→`/50`, `text-slate-400`→`slate-500` (light theme), `text-gray-400`→`gray-300` (facilitator queue); skip disabled button states diff --git a/resources/js/composables/usePhotoUrl.js b/resources/js/composables/usePhotoUrl.js new file mode 100644 index 000000000..e8ba15947 --- /dev/null +++ b/resources/js/composables/usePhotoUrl.js @@ -0,0 +1,12 @@ +/** + * Resolve a photo filename to a displayable URL. + * + * In production, filename is a full S3 URL (https://olm-s3.s3...). + * In local dev with MinIO, filename is a full MinIO URL (http://localhost:9000/...). + * If filename is a relative path, prepend the current origin. + */ +export function resolvePhotoUrl(filename) { + if (!filename) return '/assets/images/waiting.png'; + if (filename.startsWith('http') || filename.startsWith('//')) return filename; + return `${window.location.origin}${filename}`; +} diff --git a/resources/js/stores/photos/index.js b/resources/js/stores/photos/index.js index dc590a9c3..34e31f34c 100644 --- a/resources/js/stores/photos/index.js +++ b/resources/js/stores/photos/index.js @@ -71,7 +71,10 @@ export const usePhotosStore = defineStore('photos', { this.loading.stats = true; this.currentFilters = { ...this.currentFilters, ...filters }; try { - await Promise.all([this.GET_USERS_PHOTOS(page, this.currentFilters), this.GET_UNTAGGED_STATS()]); + await Promise.all([ + this.GET_USERS_PHOTOS(page, this.currentFilters), + this.GET_UNTAGGED_STATS(this.currentFilters), + ]); } finally { this.loading.photos = false; this.loading.stats = false; @@ -79,16 +82,10 @@ export const usePhotosStore = defineStore('photos', { }, /** - * Just fetch photos (for pagination) + * Fetch photos and refresh stats (for filter/pagination changes) */ async fetchPhotosOnly(page = 1, filters = {}) { - this.loading.photos = true; - this.currentFilters = { ...this.currentFilters, ...filters }; - try { - await this.GET_USERS_PHOTOS(page, this.currentFilters); - } finally { - this.loading.photos = false; - } + return this.fetchUntaggedData(page, filters); }, /** diff --git a/resources/js/stores/photos/requests.js b/resources/js/stores/photos/requests.js index ce7eabad5..9241c01ff 100644 --- a/resources/js/stores/photos/requests.js +++ b/resources/js/stores/photos/requests.js @@ -94,9 +94,26 @@ export const requests = { /** * Fetch stats separately (can be cached) */ - async GET_UNTAGGED_STATS() { + async GET_UNTAGGED_STATS(filters = {}) { try { - const response = await axios.get('/api/v3/user/photos/stats'); + const params = {}; + + if (filters.tagged !== null && filters.tagged !== undefined) { + params.tagged = filters.tagged ? 1 : 0; + } + if (filters.id) { + params.id = filters.id; + params.id_operator = filters.idOperator || '='; + } + if (filters.pickedUp && filters.pickedUp !== 'all') { + params.picked_up = filters.pickedUp; + } + if (filters.tag) params.tag = filters.tag; + if (filters.customTag) params.custom_tag = filters.customTag; + if (filters.dateFrom) params.date_from = filters.dateFrom; + if (filters.dateTo) params.date_to = filters.dateTo; + + const response = await axios.get('/api/v3/user/photos/stats', { params }); this.untaggedStats = { totalPhotos: response.data.totalPhotos || 0, diff --git a/resources/js/stores/teams/index.js b/resources/js/stores/teams/index.js index ba9cba5dd..6cc51ffd5 100644 --- a/resources/js/stores/teams/index.js +++ b/resources/js/stores/teams/index.js @@ -290,11 +290,15 @@ export const useTeamsStore = defineStore('teams', { } }, - async downloadTeamData(teamId) { + async downloadTeamData(teamId, dateFilter = {}) { try { - await axios.post('/api/teams/download', { team_id: teamId }); + await axios.post('/api/teams/download', { + team_id: teamId, + ...dateFilter, + }); } catch (e) { console.error('downloadTeamData', e); + throw e; } }, diff --git a/resources/js/views/General/Tagging/v2/components/PhotoViewer.vue b/resources/js/views/General/Tagging/v2/components/PhotoViewer.vue index e811fd527..a4b754d86 100644 --- a/resources/js/views/General/Tagging/v2/components/PhotoViewer.vue +++ b/resources/js/views/General/Tagging/v2/components/PhotoViewer.vue @@ -3,7 +3,7 @@