export data, fix unsubscribe notification

CLAUDE.md

@@ -214,6 +214,7 @@ Fully deployed. 1010+ tests passing. Facilitator queue (3-panel admin-like UI fo
 - `readme/Privacy.md` — Privacy Policy source content with GDPR legal basis (Vue component renders this)
 - `readme/Twitter.md` — Automated Twitter/X commands (schedule, data sources, tweet format, Browsershot config)
 - `readme/TagSuggestions.md` — Quick tags sync API (mobile presets, bulk-replace, cross-device sync)
+- `readme/ExportData.md` — CSV data export system (user/team/location exports, column layout, date filters, S3 pipeline)
 
 ## Daily Changelog
 After every change in a session, append a one-line entry to `readme/changelog/YYYY-MM-DD.md` (create the file if it doesn't exist for today's date). Group entries by session. This is the running record of all work done each day.

app/Actions/Teams/DownloadTeamDataAction.php

@@ -10,16 +10,21 @@
 class DownloadTeamDataAction
 {
 
-    public function run(User $user, Team $team)
+    public function run(User $user, Team $team, array $dateFilter = [])
     {
         $path = now()->format('Y') .
             "/" . now()->format('m') .
             "/" . now()->format('d') .
-            "/" . now()->getTimestamp() .
-            '/_Team_OpenLitterMap.csv';  // 2020/10/25/unix/
+            "/" . now()->getTimestamp();
+
+        if (!empty($dateFilter)) {
+            $path .= '_from_' . $dateFilter['fromDate'] . '_to_' . $dateFilter['toDate'];
+        }
+
+        $path .= '/_Team_OpenLitterMap.csv';
 
         /* Dispatch job to create CSV file for export */
-        (new CreateCSVExport(null, null, $team->id))
+        (new CreateCSVExport(null, null, $team->id, null, $dateFilter))
             ->queue($path, 's3', null, ['visibility' => 'public'])
             ->chain([
                 // These jobs are executed when above is finished.

app/Console/Commands/SendEmailToSubscribed.php

@@ -37,9 +37,9 @@ public function handle(): int
 
         $userCount = User::where('emailsub', 1)->count();
 
-        // Subscribers not already in users table (avoid duplicates)
-        $userEmails = User::where('emailsub', 1)->pluck('email')->toArray();
-        $subscriberCount = Subscriber::whereNotIn('email', $userEmails)->count();
+        // Exclude ALL user emails from subscribers (users control their own sub via emailsub flag)
+        $userEmailSubquery = User::withoutGlobalScopes()->select('email');
+        $subscriberCount = Subscriber::whereNotIn('email', $userEmailSubquery)->count();
 
         $totalCount = $userCount + $subscriberCount;
 
@@ -78,7 +78,7 @@ public function handle(): int
             });
 
         // ─── Subscribers (deduplicated) ──────────────────────────────
-        Subscriber::whereNotIn('email', $userEmails)
+        Subscriber::whereNotIn('email', $userEmailSubquery)
             ->orderBy('id')
             ->chunk($chunkSize, function ($subscribers) use (&$dispatched, $bar) {
                 foreach ($subscribers as $subscriber) {

app/Exports/CreateCSVExport.php

@@ -2,7 +2,12 @@
 
 namespace App\Exports;
 
+use App\Enums\VerificationStatus;
 use App\Models\Litter\Tags\Category;
+use App\Models\Litter\Tags\LitterObjectType;
+use App\Models\Litter\Tags\Materials;
+use App\Models\Litter\Tags\PhotoTag;
+use App\Models\Litter\Tags\PhotoTagExtraTags;
 use App\Models\Photo;
 
 use Illuminate\Bus\Queueable;
@@ -29,6 +34,15 @@ class CreateCSVExport implements FromQuery, WithMapping, WithHeadings
      */
     private array $categoryObjects = [];
 
+    /** @var array<int, array{id: int, key: string}> */
+    private array $materials = [];
+
+    /** @var array<int, array{id: int, key: string}> */
+    private array $types = [];
+
+    private bool $hasBrands = false;
+    private bool $hasCustomTags = false;
+
     public $timeout = 240;
 
     public function __construct($location_type, $location_id, $team_id = null, $user_id = null, array $dateFilter = [])
@@ -39,18 +53,69 @@ public function __construct($location_type, $location_id, $team_id = null, $user
         $this->user_id = $user_id;
         $this->dateFilter = $dateFilter;
 
+        // Pre-scan: find which columns actually have data for this export scope.
+        // Use subqueries (not pluck) so MySQL optimizes internally for large exports.
+        $photoIdQuery = $this->scopeQuery(Photo::query())->select('id');
+        $tagIdQuery = PhotoTag::whereIn('photo_id', $photoIdQuery)->select('id');
+
+        $activeObjectIds = PhotoTag::whereIn('photo_id', $photoIdQuery)
+            ->whereNotNull('category_id')
+            ->whereNotNull('litter_object_id')
+            ->select('category_id', 'litter_object_id')
+            ->distinct()
+            ->get();
+
+        $activeCatIds = $activeObjectIds->pluck('category_id')->unique()->all();
+        $activeObjMap = $activeObjectIds->groupBy('category_id')
+            ->map(fn ($rows) => $rows->pluck('litter_object_id')->all())
+            ->all();
+
+        // Single query for all extra tag types
+        $extraTagTypes = PhotoTagExtraTags::whereIn('photo_tag_id', $tagIdQuery)
+            ->select('tag_type', 'tag_type_id')
+            ->distinct()
+            ->get();
+
+        $activeMaterialIds = $extraTagTypes->where('tag_type', 'material')->pluck('tag_type_id')->all();
+        $this->hasBrands = $extraTagTypes->where('tag_type', 'brand')->isNotEmpty();
+        $this->hasCustomTags = $extraTagTypes->where('tag_type', 'custom_tag')->isNotEmpty();
+
+        $activeTypeIds = PhotoTag::whereIn('photo_id', $photoIdQuery)
+            ->whereNotNull('litter_object_type_id')
+            ->distinct()
+            ->pluck('litter_object_type_id')
+            ->all();
+
+        // Load and filter category/object columns to only those with data
         $this->categoryObjects = Category::with(['litterObjects' => fn ($q) => $q->orderBy('litter_objects.id')])
+            ->whereIn('id', $activeCatIds)
             ->orderBy('id')
             ->get()
             ->map(fn ($cat) => [
                 'id' => $cat->id,
                 'key' => $cat->key,
-                'objects' => $cat->litterObjects->map(fn ($obj) => [
-                    'id' => $obj->id,
-                    'key' => $obj->key,
-                ])->values()->toArray(),
+                'objects' => $cat->litterObjects
+                    ->filter(fn ($obj) => in_array($obj->id, $activeObjMap[$cat->id] ?? []))
+                    ->map(fn ($obj) => ['id' => $obj->id, 'key' => $obj->key])
+                    ->values()
+                    ->toArray(),
             ])
+            ->filter(fn ($cat) => !empty($cat['objects']))
+            ->values()
             ->toArray();
+
+        // Filter materials and types to only those with data
+        $this->materials = !empty($activeMaterialIds)
+            ? Materials::whereIn('id', $activeMaterialIds)->orderBy('id')->get()
+                ->map(fn ($m) => ['id' => $m->id, 'key' => $m->key])
+                ->toArray()
+            : [];
+
+        $this->types = !empty($activeTypeIds)
+            ? LitterObjectType::whereIn('id', $activeTypeIds)->orderBy('id')->get()
+                ->map(fn ($t) => ['id' => $t->id, 'key' => $t->key])
+                ->toArray()
+            : [];
     }
 
     /**
@@ -81,7 +146,29 @@ public function headings(): array
             }
         }
 
-        return array_merge($result, ['custom_tag_1', 'custom_tag_2', 'custom_tag_3']);
+        if (!empty($this->materials)) {
+            $result[] = 'MATERIALS';
+            foreach ($this->materials as $material) {
+                $result[] = $material['key'];
+            }
+        }
+
+        if (!empty($this->types)) {
+            $result[] = 'TYPES';
+            foreach ($this->types as $type) {
+                $result[] = $type['key'];
+            }
+        }
+
+        if ($this->hasBrands) {
+            $result[] = 'brands';
+        }
+
+        if ($this->hasCustomTags) {
+            $result = array_merge($result, ['custom_tag_1', 'custom_tag_2', 'custom_tag_3']);
+        }
+
+        return $result;
     }
 
     /**
@@ -93,27 +180,53 @@ public function map($row): array
     {
         $result = [
             $row->id,
-            $row->verified,
+            $row->verified->value,
             $row->model,
             $row->datetime,
             $row->created_at,
             $row->lat,
             $row->lon,
-            $row->remaining ? 'No' : 'Yes',
+            $row->picked_up ? 'Yes' : 'No',
             $row->display_name,
             $row->summary['totals']['litter'] ?? $row->total_tags,
         ];
 
         $tags = $row->summary['tags'] ?? [];
+        $brandKeys = $row->summary['keys']['brands'] ?? [];
 
-        // Build lookup: category_id → object_id → quantity (from flat tags array)
+        // Single pass: iterate flat tags array
+        // Structure: [ { clo_id, category_id, object_id, type_id, quantity, materials: [id, ...], brands: {id: qty}, custom_tags: [id, ...] } ]
         $tagLookup = [];
+        $materialLookup = [];
+        $typeLookup = [];
+        $brandParts = [];
+
         foreach ($tags as $tag) {
             $catId = $tag['category_id'] ?? 0;
             $objId = $tag['object_id'] ?? 0;
-            $tagLookup[$catId][$objId] = ($tagLookup[$catId][$objId] ?? 0) + ($tag['quantity'] ?? 0);
+            $qty = $tag['quantity'] ?? 0;
+
+            $tagLookup[$catId][$objId] = ($tagLookup[$catId][$objId] ?? 0) + $qty;
+
+            // Materials: array of IDs — each gets the parent tag's quantity
+            foreach ($tag['materials'] ?? [] as $materialId) {
+                $materialLookup[$materialId] = ($materialLookup[$materialId] ?? 0) + $qty;
+            }
+
+            // Types: type_id is in the summary
+            $typeId = $tag['type_id'] ?? null;
+            if ($typeId) {
+                $typeLookup[$typeId] = ($typeLookup[$typeId] ?? 0) + $qty;
+            }
+
+            // Brands: {id: qty} objects with independent quantities
+            foreach ($tag['brands'] ?? [] as $brandId => $brandQty) {
+                $brandName = $brandKeys[$brandId] ?? "brand_{$brandId}";
+                $brandParts[$brandName] = ($brandParts[$brandName] ?? 0) + $brandQty;
+            }
         }
 
+        // Category/object columns
         foreach ($this->categoryObjects as $category) {
             $result[] = null; // category separator column
 
@@ -122,41 +235,86 @@ public function map($row): array
             }
         }
 
-        // Custom tags from extra_tags (eager-loaded in query())
-        $customTagNames = $row->photoTags
-            ->flatMap(fn ($pt) => $pt->extraTags->where('tag_type', 'custom_tag'))
-            ->take(3)
-            ->map(fn ($extra) => $extra->extraTag?->key)
-            ->values()
-            ->toArray();
+        // Materials columns (only if any exist in export scope)
+        if (!empty($this->materials)) {
+            $result[] = null; // MATERIALS separator
+            foreach ($this->materials as $material) {
+                $result[] = $materialLookup[$material['id']] ?? null;
+            }
+        }
+
+        // Types columns (only if any exist in export scope)
+        if (!empty($this->types)) {
+            $result[] = null; // TYPES separator
+            foreach ($this->types as $type) {
+                $result[] = $typeLookup[$type['id']] ?? null;
+            }
+        }
 
-        return array_merge($result, array_pad($customTagNames, 3, null));
+        // Brands: single delimited column (only if any exist in export scope)
+        if ($this->hasBrands) {
+            $result[] = !empty($brandParts)
+                ? implode(';', array_map(fn ($name, $qty) => "{$name}:{$qty}", array_keys($brandParts), array_values($brandParts)))
+                : null;
+        }
+
+        // Custom tags (only if any exist in export scope)
+        if ($this->hasCustomTags) {
+            $customTagNames = $row->photoTags
+                ->flatMap(fn ($pt) => $pt->extraTags->where('tag_type', 'custom_tag'))
+                ->take(3)
+                ->map(fn ($extra) => $extra->extraTag?->key)
+                ->values()
+                ->toArray();
+
+            $result = array_merge($result, array_pad($customTagNames, 3, null));
+        }
+
+        return $result;
     }
 
     /**
      * Create a query which we will loop over in the map function.
      */
     public function query()
     {
-        $query = Photo::with(['photoTags.extraTags.extraTag']);
+        return $this->scopeQuery(
+            Photo::with(['photoTags.extraTags.extraTag'])
+        );
+    }
 
+    /**
+     * Apply the export scope (user/team/location + date filter + verification) to a query.
+     */
+    private function scopeQuery($query)
+    {
         if (!empty($this->dateFilter)) {
-            $query->whereBetween(
-                $this->dateFilter['column'],
-                [$this->dateFilter['fromDate'], $this->dateFilter['toDate']]
-            );
+            $allowedColumns = ['created_at', 'datetime', 'updated_at'];
+            $column = in_array($this->dateFilter['column'], $allowedColumns, true)
+                ? $this->dateFilter['column']
+                : 'datetime';
+
+            $query->whereBetween($column, [
+                $this->dateFilter['fromDate'],
+                $this->dateFilter['toDate'],
+            ]);
         }
 
         if ($this->user_id) {
-            return $query->where(['user_id' => $this->user_id]);
-        } elseif ($this->team_id) {
-            return $query->where(['team_id' => $this->team_id, 'verified' => 2]);
+            return $query->where('user_id', $this->user_id);
+        }
+
+        // Team/location exports: only approved photos (ADMIN_APPROVED and above)
+        $query->where('verified', '>=', VerificationStatus::ADMIN_APPROVED->value);
+
+        if ($this->team_id) {
+            return $query->where('team_id', $this->team_id);
         } elseif ($this->location_type === 'city') {
-            return $query->where(['city_id' => $this->location_id, 'verified' => 2]);
+            return $query->where('city_id', $this->location_id);
         } elseif ($this->location_type === 'state') {
-            return $query->where(['state_id' => $this->location_id, 'verified' => 2]);
+            return $query->where('state_id', $this->location_id);
         } else {
-            return $query->where(['country_id' => $this->location_id, 'verified' => 2]);
+            return $query->where('country_id', $this->location_id);
         }
     }
 }

app/Http/Controllers/API/TeamsController.php

@@ -4,6 +4,7 @@
 
 use App\Actions\Teams\CreateTeamAction;
 use App\Actions\Teams\DownloadTeamDataAction;
+use Carbon\Carbon;
 use App\Actions\Teams\JoinTeamAction;
 use App\Actions\Teams\LeaveTeamAction;
 use App\Actions\Teams\ListTeamMembersAction;
@@ -232,7 +233,30 @@ public function download (Request $request, DownloadTeamDataAction $action): arr
             return $this->fail('not-a-member');
         }
 
-        $action->run($user, $team);
+        if (!$team->isLeader($user->id) && !$user->hasRole('school_manager')) {
+            return $this->fail('not-authorized');
+        }
+
+        $dateFilter = [];
+        if ($request->dateField && ($request->fromDate || $request->toDate)) {
+            try {
+                $dateFilter = [
+                    'column' => in_array($request->dateField, ['created_at', 'datetime', 'updated_at'], true)
+                        ? $request->dateField
+                        : 'datetime',
+                    'fromDate' => $request->fromDate
+                        ? Carbon::parse($request->fromDate)->toDateString()
+                        : '2017-01-01',
+                    'toDate' => $request->toDate
+                        ? Carbon::parse($request->toDate)->toDateString()
+                        : now()->toDateString(),
+                ];
+            } catch (\Exception) {
+                // Invalid date format — ignore filter and export all data
+            }
+        }
+
+        $action->run($user, $team, $dateFilter);
 
         return $this->success();
     }