make-screenshots script

.gitignore

@@ -1,3 +1,4 @@
 config.php
 public/components
+public/screenshots
 vendor

classes/model/feed/marks.php

@@ -83,7 +83,7 @@ function ids_and_ts($redis_key, $query, $params = [])
   function prepare_items($results, $total = null, $params = [])
   {
     # Next?
-    $next = $params['limit'] && count($results) > $params['limit'] ? (int) array_pop($results) : null;
+    $next = !empty($params['limit']) && count($results) > $params['limit'] ? (int) array_pop($results) : null;
     # Items
     $items = empty($results) ? [] : $this->table('marks')->get(array_keys($results));
     # Result

classes/model/table/screenshots.php

@@ -9,6 +9,11 @@ class screenshots extends table
 
   public $tablename = 'bm_screenshots';
 
+  function create($set = [])
+  {
+    return parent::create(['created' => db::now()] + $set);
+  }
+
   function preload_for_marks($marks)
   {
     # Get list of link ids for which we need to retrieve screenshots
@@ -46,7 +51,7 @@ function ensure_entry_exists_for_mark($mark)
     $params = ['link' => $mark->link_id];
     $existing = $this->where($params)->and_where("created > DATE_SUB('$now', INTERVAL 1 DAY)");
     if ($existing->count() == 0) {
-      $screenshot = $this->create($params + ['status' => 0, 'created' => $now]);
+      $screenshot = $this->create($params + ['status' => 0]);
       $this->service('amqp')->push(['id' => $screenshot->id], 'take_screenshot');
     }
   }

scripts/make-screenshots.script.php

@@ -0,0 +1,85 @@
+#!/usr/bin/env php
+<?php
+
+use
+amateur\model\cache,
+amateur\model\db,
+amateur\model\runtime,
+amateur\model\table;
+
+require_once dirname(__DIR__) . '/bootstrap.php';
+
+replaceable('request_host', function () { return 'localhost:8002'; });
+
+$screenshot_extension = 'png';
+$screenshot_base_dir = root_dir . '/public/screenshots/';
+$screenshot_base_url = absolute_url('/screenshots/');
+
+$user = table('users')->get_one('login', 'znarf');
+$marks = model('marks')->from_user($user, ['limit' => 1000]);
+
+foreach ($marks['items'] as $mark) {
+  error_log("Checking {$mark->url}");
+
+  $screenshot = table('screenshots')->for_mark($mark);
+  if (!empty($screenshot['url'])) {
+    error_log("Existing {$screenshot['url']}");
+    continue;
+  }
+
+  $parsed_url = parse_url($mark->url);
+  if (empty($parsed_url['host'])) {
+    continue;
+  }
+
+  $folder = date("Y/m/d/" , time() - date('Z') );
+  $relative = $folder . md5($mark->url) . '.' . $bm_screenshot_extension;
+  $screenshot_absolute_path = $screenshot_base_dir . $relative;
+  $screenshot_absolute_url = $screenshot_base_url . $relative;
+
+  if (!file_exists($screenshot_absolute_path)) {
+    $parameters = [
+      'url'    => $parsed_url['host'],
+      'width'  => 112,
+      'height' => 83,
+      'token'  => flag('miniature_api_key'),
+    ];
+    $screenshot_api_url = 'https://api.miniature.io/' . '?' . http_build_query($parameters);
+
+    // $parameters = [
+    //   'url'    => $mark->url,
+    //   'width'  => 112,
+    // ];
+
+    // $screenshot_api_url = 'https://api.thumbnail.ws/api/ab264f9ceea81bfaa0803a670001777f487268a1f197/thumbnail/get' . '?' . http_build_query($parameters);
+
+    error_log("Asking {$screenshot_api_url}");
+    $image = file_get_contents($screenshot_api_url);
+    if (!$image) {
+      continue;
+    }
+
+    $size = strlen($image);
+    error_log("Size {$size}");
+    if ($size <= 1252 || $size === 2185 || $size === 1857) {
+       error_log("Suspicious size");
+       continue;
+    }
+
+    if (!is_dir($screenshot_dir . $folder)) {
+      error_log("Creating {$folder}");
+      mkdir($screenshot_dir . $folder, 0777, true);
+    }
+
+    error_log("Writing {$screenshot_absolute_path}");
+    file_put_contents($screenshot_absolute_path, $image);
+    chmod($screenshot_absolute_path, 0777);
+  }
+
+  $params = ['status' => 1, 'generated' => db::now(), 'url' => $screenshot_absolute_url];
+  if ($screenshot) {
+    $screenshot->update($params);
+  } else {
+    table('screenshots')->create($params + ['link' => $mark->link_id]);
+  }
+}