Add Mel::Postgres adapter

.github/workflows/test.yml

@@ -39,14 +39,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        crystal: ['1.0.0']
+        cockroachdb: ['v22.2.0', latest]
+        crystal: ['1.4.0']
         experimental: [false]
         shard_file: [shard.yml]
         include:
-          - crystal: latest
+          - cockroachdb: latest
+            crystal: latest
             experimental: false
             shard_file: shard.latest.yml
-          - crystal: nightly
+          - cockroachdb: latest
+            crystal: nightly
             experimental: true
             shard_file: shard.edge.yml
     runs-on: ubuntu-latest
@@ -74,7 +77,32 @@ jobs:
           sudo apt -y install redis
       - name: Start Redis
         run: sudo systemctl start redis
+      - name: Install Postgresql
+        run: |
+          sudo apt update
+          sudo apt -y install postgresql
+      - name: Start Postgresql
+        run: sudo systemctl start postgresql
+      - name: Set Postgres password
+        run: >-
+          sudo -u postgres psql -c
+          "ALTER USER postgres WITH PASSWORD 'password';"
+      - name: Install CockroachDB
+        run: |
+          sudo apt update
+          sudo apt -y install tar wget
+          sudo mkdir -p /usr/local/lib/cockroach
+          wget -O cockroachdb.tgz https://binaries.cockroachdb.com/cockroach-${{ matrix.cockroachdb }}.linux-amd64.tgz
+          tar -xzf cockroachdb.tgz
+          sudo cp -f cockroach-*/cockroach /usr/local/bin/
+          sudo chmod +x /usr/local/bin/cockroach
+          sudo cp -rf cockroach-*/lib/* /usr/local/lib/cockroach/
+        working-directory: /tmp
+      - name: Start CockroachDB
+        run: cockroach start-single-node --insecure --listen-addr=localhost:36257 --sql-addr=localhost:26257 --background
       - name: Run tests
         env:
+          COCKROACH_URL: postgres://root@localhost:26257/mel_spec?sslmode=disable
+          POSTGRES_URL: postgres://postgres:password@localhost:5432/mel_spec
           REDIS_URL: redis://localhost:6379/0
         run: crystal spec --error-on-warnings -Dpreview_mt

CHANGELOG.md

@@ -8,8 +8,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased] - 
 
 ### Added
+- Add `Mel::Postgres` adapter
 - Add `#be_enqueued` spec expectation
 
+### Changed
+- Bump minimum required Crystal version to 1.4
+
 ### Changed
 - Move Lua script into its own file
 

README.md

@@ -28,6 +28,8 @@ This makes the storage backend the *source of truth* for schedules, allowing to
        github: GrottoPress/mel
      #redis: # Uncomment if using the Redis backend
      #  github: jgaskins/redis
+     #pg: # Uncomment if using the Posgres backend
+     #  github: will/crystal-pg
    ```
 
 1. Run `shards update`
@@ -75,6 +77,29 @@ This makes the storage backend the *source of truth* for schedules, allowing to
      # ...
      ```
 
+   - Using the Postgres backend
+
+     ```crystal
+     # ->>> src/app/config.cr
+
+     # ...
+
+     require "mel/postgres"
+
+     db_url = "postgres://username:password@localhost:5432/database_name"
+
+     # Uncomment to create database
+     #Mel::Postgres.create_database(db_url)
+
+     Mel.configure do |settings|
+       # ...
+       settings.store = Mel::Postgres.new(db_url, namespace: "mel")
+       # ...
+     end
+
+     # ...
+     ```
+
    - Using the Memory backend (Not for production use)
 
      ```crystal
@@ -800,8 +825,10 @@ If it is a negative integer `-N`  (other than `-1`), the number of due tasks pul
 Create a `.env.sh` file:
 
 ```bash
-#!/bin/bash
+#!/usr/bin/env bash
 
+export COCKROACH_URL='postgres://root@localhost:26257/mel_spec?sslmode=disable'
+export POSTGRES_URL='postgres://postgres:password@localhost:5432/mel_spec'
 export REDIS_URL='redis://localhost:6379/0'
 ```
 

shard.yml

@@ -8,7 +8,7 @@ authors:
 
 license: MIT
 
-crystal: ~> 1.0
+crystal: ~> 1.4
 
 dependencies:
   cron_parser:
@@ -25,6 +25,9 @@ development_dependencies:
   carbon:
     github: luckyframework/carbon
     version: ~> 0.2.0
+  pg:
+    github: will/crystal-pg
+    version: ~> 0.29.0
   redis:
     github: jgaskins/redis
     version: ~> 0.8.0

spec/setup/worker.cr

@@ -4,6 +4,9 @@ Mel.configure do |settings|
   settings.timezone = Time::Location.load("America/Los_Angeles")
 end
 
+Mel::Postgres.create_database(ENV["COCKROACH_URL"])
+Mel::Postgres.create_database(ENV["POSTGRES_URL"])
+
 tasks = ->do
   Mel.stop
   Mel::RunPool.delete
@@ -14,7 +17,12 @@ end
 Spec.around_each do |spec|
   next spec.run if all_tags(spec.example).includes?("skip_around_each")
 
-  {Mel::Memory.new, Mel::Redis.new(ENV["REDIS_URL"])}.each do |store|
+  {
+    Mel::Memory.new,
+    Mel::Postgres.new(ENV["COCKROACH_URL"]),
+    Mel::Postgres.new(ENV["POSTGRES_URL"]),
+    Mel::Redis.new(ENV["REDIS_URL"])
+  }.each do |store|
     Mel.settings.store = store
     tasks.call
     spec.run

spec/spec_helper.cr

@@ -4,10 +4,11 @@ require "log/spec"
 require "carbon"
 require "timecop"
 
+require "../src/redis"
+require "../src/postgres"
 require "../src/spec"
 require "./setup/**"
 require "./support/**"
 require "../src/carbon"
-require "../src/redis"
 
 include Carbon::Expectations

src/mel/memory.cr

@@ -118,8 +118,8 @@ module Mel
     private def query(count, delete, time = nil)
       queue = sorted_queue.select do |_, value|
         if delete.nil?
-          next orphan_score <= value <= time.to_unix if time
-          next value >= orphan_score
+          next orphan_timestamp <= value <= time.to_unix if time
+          next value >= orphan_timestamp
         end
 
         next 0 <= value <= time.to_unix if time
@@ -138,7 +138,7 @@ module Mel
     end
 
     private def to_running(ids)
-      ids.each { |id| queue[id] = running_score }
+      ids.each { |id| queue[id] = running_timestamp }
       ids
     end
 

src/mel/run_pool.cr

@@ -1,8 +1,8 @@
 module Mel
-  # This saves a worker's currently running tasks in memory.
+  # Saves a worker's currently running tasks in memory.
   #
-  # Every time a worker polls for tasks, it updates the timestamp (score) of
-  # its running tasks in the store. The Pool is how it knows which tasks to
+  # Every time a worker polls for tasks, it updates the timestamp of
+  # its running tasks in the store. This pool is how it knows which tasks to
   # update.
   module RunPool
     extend self

src/mel/store.cr

@@ -89,19 +89,22 @@ module Mel
           transaction &.set_progress(id, value, description)
       end
 
-      # We assume a task is orphaned if its score has not been updated after
-      # 3 polls.
-      private def orphan_after
-        poll_interval = Mel.settings.poll_interval
-        {poll_interval * 3, poll_interval + 1.second}.max
+      private def orphan_timestamp
+        -orphan_after.ago.to_unix
       end
 
-      private def running_score
+      private def running_timestamp
         -Time.local.to_unix
       end
 
-      private def orphan_score
-        -orphan_after.ago.to_unix
+      # A task is assumed to be orphaned if its timestamp has not been updated
+      # after 3 polls.
+      #
+      # This must never be less than 1 second, since timestamps are saved in
+      # seconds.
+      private def orphan_after
+        poll_interval = Mel.settings.poll_interval
+        {poll_interval * 3, poll_interval + 1.second}.max
       end
     end
   end