From 07d7cb534e04c1de01e5cee3c3752c0b6b41ee61 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sat, 17 Jun 2017 12:38:37 -0700 Subject: [PATCH 01/12] Add parameters to capture extra keywords from response --- lib/backblaze/b2/bucket.rb | 3 ++- lib/backblaze/b2/file.rb | 3 +++ lib/backblaze/b2/file_version.rb | 10 ++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/backblaze/b2/bucket.rb b/lib/backblaze/b2/bucket.rb index d081804..bb823f9 100644 --- a/lib/backblaze/b2/bucket.rb +++ b/lib/backblaze/b2/bucket.rb @@ -9,7 +9,8 @@ class Bucket < Base # @param [#to_s] bucket_id the bucket id # @param [#to_s] bucket_type the bucket publicity type # @param [#to_s] account_id the account to which this bucket belongs - def initialize(bucket_name:, bucket_id:, bucket_type:, account_id:, cache: false) + def initialize(bucket_name:, bucket_id:, bucket_type:, account_id:, cache: false, + bucket_info: {}, lifecycle_rules: [], revision: nil) @bucket_name = bucket_name @bucket_id = bucket_id @bucket_type = bucket_type diff --git a/lib/backblaze/b2/file.rb b/lib/backblaze/b2/file.rb index 23c7c79..706d9c7 100644 --- a/lib/backblaze/b2/file.rb +++ b/lib/backblaze/b2/file.rb @@ -89,6 +89,9 @@ def create(data:, bucket:, name: nil, base_name: '', content_type: 'b2/x-auto', size: response['contentLength'], file_id: response['fileId'], upload_timestamp: Time.now.to_i * 1000, + content_length: data.size, + content_type: content_type, + content_sha1: digest, action: 'upload' } diff --git a/lib/backblaze/b2/file_version.rb b/lib/backblaze/b2/file_version.rb index 43cdfc3..5192995 100644 --- a/lib/backblaze/b2/file_version.rb +++ b/lib/backblaze/b2/file_version.rb @@ -1,12 +1,18 @@ module Backblaze::B2 class FileVersion < Base - attr_reader :file_id, :size, :action, :upload_timestamp, :file_name + attr_reader :file_id, :size, :action, :upload_timestamp, :file_name, + :content_length, :content_sha1, :content_type, :file_info - def initialize(file_id:, size:, upload_timestamp:, action:, file_name:) + def initialize(file_id:, size:, upload_timestamp:, action:, file_name:, + content_length:, content_sha1:, content_type:, file_info: {}) @file_id = file_id @size = size @action = action @file_name = file_name + @content_length = content_length + @content_sha1 = content_sha1 + @content_type = content_type + @file_info = file_info @upload_timestamp = Time.at(upload_timestamp / 1000.0) end From 7fad4d47119536b45b625fa4a5dc0941426293a0 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sat, 17 Jun 2017 13:05:59 -0700 Subject: [PATCH 02/12] Add method to get bucket by bucket name --- lib/backblaze/b2/bucket.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/backblaze/b2/bucket.rb b/lib/backblaze/b2/bucket.rb index bb823f9..34e1be4 100644 --- a/lib/backblaze/b2/bucket.rb +++ b/lib/backblaze/b2/bucket.rb @@ -152,6 +152,10 @@ def buckets new(params) end end + + def get_bucket(name:) + buckets.find { |b| b.bucket_name == name } + end end end end From d29166c446eeec9975feacf01fa3701acbe003ee Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sat, 17 Jun 2017 13:09:49 -0700 Subject: [PATCH 03/12] Alias bucket_name method to name --- lib/backblaze/b2/bucket.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/backblaze/b2/bucket.rb b/lib/backblaze/b2/bucket.rb index 34e1be4..9717601 100644 --- a/lib/backblaze/b2/bucket.rb +++ b/lib/backblaze/b2/bucket.rb @@ -22,6 +22,8 @@ def bucket_name @bucket_name end + alias_method :name, :bucket_name + # @return [String] bucket id def bucket_id @bucket_id From c837517b6a16971fff79b95b8edf3bb32aefe826 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sun, 18 Jun 2017 17:39:49 -0700 Subject: [PATCH 04/12] Add first_file keyword parameter to Bucket#file_names --- lib/backblaze/b2/bucket.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/backblaze/b2/bucket.rb b/lib/backblaze/b2/bucket.rb index 9717601..8aac94b 100644 --- a/lib/backblaze/b2/bucket.rb +++ b/lib/backblaze/b2/bucket.rb @@ -57,6 +57,7 @@ def ==(other) ## # Lists all files that are in the bucket. This is the basic building block for the search. + # @param [String] first_file first file in the bucket to start listing from # @param [Integer] limit max number of files to retreive. Set to `-1` to get all files. # This is not exact as it mainly just throws the limit into max param on the request # so it will try to grab at least `limit` files, unless there aren't enoungh in the bucket @@ -67,7 +68,7 @@ def ==(other) # @return [Array] when convert is true # @return [Array] when convert is false # @note many of these methods are for the recusion - def file_names(limit: 100, cache: false, convert: true, double_check_server: false) + def file_names(first_file: nil, limit: 100, cache: false, convert: true, double_check_server: false) if cache && !@file_name_cache.nil? if limit <= @file_name_cache[:limit] && convert == @file_name_cache[:convert] return @file_name_cache[:files] @@ -75,7 +76,7 @@ def file_names(limit: 100, cache: false, convert: true, double_check_server: fal end retreive_count = (double_check_server ? 0 : -1) - files = file_list(bucket_id: bucket_id, limit: limit, retreived: retreive_count, first_file: nil, start_field: 'startFileName'.freeze) + files = file_list(bucket_id: bucket_id, limit: limit, retreived: retreive_count, first_file: first_file, start_field: 'startFileName'.freeze) merge_params = {bucket_id: bucket_id} files.map! do |f| From 9c9dd050be1207eff47c7f2c5f6d0bbea30c5c80 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sun, 18 Jun 2017 17:43:52 -0700 Subject: [PATCH 05/12] Add support for Paperclip::UploadedFileAdapter --- lib/backblaze/b2/file.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/backblaze/b2/file.rb b/lib/backblaze/b2/file.rb index 706d9c7..5b4bd65 100644 --- a/lib/backblaze/b2/file.rb +++ b/lib/backblaze/b2/file.rb @@ -30,7 +30,7 @@ def create(data:, bucket:, name: nil, base_name: '', content_type: 'b2/x-auto', when String data.force_encoding('ASCII-8BIT') raise ArgumentError.new('Must provide a file name for data') if name.nil? - when ::File, Tempfile + when ::File, Tempfile, ::Paperclip::UploadedFileAdapter data.binmode data.rewind if name.nil? @@ -63,6 +63,10 @@ def create(data:, bucket:, name: nil, base_name: '', content_type: 'b2/x-auto', if data.is_a? String digest.update(data) req.body = data + elsif data.is_a? ::Paperclip::UploadedFileAdapter + digest.file(data.path) + data.rewind + req.body_stream = data else digest.file(data) data.rewind From a6140886c54e12ec6dd10b2daa6d2b23447f5801 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sun, 18 Jun 2017 17:44:50 -0700 Subject: [PATCH 06/12] Implement paperclip adapter for backblaze storage --- lib/backblaze.rb | 1 + lib/paperclip/storage/backblaze.rb | 87 ++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 lib/paperclip/storage/backblaze.rb diff --git a/lib/backblaze.rb b/lib/backblaze.rb index cd7b4f6..acede7d 100644 --- a/lib/backblaze.rb +++ b/lib/backblaze.rb @@ -4,6 +4,7 @@ require "backblaze/utils" require "backblaze/errors" require "backblaze/b2" +require "paperclip/storage/backblaze" module Backblaze end diff --git a/lib/paperclip/storage/backblaze.rb b/lib/paperclip/storage/backblaze.rb new file mode 100644 index 0000000..22d43cc --- /dev/null +++ b/lib/paperclip/storage/backblaze.rb @@ -0,0 +1,87 @@ +module Paperclip + module Storage + module Backblaze + def self.extended base + base.instance_eval do + login + unless @options[:url].match(/\Ab2.*url\z/) + @options[:url] = ":b2_path_url".freeze + end + end + + Paperclip.interpolates(:b2_path_url) do |attachment, style| + "#{attachment.b2_protocol(style, true)}//#{attachment.b2_host_name}/#{attachment.b2_bucket_name}/#{attachment.path(style).sub(%r{\A/}, '')}" + end unless Paperclip::Interpolations.respond_to? :b2_path_url + end + + def b2_protocol(style, with_colon = true) + with_colon ? "https:" : "https" + end + + def b2_host_name + "f001.backblazeb2.com/file" + end + + def b2_credentials(filename = @options[:b2_credentials]) + unless @b2_credentials + require 'psych' + File.open(filename, 'r') do |f| + @b2_credentials = Psych.load(f.read).symbolize_keys + end + end + @b2_credentials + end + + def login + return if ::Backblaze::B2.token + creds = b2_credentials + ::Backblaze::B2.login(account_id: creds[:account_id], application_key: creds[:application_key]) + end + + def b2_bucket + @b2_bucket ||= ::Backblaze::B2::Bucket.get_bucket(name: @options[:b2_bucket]) + end + + def b2_bucket_name + b2_bucket.bucket_name + end + + def exists?(style = default_style) + !!get_file(filename: path(style).sub(%r{\A/}, "")) + end + + def get_file(filename:) + b2_bucket.file_names(first_file: filename, limit: 1).find do |f| + f.file_name == filename + end + end + + def flush_writes + @queued_for_write.each do |style, file| + base_name = ::File.dirname(path(style)).sub(%{\A/}, "") + name = ::File.basename(path(style)) + ::Backblaze::B2::File.create data: file, bucket: b2_bucket, name: name, base_name: base_name + end + @queued_for_write = {} + end + + def flush_deletes + @queued_for_delete.each do |path| + if file = get_file(filename: path.sub(%r{\A/}, '')) + file.destroy! + end + end + @queued_for_delete = [] + end + + def copy_to_local_file(style, local_dest_path) + ::File.open(local_dest_path, 'wb') do |local_file| + file = get_file(filename: path(style).sub(%r{\A/}, '')) + body = file.get(file.latest.download_url, parse: :plain) + local_file.write(body) + end + end + + end # module Backblaze + end # module Storage +end # module Paperclip From 322df7c385dd51a0e419e0badb11f8c6c89dadde Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sun, 18 Jun 2017 20:14:30 -0700 Subject: [PATCH 07/12] Add documentation --- lib/paperclip/storage/backblaze.rb | 75 ++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 13 deletions(-) diff --git a/lib/paperclip/storage/backblaze.rb b/lib/paperclip/storage/backblaze.rb index 22d43cc..49030c6 100644 --- a/lib/paperclip/storage/backblaze.rb +++ b/lib/paperclip/storage/backblaze.rb @@ -1,5 +1,34 @@ module Paperclip module Storage + # Defines a adapter to use store Paperclip attachments to Backblaze B2 + # Cloud Storage, which is similar to Amazon's S3 service. + # + # This allows you to use has_attached_file in your models + # with :storage => :backblaze. + # + # Some required options include: + # + # :storage - This should be set to :backblaze in order to use this + # storage adapter. + # + # :b2_credentials - This should point to a YAML file containing your B2 + # account ID and application key. The contents should look something + # like: + # + # account_id: 123456789abc + # application_key: 0123456789abcdef0123456789abcdef0123456789 + # + # :b2_bucket - This should name the bucket to save files to. + # + # So for example, a model might be configured something like this: + # + # class Note < ApplicationRecord + # has_attached_file :image, + # storage: :backblaze, + # b2_credentials: Rails.root.join('config/b2.yml'), + # b2_bucket: 'bucket_for_my_app' + # ... + # module Backblaze def self.extended base base.instance_eval do @@ -10,18 +39,21 @@ def self.extended base end Paperclip.interpolates(:b2_path_url) do |attachment, style| - "#{attachment.b2_protocol(style, true)}//#{attachment.b2_host_name}/#{attachment.b2_bucket_name}/#{attachment.path(style).sub(%r{\A/}, '')}" + "#{::Backblaze::B2.download_url}/file/#{attachment.b2_bucket_name}/#{attachment.path(style).sub(%r{\A/}, '')}" end unless Paperclip::Interpolations.respond_to? :b2_path_url end - def b2_protocol(style, with_colon = true) - with_colon ? "https:" : "https" - end - - def b2_host_name - "f001.backblazeb2.com/file" - end - + # Fetch the credentials from the config file, if it hasn't already been + # loaded. + # + # filename : String - path to the YAML config file containing the + # Backblaze B2 credentials. Here is example contents of what one + # may look like: + # + # account_id: 123456789abc + # application_key: 0123456789abcdef0123456789abcdef0123456789 + # + # Returns a Hash containing the parsed credentials. def b2_credentials(filename = @options[:b2_credentials]) unless @b2_credentials require 'psych' @@ -32,39 +64,55 @@ def b2_credentials(filename = @options[:b2_credentials]) @b2_credentials end + # Authenticate with Backblaze with the account ID and secret key. This + # also caches several variables from the response related to the API, so + # it is important that it is executed at the very beginning. def login return if ::Backblaze::B2.token creds = b2_credentials ::Backblaze::B2.login(account_id: creds[:account_id], application_key: creds[:application_key]) end + # Return the Backblaze::B2::Bucket object representing the bucket + # specified by the required options[:b2_bucket]. def b2_bucket @b2_bucket ||= ::Backblaze::B2::Bucket.get_bucket(name: @options[:b2_bucket]) end + # Return the specified bucket name as a String. def b2_bucket_name b2_bucket.bucket_name end + # Return whether this attachment exists in the bucket. def exists?(style = default_style) - !!get_file(filename: path(style).sub(%r{\A/}, "")) + !!get_file(filename: get_path(style)) end + # Return a Backblaze::B2::File object representing the file named in the + # filename keyword, if it exists. def get_file(filename:) b2_bucket.file_names(first_file: filename, limit: 1).find do |f| f.file_name == filename end end + # Return this attachment's bucket file path as a String. + def get_path(style = default_style) + path(style).sub(%r{\A/}, '') + end + + # (Internal) Used by Paperclip to upload local files to storage. def flush_writes @queued_for_write.each do |style, file| - base_name = ::File.dirname(path(style)).sub(%{\A/}, "") - name = ::File.basename(path(style)) + base_name = ::File.dirname(get_path(style)) + name = ::File.basename(get_path(style)) ::Backblaze::B2::File.create data: file, bucket: b2_bucket, name: name, base_name: base_name end @queued_for_write = {} end + # (Internal) Used by Paperclip to remove remote files from storage. def flush_deletes @queued_for_delete.each do |path| if file = get_file(filename: path.sub(%r{\A/}, '')) @@ -74,9 +122,10 @@ def flush_deletes @queued_for_delete = [] end + # (Internal) def copy_to_local_file(style, local_dest_path) ::File.open(local_dest_path, 'wb') do |local_file| - file = get_file(filename: path(style).sub(%r{\A/}, '')) + file = get_file(filename: get_path(style)) body = file.get(file.latest.download_url, parse: :plain) local_file.write(body) end From 362e364ad36faec78107b12af839324a3924be75 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sun, 18 Jun 2017 21:07:49 -0700 Subject: [PATCH 08/12] Rebrand gem --- LICENSE.txt | 2 +- README.md | 61 +++++++++++++++---- lib/backblaze.rb | 1 - lib/paperclip/backblaze.rb | 4 ++ lib/paperclip/backblaze/version.rb | 5 ++ ...aze.gemspec => paperclip-backblaze.gemspec | 16 ++--- 6 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 lib/paperclip/backblaze.rb create mode 100644 lib/paperclip/backblaze/version.rb rename backblaze.gemspec => paperclip-backblaze.gemspec (59%) diff --git a/LICENSE.txt b/LICENSE.txt index c2e0b70..b0a000a 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Winston Durand +Copyright (c) 2017 Alex Tsui Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index b930987..8ee58ff 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,74 @@ -# Backblaze +# paperclip-backblaze -The Backblaze ruby gem is an implementation of the [Backblaze B2 Cloud Storage API](https://www.backblaze.com/b2/docs/). In addition to simplifying calls, it also implements an object oriented structure for dealing with files. Calling the api through different objects will not cause each to get updated. Always assume that data retrieved is just a snapshot from when the object was retrieved. +The `paperclip-backblaze` provides a [Paperclip](https://github.com/thoughtbot/paperclip) storage adapter so that +attachments can be saved to [Backblaze B2 Cloud Storage API](https://www.backblaze.com/b2/docs/). +It makes use of Winston Durand's [backblaze](https://github.com/R167/backblaze) gem +to access the B2 API behind the scenes. + +Backblaze B2 Cloud Storage is similar to Amazon's AWS S3 Storage, but it has a few selling points: + +1. They run their own hardware (that's open-sourced, including the schematics, and drive reports) +2. It's $0.005/GB/month vs S3's $0.030/GB/month +3. You actually get a free GB of bandwidth a day, so it might be nice for personal projects. ## Installation Add this line to your application's Gemfile: ```ruby -gem 'backblaze' +gem 'paperclip-backblaze' ``` - And then execute: $ bundle -Or install it yourself as: +## Usage - $ gem install backblaze +You should be familiar with configuring Paperclip attachments for your model. +If not, please start with the Paperclip documentation +[here](https://github.com/thoughtbot/paperclip#usage). + +Configuring Backblaze storage is very similar to [configuring S3 storage](http://www.rubydoc.info/gems/paperclip/Paperclip/Storage/S3). +Let's suppose we have a `Note` model with an `image` attachment that we would +like to be backed by Backblaze storage. In the model, it might be configured +like this: + +```.rb +# app/models/note.rb +class Note < ApplicationRecord + has_attached_file :image, + storage: :backblaze, + b2_credentials: Rails.root.join('config/b2.yml'), + b2_bucket: 'bucket_for_my_app' + ... +``` -## Usage +```.yml +# config/b2.yml +account_id: 123456789abc +application_key: 0123456789abcdef0123456789abcdef0123456789 +``` -TODO: Write usage instructions here +Currently, these are required options: -## Development +- `:storage` - This should be set to :backblaze in order to use this + storage adapter. -After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. +- `:b2_credentials` - This should point to a YAML file containing your B2 + account ID and application key. The contents should look something + like `b2.yml` above. -To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). +- `:b2_bucket` - This should name the bucket to save files to. ## Contributing -Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/backblaze. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct. +This started as a proof of concept for a hobby project, so there's lots of room +for improvement, and it would be great to have your help. +Bug reports and pull requests are welcome on GitHub at +https://github.com/alextsui05/paperclip-backblaze. This project is intended to be a safe, +welcoming space for collaboration, and contributors are expected to adhere to +the [Contributor Covenant](contributor-covenant.org) code of conduct. ## License diff --git a/lib/backblaze.rb b/lib/backblaze.rb index acede7d..cd7b4f6 100644 --- a/lib/backblaze.rb +++ b/lib/backblaze.rb @@ -4,7 +4,6 @@ require "backblaze/utils" require "backblaze/errors" require "backblaze/b2" -require "paperclip/storage/backblaze" module Backblaze end diff --git a/lib/paperclip/backblaze.rb b/lib/paperclip/backblaze.rb new file mode 100644 index 0000000..2375dfe --- /dev/null +++ b/lib/paperclip/backblaze.rb @@ -0,0 +1,4 @@ +require_relative '../backblaze' + +require "paperclip/backblaze/version" +require "paperclip/storage/backblaze" diff --git a/lib/paperclip/backblaze/version.rb b/lib/paperclip/backblaze/version.rb new file mode 100644 index 0000000..b122a0b --- /dev/null +++ b/lib/paperclip/backblaze/version.rb @@ -0,0 +1,5 @@ +module Paperclip + module Backblaze + VERSION = "0.1.0" + end +end diff --git a/backblaze.gemspec b/paperclip-backblaze.gemspec similarity index 59% rename from backblaze.gemspec rename to paperclip-backblaze.gemspec index c92b4fd..0907586 100644 --- a/backblaze.gemspec +++ b/paperclip-backblaze.gemspec @@ -1,17 +1,17 @@ # coding: utf-8 lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'backblaze/version' +require 'paperclip/backblaze/version' Gem::Specification.new do |spec| - spec.name = "backblaze" - spec.version = Backblaze::VERSION - spec.authors = ["Winston Durand"] - spec.email = ["me@winstondurand.com"] + spec.name = "paperclip-backblaze" + spec.version = Paperclip::Backblaze::VERSION + spec.authors = ["Alex Tsui", "Winston Durand"] + spec.email = ["alextsui05@gmail.com"] - spec.summary = %q{Interface for teh Backblaze B2 Cloud} - spec.description = %q{Intended to offer a way to interact with Backblaze B2 Cloud Storage without touching the API directly.} - spec.homepage = "https://github.com/R167/backblaze" + spec.summary = %q{Paperclip storage adapter for Backblaze B2 Cloud.} + spec.description = %q{Allows Paperclip attachments to be backed by Backblaze B2 Cloud storage as an alternative to AWS S3.} + spec.homepage = "https://github.com/alextsui05/paperclip-backblaze" spec.license = "MIT" spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } From 2d23fd014280dc6482b68f7e85e7d5720c0d0606 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sun, 18 Jun 2017 22:29:00 -0700 Subject: [PATCH 09/12] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ee58ff..ece271c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Backblaze B2 Cloud Storage is similar to Amazon's AWS S3 Storage, but it has a f Add this line to your application's Gemfile: ```ruby -gem 'paperclip-backblaze' +gem 'paperclip-backblaze', github: 'alextsui05/paperclip-backblaze' ``` And then execute: From bdbf25e610f4cde44122cd467bfd0339250f8fdd Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sun, 23 Jul 2017 12:59:20 -0700 Subject: [PATCH 10/12] Update to 0.2.0 Remove :b2_bucket option, replace with `bucket` key in the credentials file --- README.md | 52 ++++++++++++++++++++++-------- lib/paperclip/backblaze/version.rb | 2 +- lib/paperclip/storage/backblaze.rb | 21 ++++++++---- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ece271c..ef43bce 100644 --- a/README.md +++ b/README.md @@ -30,23 +30,38 @@ If not, please start with the Paperclip documentation Configuring Backblaze storage is very similar to [configuring S3 storage](http://www.rubydoc.info/gems/paperclip/Paperclip/Storage/S3). Let's suppose we have a `Note` model with an `image` attachment that we would -like to be backed by Backblaze storage. In the model, it might be configured -like this: +like to be backed by Backblaze storage. -```.rb -# app/models/note.rb -class Note < ApplicationRecord - has_attached_file :image, - storage: :backblaze, - b2_credentials: Rails.root.join('config/b2.yml'), - b2_bucket: 'bucket_for_my_app' - ... -``` + + +First, put your credentials and bucket name in a YML file. ```.yml # config/b2.yml account_id: 123456789abc application_key: 0123456789abcdef0123456789abcdef0123456789 +bucket: my_b2_bucket_name +``` + +Then let Paperclip know about it. You can put it in your environment config: + +```.rb +# config/environments/production.rb + +... +config.paperclip_defaults = { + storage: :backblaze, + b2_credentials: Rails.root.join('config/b2.yml') +} +``` + +Now just specify the attachment in the model, and it will use your backblaze configuration: + +```.rb +# app/models/note.rb +class Note < ApplicationRecord + has_attached_file :image + ... ``` Currently, these are required options: @@ -58,8 +73,6 @@ Currently, these are required options: account ID and application key. The contents should look something like `b2.yml` above. -- `:b2_bucket` - This should name the bucket to save files to. - ## Contributing This started as a proof of concept for a hobby project, so there's lots of room @@ -73,3 +86,16 @@ the [Contributor Covenant](contributor-covenant.org) code of conduct. ## License The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). + +## Version Log + +### 0.2.0 + +- Remove `:b2_bucket` option. It should now be specified in the credentials file with a `bucket` key. +- Update example that doesn't pollute model with paperclip configuration + +### 0.1.0 + +First release + + diff --git a/lib/paperclip/backblaze/version.rb b/lib/paperclip/backblaze/version.rb index b122a0b..93a6962 100644 --- a/lib/paperclip/backblaze/version.rb +++ b/lib/paperclip/backblaze/version.rb @@ -1,5 +1,5 @@ module Paperclip module Backblaze - VERSION = "0.1.0" + VERSION = "0.2.0" end end diff --git a/lib/paperclip/storage/backblaze.rb b/lib/paperclip/storage/backblaze.rb index 49030c6..89639cb 100644 --- a/lib/paperclip/storage/backblaze.rb +++ b/lib/paperclip/storage/backblaze.rb @@ -12,13 +12,12 @@ module Storage # storage adapter. # # :b2_credentials - This should point to a YAML file containing your B2 - # account ID and application key. The contents should look something - # like: + # account ID, application key, and bucket. The contents should look + # something like: # # account_id: 123456789abc # application_key: 0123456789abcdef0123456789abcdef0123456789 - # - # :b2_bucket - This should name the bucket to save files to. + # bucket: my_b2_bucket_name # # So for example, a model might be configured something like this: # @@ -26,9 +25,19 @@ module Storage # has_attached_file :image, # storage: :backblaze, # b2_credentials: Rails.root.join('config/b2.yml'), - # b2_bucket: 'bucket_for_my_app' # ... # + # You can put the backblaze config in the environment config file e.g.: + # + # # config/environments/production.rb + # + # config.paperclip_defaults = { + # storage: :backblaze, + # b2_credentials: Rails.root.join('config/b2.yml') + # } + # + # If you do it this way, then you don't need to put it in the model. + # module Backblaze def self.extended base base.instance_eval do @@ -76,7 +85,7 @@ def login # Return the Backblaze::B2::Bucket object representing the bucket # specified by the required options[:b2_bucket]. def b2_bucket - @b2_bucket ||= ::Backblaze::B2::Bucket.get_bucket(name: @options[:b2_bucket]) + @b2_bucket ||= ::Backblaze::B2::Bucket.get_bucket(name: b2_credentials[:bucket]) end # Return the specified bucket name as a String. From fe6371aa39e72d1d2bc293a8d8edb72ebea119f3 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sat, 9 Dec 2017 22:07:05 -0800 Subject: [PATCH 11/12] Add fixes from paperclip-backblaze --- lib/backblaze/b2.rb | 69 +++++++++++++------------------- lib/backblaze/b2/base.rb | 36 ++++++++--------- lib/backblaze/b2/bucket.rb | 66 ++++++++++++++---------------- lib/backblaze/b2/file.rb | 41 +++++++++---------- lib/backblaze/b2/file_version.rb | 10 ++--- lib/backblaze/errors.rb | 9 +---- lib/backblaze/utils.rb | 12 +++--- spec/b2/bucket_spec.rb | 15 ++++--- spec/b2_spec.rb | 34 ++++++++-------- spec/helpers.rb | 3 +- 10 files changed, 129 insertions(+), 166 deletions(-) diff --git a/lib/backblaze/b2.rb b/lib/backblaze/b2.rb index f1fe2e0..78a1b84 100644 --- a/lib/backblaze/b2.rb +++ b/lib/backblaze/b2.rb @@ -1,10 +1,10 @@ -require "backblaze/b2/base" -require "backblaze/b2/bucket" -require "backblaze/b2/file" -require "backblaze/b2/file_version" -require 'net/http' +require 'backblaze/b2/base' +require 'backblaze/b2/bucket' +require 'backblaze/b2/file' +require 'backblaze/b2/file_version' require 'tempfile' require 'digest/sha1' +require 'base64' module Backblaze::B2 class << self @@ -17,53 +17,38 @@ class << self # @param [#to_s] application_key the private app key # @raise [Backblaze::AuthError] when unable to authenticate # @return [void] - def login(account_id:, application_key:, api_path: '/b2api/v1/') - options = { - basic_auth: {username: account_id, password: application_key} + def login(options) + api_path = options.fetch(:api_path) { '/b2api/v1/' } + + params = { + headers: { + 'Content-Type' => 'application/json', + 'Accept' => 'application/json', + 'Authorization' => bearer(options) + } } api_path = "/#{api_path}/".gsub(/\/+/, '/') - response = HTTParty.get("https://api.backblazeb2.com#{api_path}b2_authorize_account", options) - raise Backblaze::AuthError.new(response) unless response.code == 200 + + response = HTTParty.get("https://api.backblazeb2.com#{api_path}b2_authorize_account", params) + + raise Backblaze::AuthError, response unless response.success? @account_id = response['accountId'] @token = response['authorizationToken'] @api_url = response['apiUrl'] @download_url = response['downloadUrl'] @api_path = api_path - - Backblaze::B2::Base.base_uri "#{@api_url}#{api_path}" - Backblaze::B2::Base.headers 'Authorization' => @token, 'Content-Type' => 'application/json' + Backblaze::B2::Base.base_uri("#{@api_url}#{api_path}") + Backblaze::B2::Base.headers('Authorization' => token, + 'Content-Type' => 'application/json') end - def credentials_file(filename, raise_errors: true, logging: false) - opts = nil - open(filename, 'r') do |f| - if ::File.extname(filename) == '.json' - require 'json' - opts = JSON.load(f) - else - require 'psych' - opts = Psych.load(f.read) - end - end - parsed = {} - [:application_key, :account_id, :api_path].each do |key| - if opts[key.to_s].is_a? String - parsed[key] = opts[key.to_s] - end - end - if [:application_key, :account_id].inject(true) { |status, key| status && !parsed[key].nil? } - puts "Attempting #{parsed[:account_id]}" if logging - login(parsed) - true - else - puts "Missing params" if logging - false - end - rescue => e - puts e if logging - raise e if raise_errors - false + def bearer(options) + account_id = options.fetch(:account_id) + application_key = options.fetch(:application_key) + token = Base64.strict_encode64("#{account_id}:#{application_key}") + + "Basic #{token}" end end end diff --git a/lib/backblaze/b2/base.rb b/lib/backblaze/b2/base.rb index 2b899d1..1830f7f 100644 --- a/lib/backblaze/b2/base.rb +++ b/lib/backblaze/b2/base.rb @@ -18,8 +18,8 @@ class Base # @!method put(path, options={}, &block) # (see #get) - [:get, :head, :post, :put].each do |req| - define_method(req) do |path, options={}, &block| + %i[get head post put].each do |req| + define_method(req) do |path, options = {}, &block| self.class.send(req, path, options, &block) end end @@ -30,18 +30,20 @@ def file_versions(bucket_id:, convert:, limit:, double_check_server:, file_name: retreive_count = (double_check_server ? 0 : -1) files = file_list(bucket_id: bucket_id, limit: limit, retreived: retreive_count, file_name: file_name, first_file: nil, start_field: 'startFileId'.freeze) - files.map! do |f| - if block.nil? - Backblaze::B2::FileVersion.new(f) - else - block.call(f) + if convert + files.map! do |f| + if block.nil? + Backblaze::B2::FileVersion.new(f) + else + yield(f) + end end - end if convert + end files.compact end def file_list(limit:, retreived:, first_file:, start_field:, bucket_id:, file_name: nil, first: true) - params = {'bucketId'.freeze => bucket_id} + params = { 'bucketId'.freeze => bucket_id } if limit == -1 params['maxFileCount'.freeze] = 1000 elsif limit > 1000 @@ -59,31 +61,25 @@ def file_list(limit:, retreived:, first_file:, start_field:, bucket_id:, file_na params[start_field] = first_file end - p params - response = post("/b2_list_file_#{start_field == 'startFileName' ? 'names' : 'versions'}", body: params.to_json) - raise Backblaze::FileError.new(response) unless response.code == 200 + raise Backblaze::FileError, response unless response.code == 200 files = response['files'.freeze] halt = false files.map! do |f| if halt - p f nil else - ret = Hash[f.map{|k,v| [Backblaze::Utils.underscore(k).to_sym, v]}] - p ret - if file_name && file_name != ret[:file_name] - halt = true - end + ret = Hash[f.map { |k, v| [Backblaze::Utils.underscore(k).to_sym, v] }] + halt = true if file_name && file_name != ret[:file_name] halt ? nil : ret end end.compact! - retreived = retreived + files.size if retreived >= 0 + retreived += files.size if retreived >= 0 if limit > 0 - limit = limit - (retreived >= 0 ? files.size : 1000) + limit -= (retreived >= 0 ? files.size : 1000) limit = 0 if limit < 0 end diff --git a/lib/backblaze/b2/bucket.rb b/lib/backblaze/b2/bucket.rb index 8aac94b..b39030e 100644 --- a/lib/backblaze/b2/bucket.rb +++ b/lib/backblaze/b2/bucket.rb @@ -1,5 +1,4 @@ module Backblaze::B2 - ## # A class to represent the online buckets. Mostly used for file access class Bucket < Base @@ -9,25 +8,20 @@ class Bucket < Base # @param [#to_s] bucket_id the bucket id # @param [#to_s] bucket_type the bucket publicity type # @param [#to_s] account_id the account to which this bucket belongs - def initialize(bucket_name:, bucket_id:, bucket_type:, account_id:, cache: false, - bucket_info: {}, lifecycle_rules: [], revision: nil) - @bucket_name = bucket_name - @bucket_id = bucket_id - @bucket_type = bucket_type - @account_id = account_id + def initialize(options) + @bucket_name = options.fetch(:bucket_name) + @bucket_id = options.fetch(:bucket_id) + @bucket_type = options.fetch(:bucket_type) + @account_id = options.fetch(:account_id) end # @return [String] bucket name - def bucket_name - @bucket_name - end + attr_reader :bucket_name - alias_method :name, :bucket_name + alias name bucket_name # @return [String] bucket id - def bucket_id - @bucket_id - end + attr_reader :bucket_id # @return [Boolean] is the bucket public def public? @@ -40,14 +34,10 @@ def private? end # @return [String] account id - def account_id - @account_id - end + attr_reader :account_id # @return [String] bucket type - def bucket_type - @bucket_type - end + attr_reader :bucket_type # Check if eqivalent. Takes advantage of globally unique names # @return [Boolean] equality @@ -78,12 +68,14 @@ def file_names(first_file: nil, limit: 100, cache: false, convert: true, double_ retreive_count = (double_check_server ? 0 : -1) files = file_list(bucket_id: bucket_id, limit: limit, retreived: retreive_count, first_file: first_file, start_field: 'startFileName'.freeze) - merge_params = {bucket_id: bucket_id} - files.map! do |f| - Backblaze::B2::File.new(f.merge(merge_params)) - end if convert + merge_params = { bucket_id: bucket_id } + if convert + files.map! do |f| + Backblaze::B2::File.new(f.merge(merge_params)) + end + end if cache - @file_name_cache = {limit: limit, convert: convert, files: files} + @file_name_cache = { limit: limit, convert: convert, files: files } end files end @@ -95,17 +87,17 @@ def file_versions(limit: 100, cache: false, convert: true, double_check_server: end end file_versions = super(limit: 100, convert: convert, double_check_server: double_check_server, bucket_id: bucket_id) - files = file_versions.group_by {|version| convert ? version.file_name : version[:file_name]} + files = file_versions.group_by { |version| convert ? version.file_name : version[:file_name] } if convert files = files.map do |name, versions| File.new(file_name: name, bucket_id: bucket_id, versions: versions) end end - if cache - @file_versions_cache = {limit: limit, convert: convert, files: files} - else - @file_versions_cache = {} - end + @file_versions_cache = if cache + { limit: limit, convert: convert, files: files } + else + {} + end files end @@ -129,17 +121,17 @@ def create(name:, type:) } response = post('/b2_create_bucket', body: body.to_json) - raise Backblaze::BucketError.new(response) unless response.code / 100 == 2 + raise Backblaze::BucketError, response unless response.code / 100 == 2 - params = Hash[response.map{|k,v| [Backblaze::Utils.underscore(k).to_sym, v]}] + params = Hash[response.map { |k, v| [Backblaze::Utils.underscore(k).to_sym, v] }] new(params) end def upload_url(bucket_id:) - response = post('/b2_get_upload_url', body: {bucketId: bucket_id}.to_json) - raise Backblaze::BucketError.new(response) unless response.code / 100 == 2 - {url: response['uploadUrl'], token: response['authorizationToken']} + response = post('/b2_get_upload_url', body: { bucketId: bucket_id }.to_json) + raise Backblaze::BucketError, response unless response.code / 100 == 2 + { url: response['uploadUrl'], token: response['authorizationToken'] } end ## @@ -151,7 +143,7 @@ def buckets } response = post('/b2_list_buckets', body: body.to_json) response['buckets'].map do |bucket| - params = Hash[bucket.map{|k,v| [Backblaze::Utils.underscore(k).to_sym, v]}] + params = Hash[bucket.map { |k, v| [Backblaze::Utils.underscore(k).to_sym, v] }] new(params) end end diff --git a/lib/backblaze/b2/file.rb b/lib/backblaze/b2/file.rb index 5b4bd65..399ddfc 100644 --- a/lib/backblaze/b2/file.rb +++ b/lib/backblaze/b2/file.rb @@ -14,7 +14,7 @@ def initialize(file_name:, bucket_id:, versions: nil, **file_version_args) class << self def create(data:, bucket:, name: nil, base_name: '', content_type: 'b2/x-auto', info: {}) - raise ArgumentError.new('data must not be nil') if data.nil? + raise ArgumentError, 'data must not be nil' if data.nil? case bucket when String @@ -22,42 +22,40 @@ def create(data:, bucket:, name: nil, base_name: '', content_type: 'b2/x-auto', when Bucket upload_url = bucket.upload_url else - raise ArgumentError.new('You must pass a bucket') + raise ArgumentError, 'You must pass a bucket' end - close_file = false case data when String data.force_encoding('ASCII-8BIT') - raise ArgumentError.new('Must provide a file name for data') if name.nil? + raise ArgumentError, 'Must provide a file name for data' if name.nil? when ::File, Tempfile, ::Paperclip::UploadedFileAdapter data.binmode data.rewind if name.nil? - raise ArgumentError.new('Must provide a file name with Tempfiles') if data.is_a? Tempfile + raise ArgumentError, 'Must provide a file name with Tempfiles' if data.is_a? Tempfile name = ::File.basename(data) end else - raise ArgumentError.new('Must provide a file name with streams') if name.nil? + raise ArgumentError, 'Must provide a file name with streams' if name.nil? if data.respond_to?(:read) - close_file = true temp = Tempfile.new(name) temp.binmode IO.copy_stream(data, temp) data = temp data.rewind else - raise ArgumentError.new('Unsuitable data type. Please read the docs.') + raise ArgumentError, 'Unsuitable data type. Please read the docs.' end end uri = URI(upload_url[:url]) req = Net::HTTP::Post.new(uri) - req.add_field("Authorization", upload_url[:token]) - req.add_field("X-Bz-File-Name", "#{base_name}/#{name}".tr_s('/', '/').sub(/\A\//, '')) - req.add_field("Content-Type", content_type) - req.add_field("Content-Length", data.size) + req.add_field('Authorization', upload_url[:token]) + req.add_field('X-Bz-File-Name', "#{base_name}/#{name}".tr_s('/', '/').sub(/\A\//, '')) + req.add_field('Content-Type', content_type) + req.add_field('Content-Length', data.size) digest = Digest::SHA1.new if data.is_a? String @@ -73,19 +71,20 @@ def create(data:, bucket:, name: nil, base_name: '', content_type: 'b2/x-auto', req.body_stream = data end - req.add_field("X-Bz-Content-Sha1", digest) + req.add_field('X-Bz-Content-Sha1', digest) info.first(10).map do |key, value| - req.add_field("X-Bz-Info-#{URI.encode(key)}", value) + encoded_key = URI.encode_www_form_component(key) + req.add_field("X-Bz-Info-#{encoded_key}", value) end http = Net::HTTP.new(req.uri.host, req.uri.port) http.use_ssl = (req.uri.scheme == 'https') - res = http.start {|make| make.request(req)} + res = http.start { |make| make.request(req) } response = JSON.parse(res.body) - raise Backblaze::FileError.new(response) unless res.code.to_i == 200 + raise Backblaze::FileError, response unless res.code.to_i == 200 params = { file_name: response['fileName'], @@ -103,10 +102,8 @@ def create(data:, bucket:, name: nil, base_name: '', content_type: 'b2/x-auto', end end - def file_name - @file_name - end - alias_method :name, :file_name + attr_reader :file_name + alias name file_name def versions unless @fetched_all @@ -150,9 +147,7 @@ def destroy!(thread_count: 4) end threads.map(&:join) @destroyed = true - if errors.any? - raise Backblaze::DestroyErrors.new(errors) - end + raise Backblaze::DestroyErrors, errors if errors.any? end def exists? diff --git a/lib/backblaze/b2/file_version.rb b/lib/backblaze/b2/file_version.rb index 5192995..80604e4 100644 --- a/lib/backblaze/b2/file_version.rb +++ b/lib/backblaze/b2/file_version.rb @@ -18,10 +18,10 @@ def initialize(file_id:, size:, upload_timestamp:, action:, file_name:, def get_info unless defined?(@get_info) - response = post('/b2_get_file_info', body: {fileId: file_id}.to_json) - raise Backblaze::FileError.new(response) unless response.code == 200 + response = post('/b2_get_file_info', body: { fileId: file_id }.to_json) + raise Backblaze::FileError, response unless response.code == 200 - @get_info = Hash[response.map{|k,v| [Backblaze::Utils.underscore(k).to_sym, v]}] + @get_info = Hash[response.map { |k, v| [Backblaze::Utils.underscore(k).to_sym, v] }] end @get_info end @@ -31,8 +31,8 @@ def download_url end def destroy! - response = post('/b2_delete_file_version', body: {fileName: file_name, fileId: file_id}.to_json) - raise Backblaze::FileError.new(response) unless response.code == 200 + response = post('/b2_delete_file_version', body: { fileName: file_name, fileId: file_id }.to_json) + raise Backblaze::FileError, response unless response.code == 200 @destroyed = true end diff --git a/lib/backblaze/errors.rb b/lib/backblaze/errors.rb index 7231be3..ea3b89d 100644 --- a/lib/backblaze/errors.rb +++ b/lib/backblaze/errors.rb @@ -19,9 +19,7 @@ def initialize(response) ## # The response from the server # @return [HTTParty::Response] the response - def response - @response - end + attr_reader :response ## # The Backblaze B2 error code @@ -44,7 +42,6 @@ def message self['message'] end - ## # Shortcut to access the response keys # @return [Object] the object stored at `key` in the response @@ -66,9 +63,7 @@ def initialize(errors) ## # The Backblaze B2 error messages which broke things # @return [Array] errors errors raised destroying files - def errors - @errors - end + attr_reader :errors end ## diff --git a/lib/backblaze/utils.rb b/lib/backblaze/utils.rb index c29da84..4e00ceb 100644 --- a/lib/backblaze/utils.rb +++ b/lib/backblaze/utils.rb @@ -1,13 +1,13 @@ module Backblaze::Utils def underscore(word) - word.to_s. - gsub(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2'). - gsub(/([a-z\d])([A-Z])/,'\1_\2'). - tr("-", "_"). - downcase + word.to_s + .gsub(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2') + .gsub(/([a-z\d])([A-Z])/, '\1_\2') + .tr('-', '_') + .downcase end - def camelize(word, capitalize=false) + def camelize(word, capitalize = false) word = word.to_s "#{capitalize ? word[0, 1].upcase : word[0, 1].downcase}#{word.split('_').map(&:capitalize).join('')[1..-1]}" end diff --git a/spec/b2/bucket_spec.rb b/spec/b2/bucket_spec.rb index f6ec75a..1d71a77 100644 --- a/spec/b2/bucket_spec.rb +++ b/spec/b2/bucket_spec.rb @@ -5,16 +5,16 @@ context 'success' do let(:success) do { - "bucketId" => "4a48fe8875c6214145260818", - "accountId" => "010203040506", - "bucketName" => "some_bucket", - "bucketType" => "allPublic" + 'bucketId' => '4a48fe8875c6214145260818', + 'accountId' => '010203040506', + 'bucketName' => 'some_bucket', + 'bucketType' => 'allPublic' } end it 'should create a bucket' do stub_request(:post, /.*\/b2_create_bucket.*/).to_return( - headers: {'Content-Type' => 'application/json'}, + headers: { 'Content-Type' => 'application/json' }, body: success.to_json, status: 201 ) @@ -29,7 +29,7 @@ end describe '.files' do - let(:bucket){Backblaze::B2::Bucket.new(bucket_type: 'allPublic', bucket_name: 'generic_bucket', bucket_id: 'fhdjsfhdkja', account_id: 'fhdjkafd')} + let(:bucket) { Backblaze::B2::Bucket.new(bucket_type: 'allPublic', bucket_name: 'generic_bucket', bucket_id: 'fhdjsfhdkja', account_id: 'fhdjkafd') } context 'success' do before do @@ -38,7 +38,7 @@ 4.times do files = file_list(size: 10, next_item: next_item) next_item = files['files'][9]['fileName'] - list.insert(0, {body: files.to_json, status: 200}) + list.insert(0, body: files.to_json, status: 200) end stub_request(:post, /.*\/b2_list_file_names.*/).to_return(*list) end @@ -62,7 +62,6 @@ expect(files1.size).to eq 10 expect(files2).to eq files1 end - end end end diff --git a/spec/b2_spec.rb b/spec/b2_spec.rb index 53f97a0..16dc375 100644 --- a/spec/b2_spec.rb +++ b/spec/b2_spec.rb @@ -8,32 +8,33 @@ end it 'should raise AuthError on failure' do - stub_request(:get, 'https://failed:login@api.backblazeb2.com/b2api/v1/b2_authorize_account').to_return( - body: "{\"code\":\"unauthorized\",\"message\":\"invalid_authorization_headers\",\"status\":401}", - headers: {'Content-Type' => 'application/json'}, - status: 401 - ) - expect {Backblaze::B2.login(account_id: 'failed', application_key: 'login')}.to raise_error(Backblaze::AuthError) + stub_request(:get, 'https://api.backblazeb2.com/b2api/v1/b2_authorize_account') + .to_return( + body: '{"code":"unauthorized","message":"invalid_authorization_headers","status":401}', + headers: { 'Content-Type' => 'application/json' }, + status: 401 + ) + expect { Backblaze::B2.login(account_id: 'failed', application_key: 'login') }.to raise_error(Backblaze::AuthError) end end context 'successful login' do let(:success) do { - accountId: "YOUR_ACCOUNT_ID", - apiUrl: "https://api900.backblaze.com", - authorizationToken: "2_20150807002553_443e98bf57f978fa58c284f8_24d25d99772e3ba927778b39c9b0198f412d2163_acct", - downloadUrl: "https://f900.backblaze.com" + accountId: 'YOUR_ACCOUNT_ID', + apiUrl: 'https://api900.backblaze.com', + authorizationToken: '2_20150807002553_443e98bf57f978fa58c284f8_24d25d99772e3ba927778b39c9b0198f412d2163_acct', + downloadUrl: 'https://f900.backblaze.com' } end before do - stub_request(:get, 'https://real:login@api.backblazeb2.com/b2api/v1/b2_authorize_account').to_return( - body: success.to_json, - headers: {'Content-Type' => 'application/json'}, - status: 200, - - ) + stub_request(:get, 'https://api.backblazeb2.com/b2api/v1/b2_authorize_account') + .to_return( + status: 200, + body: success.to_json, + headers: { 'Content-Type' => 'application/json' } + ) end it 'should succeed' do @@ -42,7 +43,6 @@ it 'should set variables' do Backblaze::B2.login(account_id: 'real', application_key: 'login') - expect(Backblaze::B2.api_url).to eq(success[:apiUrl]) expect(Backblaze::B2.account_id).to eq(success[:accountId]) expect(Backblaze::B2.token).to eq(success[:authorizationToken]) diff --git a/spec/helpers.rb b/spec/helpers.rb index 6ce41ce..35f7265 100644 --- a/spec/helpers.rb +++ b/spec/helpers.rb @@ -1,11 +1,12 @@ module Helpers + require 'securerandom' def file_list(size: 10, next_item: nil) files = [] size.times do files << { 'action' => 'upload', 'fileId' => SecureRandom.uuid.tr('-', '_'), - 'fileName' => "random_file_#{rand(0..10000)}.txt", + 'fileName' => "random_file_#{rand(0..10_000)}.txt", 'size' => rand(10..1000), 'uploadTimestamp' => Time.now.to_i } From ae964d6228ae13fec02b36312060e9d4b2a17bc7 Mon Sep 17 00:00:00 2001 From: Alex Tsui Date: Sat, 9 Dec 2017 22:29:51 -0800 Subject: [PATCH 12/12] Rip out paperclip files and declare independence --- .travis.yml | 11 +- README.md | 85 ++-------- ...lip-backblaze.gemspec => backblaze.gemspec | 12 +- lib/backblaze/version.rb | 2 +- lib/paperclip/backblaze.rb | 4 - lib/paperclip/backblaze/version.rb | 5 - lib/paperclip/storage/backblaze.rb | 145 ------------------ 7 files changed, 30 insertions(+), 234 deletions(-) rename paperclip-backblaze.gemspec => backblaze.gemspec (67%) delete mode 100644 lib/paperclip/backblaze.rb delete mode 100644 lib/paperclip/backblaze/version.rb delete mode 100644 lib/paperclip/storage/backblaze.rb diff --git a/.travis.yml b/.travis.yml index ed83bc1..9e4ab53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,13 @@ language: ruby +sudo: false rvm: - 2.2.2 -before_install: gem install bundler -v 1.10.5 + - 2.4.2 +before_install: + - rvm @global do gem install bundler -v 1.12.5 + +gemfile: + - Gemfile + +script: + bundle exec rake diff --git a/README.md b/README.md index ef43bce..4d32ccd 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -# paperclip-backblaze +# backblaze -The `paperclip-backblaze` provides a [Paperclip](https://github.com/thoughtbot/paperclip) storage adapter so that -attachments can be saved to [Backblaze B2 Cloud Storage API](https://www.backblaze.com/b2/docs/). -It makes use of Winston Durand's [backblaze](https://github.com/R167/backblaze) gem -to access the B2 API behind the scenes. +The `paperclip` gem abstracts the [Backblaze B2 Cloud Storage API](https://www.backblaze.com/b2/docs/). +It was started by Winston Durand at [R167/backblaze](https://github.com/R167/backblaze). Backblaze B2 Cloud Storage is similar to Amazon's AWS S3 Storage, but it has a few selling points: @@ -11,75 +9,25 @@ Backblaze B2 Cloud Storage is similar to Amazon's AWS S3 Storage, but it has a f 2. It's $0.005/GB/month vs S3's $0.030/GB/month 3. You actually get a free GB of bandwidth a day, so it might be nice for personal projects. -## Installation - -Add this line to your application's Gemfile: - -```ruby -gem 'paperclip-backblaze', github: 'alextsui05/paperclip-backblaze' -``` -And then execute: - - $ bundle - ## Usage -You should be familiar with configuring Paperclip attachments for your model. -If not, please start with the Paperclip documentation -[here](https://github.com/thoughtbot/paperclip#usage). - -Configuring Backblaze storage is very similar to [configuring S3 storage](http://www.rubydoc.info/gems/paperclip/Paperclip/Storage/S3). -Let's suppose we have a `Note` model with an `image` attachment that we would -like to be backed by Backblaze storage. +After fetching the repo and running bundle, run `bin/console` and do this: - - -First, put your credentials and bucket name in a YML file. - -```.yml -# config/b2.yml -account_id: 123456789abc -application_key: 0123456789abcdef0123456789abcdef0123456789 -bucket: my_b2_bucket_name ``` - -Then let Paperclip know about it. You can put it in your environment config: - -```.rb -# config/environments/production.rb - -... -config.paperclip_defaults = { - storage: :backblaze, - b2_credentials: Rails.root.join('config/b2.yml') -} +# log in +> Backblaze::B2.login(account_id: "your_backblaze_account_id", application_key: "your_application_key") +# list buckets +> Backblaze::B2::Bucket.buckets ``` -Now just specify the attachment in the model, and it will use your backblaze configuration: - -```.rb -# app/models/note.rb -class Note < ApplicationRecord - has_attached_file :image - ... -``` - -Currently, these are required options: - -- `:storage` - This should be set to :backblaze in order to use this - storage adapter. +## Applications -- `:b2_credentials` - This should point to a YAML file containing your B2 - account ID and application key. The contents should look something - like `b2.yml` above. +This is used as a storage backend for Paperclip attachments in Rails. See [paperclip-backblaze](https://github.com/alextsui05/paperclip-backblaze) ## Contributing -This started as a proof of concept for a hobby project, so there's lots of room -for improvement, and it would be great to have your help. - Bug reports and pull requests are welcome on GitHub at -https://github.com/alextsui05/paperclip-backblaze. This project is intended to be a safe, +https://github.com/alextsui05/backblaze. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct. @@ -89,13 +37,6 @@ The gem is available as open source under the terms of the [MIT License](http:// ## Version Log -### 0.2.0 - -- Remove `:b2_bucket` option. It should now be specified in the credentials file with a `bucket` key. -- Update example that doesn't pollute model with paperclip configuration - -### 0.1.0 - -First release - +### 0.4.0 +Officially hijacking the gem from R167 and bumping the version. diff --git a/paperclip-backblaze.gemspec b/backblaze.gemspec similarity index 67% rename from paperclip-backblaze.gemspec rename to backblaze.gemspec index 0907586..dd8695f 100644 --- a/paperclip-backblaze.gemspec +++ b/backblaze.gemspec @@ -1,17 +1,17 @@ # coding: utf-8 lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'paperclip/backblaze/version' +require 'backblaze/version' Gem::Specification.new do |spec| - spec.name = "paperclip-backblaze" - spec.version = Paperclip::Backblaze::VERSION + spec.name = "backblaze" + spec.version = Backblaze::VERSION spec.authors = ["Alex Tsui", "Winston Durand"] spec.email = ["alextsui05@gmail.com"] - spec.summary = %q{Paperclip storage adapter for Backblaze B2 Cloud.} - spec.description = %q{Allows Paperclip attachments to be backed by Backblaze B2 Cloud storage as an alternative to AWS S3.} - spec.homepage = "https://github.com/alextsui05/paperclip-backblaze" + spec.summary = %q{Interface for the Backblaze B2 Cloud.} + spec.description = %q{Abstraction for the Backblaze B2 Cloud Storage API. Original by github.com:R167/backblaze} + spec.homepage = "https://github.com/alextsui05/backblaze" spec.license = "MIT" spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } diff --git a/lib/backblaze/version.rb b/lib/backblaze/version.rb index ffb27fc..ad4dd6a 100644 --- a/lib/backblaze/version.rb +++ b/lib/backblaze/version.rb @@ -1,3 +1,3 @@ module Backblaze - VERSION = "0.3.0" + VERSION = '0.4.0'.freeze end diff --git a/lib/paperclip/backblaze.rb b/lib/paperclip/backblaze.rb deleted file mode 100644 index 2375dfe..0000000 --- a/lib/paperclip/backblaze.rb +++ /dev/null @@ -1,4 +0,0 @@ -require_relative '../backblaze' - -require "paperclip/backblaze/version" -require "paperclip/storage/backblaze" diff --git a/lib/paperclip/backblaze/version.rb b/lib/paperclip/backblaze/version.rb deleted file mode 100644 index 93a6962..0000000 --- a/lib/paperclip/backblaze/version.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Paperclip - module Backblaze - VERSION = "0.2.0" - end -end diff --git a/lib/paperclip/storage/backblaze.rb b/lib/paperclip/storage/backblaze.rb deleted file mode 100644 index 89639cb..0000000 --- a/lib/paperclip/storage/backblaze.rb +++ /dev/null @@ -1,145 +0,0 @@ -module Paperclip - module Storage - # Defines a adapter to use store Paperclip attachments to Backblaze B2 - # Cloud Storage, which is similar to Amazon's S3 service. - # - # This allows you to use has_attached_file in your models - # with :storage => :backblaze. - # - # Some required options include: - # - # :storage - This should be set to :backblaze in order to use this - # storage adapter. - # - # :b2_credentials - This should point to a YAML file containing your B2 - # account ID, application key, and bucket. The contents should look - # something like: - # - # account_id: 123456789abc - # application_key: 0123456789abcdef0123456789abcdef0123456789 - # bucket: my_b2_bucket_name - # - # So for example, a model might be configured something like this: - # - # class Note < ApplicationRecord - # has_attached_file :image, - # storage: :backblaze, - # b2_credentials: Rails.root.join('config/b2.yml'), - # ... - # - # You can put the backblaze config in the environment config file e.g.: - # - # # config/environments/production.rb - # - # config.paperclip_defaults = { - # storage: :backblaze, - # b2_credentials: Rails.root.join('config/b2.yml') - # } - # - # If you do it this way, then you don't need to put it in the model. - # - module Backblaze - def self.extended base - base.instance_eval do - login - unless @options[:url].match(/\Ab2.*url\z/) - @options[:url] = ":b2_path_url".freeze - end - end - - Paperclip.interpolates(:b2_path_url) do |attachment, style| - "#{::Backblaze::B2.download_url}/file/#{attachment.b2_bucket_name}/#{attachment.path(style).sub(%r{\A/}, '')}" - end unless Paperclip::Interpolations.respond_to? :b2_path_url - end - - # Fetch the credentials from the config file, if it hasn't already been - # loaded. - # - # filename : String - path to the YAML config file containing the - # Backblaze B2 credentials. Here is example contents of what one - # may look like: - # - # account_id: 123456789abc - # application_key: 0123456789abcdef0123456789abcdef0123456789 - # - # Returns a Hash containing the parsed credentials. - def b2_credentials(filename = @options[:b2_credentials]) - unless @b2_credentials - require 'psych' - File.open(filename, 'r') do |f| - @b2_credentials = Psych.load(f.read).symbolize_keys - end - end - @b2_credentials - end - - # Authenticate with Backblaze with the account ID and secret key. This - # also caches several variables from the response related to the API, so - # it is important that it is executed at the very beginning. - def login - return if ::Backblaze::B2.token - creds = b2_credentials - ::Backblaze::B2.login(account_id: creds[:account_id], application_key: creds[:application_key]) - end - - # Return the Backblaze::B2::Bucket object representing the bucket - # specified by the required options[:b2_bucket]. - def b2_bucket - @b2_bucket ||= ::Backblaze::B2::Bucket.get_bucket(name: b2_credentials[:bucket]) - end - - # Return the specified bucket name as a String. - def b2_bucket_name - b2_bucket.bucket_name - end - - # Return whether this attachment exists in the bucket. - def exists?(style = default_style) - !!get_file(filename: get_path(style)) - end - - # Return a Backblaze::B2::File object representing the file named in the - # filename keyword, if it exists. - def get_file(filename:) - b2_bucket.file_names(first_file: filename, limit: 1).find do |f| - f.file_name == filename - end - end - - # Return this attachment's bucket file path as a String. - def get_path(style = default_style) - path(style).sub(%r{\A/}, '') - end - - # (Internal) Used by Paperclip to upload local files to storage. - def flush_writes - @queued_for_write.each do |style, file| - base_name = ::File.dirname(get_path(style)) - name = ::File.basename(get_path(style)) - ::Backblaze::B2::File.create data: file, bucket: b2_bucket, name: name, base_name: base_name - end - @queued_for_write = {} - end - - # (Internal) Used by Paperclip to remove remote files from storage. - def flush_deletes - @queued_for_delete.each do |path| - if file = get_file(filename: path.sub(%r{\A/}, '')) - file.destroy! - end - end - @queued_for_delete = [] - end - - # (Internal) - def copy_to_local_file(style, local_dest_path) - ::File.open(local_dest_path, 'wb') do |local_file| - file = get_file(filename: get_path(style)) - body = file.get(file.latest.download_url, parse: :plain) - local_file.write(body) - end - end - - end # module Backblaze - end # module Storage -end # module Paperclip