diff --git a/README.md b/README.md index 39947fc..c067527 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,12 @@ ## About -A Flask extension to **access, upload, download, save and delete** files on cloud storage providers such as: +A Flask extension to **access, upload, download, save and delete** files on cloud storage providers such as: AWS S3, Google Storage, Microsoft Azure, Rackspace Cloudfiles, and even Local file system. For local file storage, it also provides a flask endpoint to access the files. - - + + Version: 1.x.x --- @@ -17,38 +17,38 @@ Version: 1.x.x from flask import Flask, request from flask_cloudy import Storage - + app = Flask(__name__) - - # Update the config + + # Update the config app.config.update({ - "STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc... + "STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc... "STORAGE_KEY": "", "STORAGE_SECRET": "", "STORAGE_CONTAINER": "./", # a directory path for local, bucket name of cloud "STORAGE_SERVER": True, "STORAGE_SERVER_URL": "/files" # The url endpoint to access files on LOCAL provider }) - + # Setup storage storage = Storage() - storage.init_app(app) - + storage.init_app(app) + @app.route("/upload", methods=["POST", "GET"]): def upload(): if request.method == "POST": file = request.files.get("file") my_upload = storage.upload(file) - + # some useful properties name = my_upload.name extension = my_upload.extension size = my_upload.size url = my_upload.url - + return url - - # Pretending the file uploaded is "my-picture.jpg" + + # Pretending the file uploaded is "my-picture.jpg" # it will return a url in the format: http://domain.com/files/my-picture.jpg @@ -59,17 +59,17 @@ Version: 1.x.x if my_object: download_url = my_object.download() return download_url - else: + else: abort(404, "File doesn't exist") ---- - +--- + Go to the "example" directory to get a workable flask-cloud example ---- - - +--- + + ### Features: - Browse files @@ -100,8 +100,8 @@ Go to the "example" directory to get a workable flask-cloud example - Flask -- Apache-Libcloud - +- Apache-Libcloud + --- ## Install & Config @@ -116,9 +116,9 @@ Go to the "example" directory to get a workable flask-cloud example Within your Flask application's settings you can provide the following settings to control the behavior of Flask-Cloudy - -**- STORAGE_PROVIDER** (str) + +**- STORAGE_PROVIDER** (str) - LOCAL - S3 @@ -148,7 +148,7 @@ None for LOCAL The *BUCKET NAME* for cloud storage providers -For *LOCAL* provider, this is the local directory path +For *LOCAL* provider, this is the local directory path **STORAGE_ALLOWED_EXTENSIONS** (list) @@ -159,7 +159,7 @@ Example: ["png", "jpg", "jpeg", "mp3"] **STORAGE_SERVER** (bool) -For *LOCAL* provider only. +For *LOCAL* provider only. True to expose the files in the container so they can be accessed @@ -169,7 +169,7 @@ Default: *True* For *LOCAL* provider only. -The endpoint to access the files from the local storage. +The endpoint to access the files from the local storage. Default: */files* @@ -181,17 +181,17 @@ Flask-Cloudy is a wrapper around Apache-Libcloud, the Storage class gives you ac *Lexicon:* -Object: A file or a file path. +Object: A file or a file path. Container: The main directory, or a bucket name containing all the objects -Provider: The method +Provider: The method -Storage: +Storage: ### flask_cloudy.Storage -The **Storage** class allows you to access, upload, get an object from the Storage. +The **Storage** class allows you to access, upload, get an object from the Storage. #### Storage(provider, key=None, secret=None, container=None, allowed_extensions=None) @@ -212,12 +212,12 @@ The **Storage** class allows you to access, upload, get an object from the Stora - secret: The secret access key of the cloud storage. None when provider is LOCAL -- container: +- container: + + - For cloud storage, use the **BUCKET NAME** + + - For LOCAL provider, it's the directory path where to access the files - - For cloud storage, use the **BUCKET NAME** - - - For LOCAL provider, it's the directory path where to access the files - - allowed_extensions: List of extensions to upload to upload @@ -229,40 +229,40 @@ It will also setup a server endpoint when STORAGE_PROVIDER == LOCAL from flask import Flask, request from flask_cloudy import Storage - + app = Flask(__name__) - - # Update the config + + # Update the config app.config.update({ - "STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc... + "STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc... "STORAGE_KEY": "", "STORAGE_SECRET": "", "STORAGE_CONTAINER": "./", # a directory path for local, bucket name of cloud "STORAGE_SERVER": True, "STORAGE_SERVER_URL": "/files" }) - + # Setup storage storage = Storage() - storage.init_app(app) - + storage.init_app(app) + @app.route("/upload", methods=["POST", "GET"]): def upload(): if request.method == "POST": file = request.files.get("file") my_upload = storage.upload(file) - + # some useful properties name = my_upload.name extension = my_upload.extension size = my_upload.size url = my_upload.url - + return url - - # Pretending the file uploaded is "my-picture.jpg" + + # Pretending the file uploaded is "my-picture.jpg" # it will return a url in the format: http://domain.com/files/my-picture.jpg - + #### Storage.get(object_name) @@ -277,9 +277,9 @@ Some valid object names, they can contains slashes to indicate it's a directory - file.txt - + - my_dir/file.txt - + - my_dir/sub_dir/file.txt . @@ -287,8 +287,8 @@ Some valid object names, they can contains slashes to indicate it's a directory storage = Storage(provider, key, secret, container) object_name = "hello.txt" my_object = storage.get(object_name) - - + + #### Storage.upload(file, name=None, prefix=None, extension=[], overwrite=Flase, public=False, random_name=False) @@ -298,7 +298,7 @@ To save or upload a file in the container - name: to give the file a new name -- prefix: a name to add in front of the file name. Add a slash at the end of +- prefix: a name to add in front of the file name. Add a slash at the end of prefix to make it a directory otherwise it will just append it to the name - extensions: list of extensions @@ -313,44 +313,44 @@ prefix to make it a directory otherwise it will just append it to the name storage = Storage(provider, key, secret, container) my_file = "my_dir/readme.md" - - + + **1) Upload file + file name is the name of the uploaded file ** - - storage.upload(my_file) - - + + storage.upload(my_file) + + **2) Upload file + file name is now `new_readme`. It will will keep the extension of the original file** - + storage.upload(my_file, name="new_readme") - + The uploaded file will be named: **new_readme.md** - - + + **3) Upload file to a different path using `prefix`** - - + + storage.upload(my_file, name="new_readme", prefix="my_dir/") - + now the filename becomes **my_dir/new_readme.md** -On LOCAL it will create the directory *my_dir* if it doesn't exist. +On LOCAL it will create the directory *my_dir* if it doesn't exist. + - storage.upload(my_file, name="new_readme", prefix="my_new_path-") - + now the filename becomes **my_new_path-new_readme.md** ATTENTION: If you want the file to be place in a subdirectory, `prefix` must have the trailing slash - + **4a.) Public upload** storage.upload(my_file, public=True) - - + + **4b.) Private upload** storage.upload(my_file, public=False) @@ -358,27 +358,27 @@ ATTENTION: If you want the file to be place in a subdirectory, `prefix` must hav **5) Upload + random name** storage.upload(my_file, random_name=True) - + **6) Upload with external url*** - - You can upload an item from the internet directly to your storage - + + You can upload an item from the internet directly to your storage + storage.upload("http://the.site.path.com/abc.png") - + It will save the image to your storage #### Storage.create(object_name, size=0, hash=None, extra=None, metda_data=None) -Explicitly create an object that may exist already. Usually, when paramameters (name, size, hash, etc...) are already saved, let's say in the database, and you want Storage to manipulate the file. +Explicitly create an object that may exist already. Usually, when paramameters (name, size, hash, etc...) are already saved, let's say in the database, and you want Storage to manipulate the file. storage = Storage(provider, key, secret, container) existing_name = "holla.txt" existing_size = "8000" # in bytes new_object = storage.create(object_name=existing_name, size=existing_size) - + # Now I can do - url = new_object.url + url = new_object.url size = len(new_object) @@ -387,7 +387,7 @@ Explicitly create an object that may exist already. Usually, when paramameters ( A context manager to temporarily use a different container on the same provider storage = Storage(provider, key, secret, container) - + with storage.use(another_container_name) as s3: s3.upload(newfile) @@ -413,7 +413,7 @@ Each object is an instance on **flask_cloudy.Object** storage = Storage(provider, key, secret, container) my_file = "hello.txt" - + if my_file in storage: print("File is in the storage") @@ -428,10 +428,10 @@ Usually, you will get a cloud object by accessing an object in the container. storage = Storage(provider, key, secret, container) my_object = storage.get("my_object.txt") - + Properties: - -#### Object.name + +#### Object.name The name of the object @@ -454,7 +454,7 @@ On LOCAL, it will return the url without the domain name ( ie: /files/my-file.jp For cloud providers it will return the full url -#### Object.full_url +#### Object.full_url Returns the full url of the object @@ -463,27 +463,27 @@ Specially for LOCAL provider, it will return the url with the domain. For cloud providers, it will return the full url just like **Object.url** -#### Object.secure_url +#### Object.secure_url -Return a secured url, with **https://** +Return a secured url, with **https://** #### Object.path The path of the object relative to the container - + #### Object.full_path For Local, it will show the full path of the object, otherwise it just returns the Object.path - -#### Object.provider_name + +#### Object.provider_name The provider name: ie: Local, S3,... -#### Object.type +#### Object.type The type of the object, ie: IMAGE, AUDIO, TEXT,... OTHER @@ -496,7 +496,7 @@ Methods: #### Object.save_to(destination, name=None, overwrite=False, delete_on_failure=True) -To save the object to a local path +To save the object to a local path - destination: The directory to save the object to @@ -512,7 +512,7 @@ To save the object to a local path my_object = storage.get("my_object.txt") my_new_path = "/my/new/path" my_new_file = my_object.save_to(my_new_path) - + print(my_new_file) # Will print -> /my/new/path/my_object.txt @@ -528,8 +528,8 @@ Return a URL that triggers the browser download of the file. On cloud providers storage = Storage(provider, key, secret, container) my_object = storage.get("my_object.txt") - download_url = my_object.download_url() - + download_url = my_object.download_url() + # or with flask @app.route("/download/"): @@ -538,17 +538,32 @@ Return a URL that triggers the browser download of the file. On cloud providers if my_object: download_url = my_object.download_url() return redirect(download_url) - else: + else: abort(404, "File doesn't exist") - + +--- + +## Tests + +To run the tests you should install tox package into your virtual environment. + +By default the tests are configured for LOCAL provider, so you will need to: + + mkdir tests/container_1 tests/container_2 + +Then run: + + tox + +Highly recommended to clean these directories before tox running. + --- I hope you find this library useful, enjoy! -Mardix :) +Mardix :) --- License: MIT - Copyright 2017 Mardix - diff --git a/flask_cloudy.py b/flask_cloudy.py index ab9313a..890cfc6 100644 --- a/flask_cloudy.py +++ b/flask_cloudy.py @@ -10,7 +10,6 @@ import hashlib import warnings from contextlib import contextmanager -import copy from werkzeug.utils import secure_filename from werkzeug.datastructures import FileStorage from importlib import import_module @@ -36,7 +35,8 @@ "AUDIO": ["wav", "mp3", "aac", "ogg", "oga", "flac"], "DATA": ["csv", "ini", "json", "plist", "xml", "yaml", "yml"], "SCRIPT": ["js", "php", "pl", "py", "rb", "sh"], - "ARCHIVE": ["gz", "bz2", "zip", "tar", "tgz", "txz", "7z"] + "ARCHIVE": ["gz", "bz2", "zip", "tar", "tgz", "txz", "7z"], + "CAD": ["stl", "dxf"], } ALL_EXTENSIONS = EXTENSIONS["TEXT"] \ @@ -44,13 +44,16 @@ + EXTENSIONS["IMAGE"] \ + EXTENSIONS["AUDIO"] \ + EXTENSIONS["DATA"] \ - + EXTENSIONS["ARCHIVE"] + + EXTENSIONS["ARCHIVE"] \ + + EXTENSIONS["CAD"] URL_REGEXP = re.compile(r'^(http|https|ftp|ftps)://') + class InvalidExtensionError(Exception): pass + def get_file_name(filename): """ Return the filename without the path @@ -59,6 +62,7 @@ def get_file_name(filename): """ return os.path.basename(filename) + def get_file_extension(filename): """ Return a file extension @@ -67,6 +71,7 @@ def get_file_extension(filename): """ return os.path.splitext(filename)[1][1:].lower() + def get_file_extension_type(filename): """ Return the group associated to the file @@ -80,6 +85,7 @@ def get_file_extension_type(filename): return name return "OTHER" + def get_driver_class(provider): """ Return the driver class @@ -100,6 +106,7 @@ def get_driver_class(provider): driver = getattr(Provider, provider.upper()) return get_driver(driver) + def get_provider_name(driver): """ Return the provider name from the driver class @@ -125,6 +132,7 @@ class Storage(object): DATA = EXTENSIONS["DATA"] SCRIPT = EXTENSIONS["SCRIPT"] ARCHIVE = EXTENSIONS["ARCHIVE"] + CAD = EXTENSIONS["CAD"] allowed_extensions = TEXT + DOCUMENT + IMAGE + AUDIO + DATA @@ -304,7 +312,7 @@ def upload(self, To upload file :param file: FileStorage object or string location :param name: The name of the object. - :param prefix: A prefix for the object. Can be in the form of directory tree + :param prefix: A prefix for the object. Can be in the form of directory tree or generator. :param extensions: list of extensions to allow. If empty, it will use all extension. :param overwrite: bool - To overwrite if file exists :param public: bool - To set acl to private or public-read. Having acl in kwargs will override it @@ -348,7 +356,10 @@ def upload(self, name = secure_filename(name) if prefix: - name = prefix.lstrip("/") + name + if callable(prefix): + name = prefix().lstrip("/") + name + else: + name = prefix.lstrip("/") + name if not overwrite: name = self._safe_object_name(name) @@ -486,7 +497,7 @@ def info(self): def get_url(self, secure=False, longurl=False): """ - Return the url + Return the url :param secure: bool - To use https :param longurl: bool - On local, reference the local path with the domain ie: http://site.com/files/object.png otherwise /files/object.png @@ -638,8 +649,7 @@ def download_url(self, timeout=60, name=None): _external=True) else: driver_name = self.driver.name.lower() - expires = (datetime.datetime.now() - + datetime.timedelta(seconds=timeout)).strftime("%s") + expires = (datetime.datetime.now() + datetime.timedelta(seconds=timeout)).strftime("%s") if 's3' in driver_name or 'google' in driver_name: @@ -658,8 +668,8 @@ def download_url(self, timeout=60, name=None): elif 'cloudfiles' in driver_name: return self.driver.ex_get_object_temp_url(self._obj, - method="GET", - timeout=expires) + method="GET", + timeout=expires) else: raise NotImplemented("This provider '%s' doesn't support or " "doesn't have a signed url " diff --git a/tests/config.py b/tests/config.py index 532a195..2300f2a 100644 --- a/tests/config.py +++ b/tests/config.py @@ -8,4 +8,4 @@ # FOR LOCAL PROVIDER = "LOCAL" CONTAINER = "container_1" -CONTAINER2 = "container_2" \ No newline at end of file +CONTAINER2 = "container_2" diff --git a/tests/test_cloudy.py b/tests/test_cloudy.py index 964d4fb..e36c7ad 100644 --- a/tests/test_cloudy.py +++ b/tests/test_cloudy.py @@ -2,13 +2,13 @@ import pytest from libcloud.storage.base import (StorageDriver, Container) from flask_cloudy import (get_file_extension, - get_file_extension_type, - get_file_name, - get_driver_class, - get_provider_name, - Storage, - Object, - InvalidExtensionError) + get_file_extension_type, + get_file_name, + get_driver_class, + get_provider_name, + Storage, + Object, + InvalidExtensionError) from tests import config CWD = os.path.dirname(__file__) @@ -16,6 +16,7 @@ CONTAINER = "%s/%s" % (CWD, config.CONTAINER) if config.PROVIDER == "LOCAL" else config.CONTAINER CONTAINER2 = "%s/%s" % (CWD, config.CONTAINER2) if config.PROVIDER == "LOCAL" else config.CONTAINER2 + class App(object): config = dict( STORAGE_PROVIDER=config.PROVIDER, @@ -30,59 +31,71 @@ def test_get_file_extension(): filename = "hello.jpg" assert get_file_extension(filename) == "jpg" + def test_get_file_extension_type(): filename = "hello.mp3" assert get_file_extension_type(filename) == "AUDIO" + def test_get_file_name(): filename = "/dir1/dir2/dir3/hello.jpg" assert get_file_name(filename) == "hello.jpg" + def test_get_provider_name(): class GoogleStorageDriver(object): pass driver = GoogleStorageDriver() assert get_provider_name(driver) == "google_storage" -#--- +# --- app = App() + def app_storage(): return Storage(app=App()) + def test_get_driver_class(): driver = get_driver_class("S3") assert isinstance(driver, type) + def test_driver(): storage = app_storage() assert isinstance(storage.driver, StorageDriver) + def test_container(): storage = app_storage() assert isinstance(storage.container, Container) + def test_flask_app(): storage = app_storage() assert isinstance(storage.driver, StorageDriver) + def test_iter(): storage = app_storage() l = [o for o in storage] assert isinstance(l, list) + def test_storage_object_not_exists(): object_name = "hello.png" storage = app_storage() assert object_name not in storage + def test_storage_object(): object_name = "hello.txt" storage = app_storage() o = storage.create(object_name) assert isinstance(o, Object) + def test_object_type_extension(): object_name = "hello.jpg" storage = app_storage() @@ -90,12 +103,14 @@ def test_object_type_extension(): assert o.type == "IMAGE" assert o.extension == "jpg" + def test_object_provider_name(): object_name = "hello.jpg" storage = app_storage() o = storage.create(object_name) assert o.provider_name == config.PROVIDER.lower() + def test_object_object_path(): object_name = "hello.jpg" storage = app_storage() @@ -103,12 +118,14 @@ def test_object_object_path(): p = "%s/%s" % (o.container.name, o.name) assert o.path.endswith(p) + def test_storage_upload_invalid(): storage = app_storage() object_name = "my-js/hello.js" with pytest.raises(InvalidExtensionError): storage.upload(CWD + "/data/hello.js", name=object_name) + def test_storage_upload_ovewrite(): storage = app_storage() object_name = "my-txt-hello.txt" @@ -116,6 +133,7 @@ def test_storage_upload_ovewrite(): assert isinstance(o, Object) assert o.name == object_name + def test_storage_get(): storage = app_storage() object_name = "my-txt-helloIII.txt" @@ -123,11 +141,13 @@ def test_storage_get(): o2 = storage.get(o.name) assert isinstance(o2, Object) + def test_storage_get_none(): storage = app_storage() o2 = storage.get("idonexist") assert o2 is None + def test_storage_upload(): storage = app_storage() object_name = "my-txt-hello2.txt" @@ -136,18 +156,21 @@ def test_storage_upload(): assert isinstance(o, Object) assert o.name != object_name + def test_storage_upload_use_filename_name(): storage = app_storage() object_name = "hello.js" o = storage.upload(CWD + "/data/hello.js", overwrite=True, extensions=["js"]) assert o.name == object_name + def test_storage_upload_append_extension(): storage = app_storage() object_name = "my-txt-hello-hello" o = storage.upload(CWD + "/data/hello.txt", object_name, overwrite=True) assert get_file_extension(o.name) == "txt" + def test_storage_upload_with_prefix(): storage = app_storage() object_name = "my-txt-hello-hello" @@ -158,6 +181,19 @@ def test_storage_upload_with_prefix(): assert o.name == full_name +def test_storage_upload_with_prefix_function(): + def upload_to(): + tree = ["dir1", "dir2", "dir3"] + return "/".join(tree) + "/" + + storage = app_storage() + object_name = "my-txt-hello-hello" + full_name = "%s%s.%s" % (upload_to(), object_name, "txt") + o = storage.upload(CWD + "/data/hello.txt", name=object_name, prefix=upload_to, overwrite=True) + assert full_name in storage + assert o.name == full_name + + def test_save_to(): storage = app_storage() object_name = "my-txt-hello-to-save.txt" @@ -167,6 +203,7 @@ def test_save_to(): assert os.path.isfile(file) assert file2 == CWD + "/data/my_new_file.txt" + def test_delete(): storage = app_storage() object_name = "my-txt-hello-to-delete.txt" @@ -175,6 +212,7 @@ def test_delete(): o.delete() assert object_name not in storage + def test_use(): storage = app_storage() object_name = "my-txt-hello-to-save-with-use.txt" @@ -187,6 +225,7 @@ def test_use(): assert isinstance(o1, Object) assert o1.name == object_name + def test_werkzeug_upload(): try: import werkzeug @@ -207,7 +246,8 @@ def test_werkzeug_upload(): def test_random(): storage = app_storage() o = storage.upload(CWD + "/data/hello.js", overwrite=True, extensions=["js"], random_name=True) - assert len(o.name) == 32 + 3 # 3 extensions + assert len(o.name) == 32 + 3 # 3 extensions + def test_upload_image_from_url(): storage = app_storage() diff --git a/tox.ini b/tox.ini index b3cd340..36a776a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py26,py27 +envlist = py27 [testenv] deps=pytest -commands=py.test \ No newline at end of file +commands=py.test