Better channel restoration for local dev with content proxying

.run/devserver.run.xml

@@ -13,7 +13,7 @@
     <option name="ADD_SOURCE_ROOTS" value="false" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
     <option name="SCRIPT_NAME" value="contentcuration/manage.py" />
-    <option name="PARAMETERS" value="runserver --settings=contentcuration.dev_settings 0.0.0.0:8080" />
+    <option name="PARAMETERS" value="runserver --settings=contentcuration.dev_settings 0.0.0.0:8081" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="false" />

Makefile

@@ -200,7 +200,7 @@ dctest: .docker/minio .docker/postgres
 
 dcservicesup: .docker/minio .docker/postgres
 	# launch all studio's dependent services using docker-compose
-	$(DOCKER_COMPOSE) -f docker-compose.yml -f docker-compose.alt.yml up minio postgres redis
+	$(DOCKER_COMPOSE) -f docker-compose.yml -f docker-compose.alt.yml up minio postgres redis studio-nginx
 
 dcservicesdown:
 	# stop services that were started using dcservicesup

contentcuration/contentcuration/management/commands/restore_channel.py

@@ -2,27 +2,65 @@
 
 from django.core.management.base import BaseCommand
 
-from contentcuration.utils.import_tools import import_channel
+from contentcuration.utils.import_tools import ImportManager
 
 logger = logging.getLogger("command")
 
 
 class Command(BaseCommand):
+    """
+    This command is used to restore a channel from another Studio instance. This is for
+    development purposes only and should not be used in production.
+    """
+
     def add_arguments(self, parser):
         # ID of channel to read data from
         parser.add_argument("source_id", type=str)
 
         # ID of channel to write data to (can be same as source channel)
-        parser.add_argument("--target", help="restore channel db to TARGET CHANNEL ID")
-        parser.add_argument("--download-url", help="where to download db from")
-        parser.add_argument("--editor", help="add user as editor to channel")
+        parser.add_argument(
+            "--target",
+            help="A different channel ID for which to restore the channel. If not provided, the source channel ID will be used.",
+        )
+        parser.add_argument(
+            "--source-url",
+            default="http://localhost:8080",
+            help="Studio instance from which to download the channel DB or content files",
+        )
+        parser.add_argument("--token", help="API token for the Studio instance")
+        parser.add_argument(
+            "--editor",
+            default="a@a.com",
+            help="Add user as editor to channel with provided email address",
+        )
+        parser.add_argument(
+            "--download-content",
+            action="store_true",
+            default=False,
+            help="Whether to download content files",
+        )
+        parser.add_argument(
+            "--public",
+            action="store_true",
+            default=False,
+            help="Whether to make the channel public",
+        )
+        parser.add_argument(
+            "--publish",
+            action="store_true",
+            default=False,
+            help="Whether to publish the channel after restoration",
+        )
 
     def handle(self, *args, **options):
-        # Set up variables for restoration process
-        logger.info("\n\n********** STARTING CHANNEL RESTORATION **********")
-        source_id = options["source_id"]
-        target_id = options.get("target") or source_id
-        download_url = options.get("download_url")
-        editor = options.get("editor")
-
-        import_channel(source_id, target_id, download_url, editor, logger=logger)
+        manager = ImportManager(
+            options["source_url"],
+            options["source_id"],
+            target_id=options.get("target"),
+            editor=options.get("editor"),
+            public=options.get("public"),
+            publish=options.get("publish"),
+            token=options.get("token"),
+            download_content=options.get("download_content"),
+        )
+        manager.run()

contentcuration/contentcuration/management/commands/set_content_mimetypes.py

@@ -14,7 +14,7 @@
 from django.core.files.storage import default_storage
 from django.core.management.base import BaseCommand
 
-from contentcuration.utils.storage_common import determine_content_type
+from contentcuration.utils.storage.common import determine_content_type
 
 
 class Command(BaseCommand):

contentcuration/contentcuration/models.py

@@ -2,7 +2,6 @@
 import json
 import logging
 import os
-import urllib.parse
 import uuid
 from datetime import datetime
 
@@ -671,44 +670,10 @@ def generate_storage_url(filename, request=None, *args):
 
     path = generate_object_storage_name(os.path.splitext(filename)[0], filename)
 
-    # There are three scenarios where Studio might be run as:
-    #
-    # 1. In normal kubernetes, nginx will proxy for us. We'll know we're in kubernetes when the
-    # environment variable RUN_MODE=k8s
-    #
-    # 2. In Docker Compose and bare metal runserver, we'll be running in runserver, and minio
-    # will be exposed in port 9000 in the host's localhost network.
-
-    # Note (aron): returning the true storage URL (e.g. https://storage.googleapis.com/storage/a.mp4)
-    # isn't too important, because we have CDN in front of our servers, so it should be cached.
-    # But change the logic here in case there is a potential for bandwidth and latency improvement.
-
-    # Detect our current state first
-    run_mode = os.getenv("RUN_MODE")
-
-    # if we're running inside k8s, then just serve the normal /content/{storage,databases} URL,
-    # and let nginx handle proper proxying.
-    if run_mode == "k8s":
-        url = "/content/{path}".format(
-            path=path,
-        )
-
-    # if we're in docker-compose or in baremetal, just return the object storage URL as localhost:9000
-    elif run_mode == "docker-compose" or run_mode is None:
-        # generate the minio storage URL, so we can get the GET parameters that give everyone
-        # access even if they don't need to log in
-        params = urllib.parse.urlparse(default_storage.url(path)).query
-        host = "localhost"
-        port = 9000  # hardcoded to the default minio IP address
-        url = "http://{host}:{port}/{bucket}/{path}?{params}".format(
-            host=host,
-            port=port,
-            bucket=settings.AWS_S3_BUCKET_NAME,
-            path=path,
-            params=params,
-        )
-
-    return url
+    # requires that we always have a proxy of /content to storage bucket, handled by nginx in dev
+    return "/content/{path}".format(
+        path=path,
+    )
 
 
 class FileOnDiskStorage(FileSystemStorage):

contentcuration/contentcuration/production_settings.py

@@ -10,7 +10,7 @@
 
 MEDIA_ROOT = base_settings.STORAGE_ROOT
 
-DEFAULT_FILE_STORAGE = "contentcuration.utils.gcs_storage.CompositeGCS"
+DEFAULT_FILE_STORAGE = "contentcuration.utils.storage.gcs.CompositeGCS"
 SESSION_ENGINE = "django.contrib.sessions.backends.db"
 
 # email settings

contentcuration/contentcuration/sandbox_settings.py

@@ -3,7 +3,7 @@
 
 DEBUG = True
 
-DEFAULT_FILE_STORAGE = "contentcuration.utils.gcs_storage.CompositeGCS"
+DEFAULT_FILE_STORAGE = "contentcuration.utils.storage.gcs.CompositeGCS"
 
 LANGUAGES += (("ar", gettext("Arabic")),)  # noqa
 

contentcuration/contentcuration/settings.py

@@ -357,7 +357,7 @@ def gettext(s):
 ORPHAN_DATE_CLEAN_UP_THRESHOLD = TWO_WEEKS_AGO
 
 # CLOUD STORAGE SETTINGS
-DEFAULT_FILE_STORAGE = "django_s3_storage.storage.S3Storage"
+DEFAULT_FILE_STORAGE = "contentcuration.utils.storage.dev.CompositeStorage"
 AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") or "development"
 AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") or "development"
 AWS_S3_BUCKET_NAME = os.getenv("AWS_BUCKET_NAME") or "content"