diff --git a/config/environments/development.rb b/config/environments/development.rb index 9d411205a7..d2ab5c928a 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -34,7 +34,7 @@ end # Store uploaded files on the local file system (see config/storage.yml for options). - config.active_storage.service = :local + config.active_storage.service = :generic_s3 # Don't care if the mailer can't send. config.action_mailer.raise_delivery_errors = false diff --git a/config/settings.yml b/config/settings.yml index ae96bcb0b1..3ddbc09e0b 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -6,10 +6,8 @@ domain: protocol: http # End UMD Customization dropbox: - # UMD Customization - path: /masterfiles/dropbox - upload_uri: /masterfiles/dropbox - # End UMD Customization + path: /srv/avalon/dropbox + upload_uri: sftp://avalon.example.edu # google_drive: # client_id: id # client_secret: secret diff --git a/docker-compose.yml b/docker-compose.yml index f6b36de338..6a50cca7be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,10 +8,7 @@ volumes: fedora: solr: npms: - # UMD Customization - # data: - work: - # End UMD Customization + data: networks: internal: @@ -69,12 +66,12 @@ services: hls: # UMD Customization - image: docker.lib.umd.edu/nginx:avalon-7.6-umd-0 + image: docker.lib.umd.edu/nginx:avalon-7.6-umd-LIBAVALON-373 platform: linux/amd64 # End UMD Customization environment: - AVALON_DOMAIN=http://avalon:3000 - - AVALON_STREAMING_BASE_URL=http://localhost:8880 + - AVALON_STREAMING_BUCKET_URL=http://minio:9000/derivatives/ # UMD Customization volumes: - streaming:/data @@ -82,6 +79,7 @@ services: ports: - '8880:80' networks: + internal: external: redis: &redis @@ -105,6 +103,7 @@ services: - solr - redis - hls + - minio environment: # UMD Customization - APP_NAME=avalon @@ -112,23 +111,38 @@ services: - SETTINGS__DOMAIN_HOST=av-local - SETTINGS__DOMAIN_PORT=3000 - BUNDLE_FLAGS=--with development postgres --without production test - - ENCODE_WORK_DIR=/streamfiles + - ENCODE_WORK_DIR=/tmp - CONTROLLED_VOCABULARY=config/controlled_vocabulary.yml - DATABASE_URL=postgres://postgres:password@db/avalon - DIGITAL_COLLECTIONS_URL=https://digital.lib.umd.edu/ - DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL=true - FEDORA_NAMESPACE=avalon - FEDORA_URL=http://fedoraAdmin:fedoraAdmin@fedora:8080/fedora/rest + - DIGITAL_COLLECTIONS_URL=https://digital.lib.umd.edu/rest - RAILS_ENV=development - RAILS_ADDITIONAL_HOSTS=avalon - SETTINGS__REDIS__HOST=redis - SETTINGS__REDIS__PORT=6379 - SECRET_KEY_BASE=abcd - SOLR_URL=http://solr:8983/solr/avalon - - SETTINGS__STREAMING__CONTENT_PATH=/streamfiles + # - SETTINGS__STREAMING__CONTENT_PATH=/streamfiles + - AWS_REGION=us-east-1 + - SETTINGS__ACTIVE_STORAGE__BUCKET=supplementalfiles + - SETTINGS__ACTIVE_STORAGE__SERVICE=generic_s3 - SETTINGS__FFMPEG__PATH=/usr/bin/ffmpeg + - SETTINGS__MINIO__ENDPOINT=http://minio:9000 + - SETTINGS__MINIO__PUBLIC_HOST=http://localhost:9000 + - SETTINGS__MINIO__ACCESS=minio + - SETTINGS__MINIO__SECRET=minio123 + - SETTINGS__ENCODING__MASTERFILE_BUCKET=masterfiles + - SETTINGS__ENCODING__DERIVATIVE_BUCKET=derivatives + - SETTINGS__DROPBOX__PATH=s3://masterfiles/dropbox/ + - SETTINGS__DROPBOX__UPLOAD_URI=s3://masterfiles/dropbox/ + - SETTINGS__MASTER_FILE_MANAGEMENT__PATH=s3://masterfiles/archive/ + - SETTINGS__MASTER_FILE_MANAGEMENT__STRATEGY=move + - SETTINGS__STREAMING__CONTENT_PATH=/ - SETTINGS__STREAMING__STREAM_DEFAULT_QUALITY=medium - - SETTINGS__STREAMING__HTTP_BASE=http://localhost:8880/avalon + - SETTINGS__STREAMING__HTTP_BASE=http://localhost:8880/s3-avalon - SETTINGS__STREAMING__SERVER=nginx - SETTINGS__STREAMING__STREAM_TOKEN_TTL=20 - SYSTEM_GROUPS=administrator,group_manager,manager @@ -179,41 +193,44 @@ services: <<: *avalon command: dumb-init -- bash -c "bundle install && bundle exec sidekiq -C config/sidekiq.yml" ports: [] -# UMD Customization - # minio: - # image: minio/minio:RELEASE.2019-10-12T01-39-57Z - # command: minio server /data - # environment: - # MINIO_ACCESS_KEY: minio - # MINIO_SECRET_KEY: minio123 - # volumes: - # - data:/data - # ports: - # - 9000:9000 - # networks: - # internal: - # external: - # healthcheck: - # test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] - # interval: 30s - # timeout: 20s - # retries: 3 - # createbuckets: - # image: minio/mc - # depends_on: - # - minio - # entrypoint: > - # /bin/sh -c " - # /usr/bin/mc config host add myminio http://minio:9000 minio minio123; - # /usr/bin/mc mb -p myminio/fcrepo myminio/masterfiles myminio/derivatives myminio/supplementalfiles; - # /usr/bin/mc policy set download myminio/derivatives; - # /usr/bin/mc policy set download myminio/supplementalfiles; - # exit 0; - # " - # networks: - # internal: -# End UMD Customization + # UMD Customization + # Ported in changes from Avalon 7.8 + minio: + image: minio/minio:RELEASE.2024-01-29T03-56-32Z + command: minio server /data --console-address ":9090" + environment: + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + volumes: + - data:/data + ports: + - 9000:9000 + - 9090:9090 + networks: + internal: + external: + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minio minio123; + /usr/bin/mc mb -p myminio/fcrepo myminio/masterfiles myminio/derivatives myminio/supplementalfiles myminio/preserves; + /usr/bin/mc anonymous set download myminio/derivatives; + /usr/bin/mc anonymous set download myminio/supplementalfiles; + exit 0; + " + networks: + internal: + # End UMD Customization cypress: network_mode: host diff --git a/umd_docs/DockerDevelopmentEnvironment.md b/umd_docs/DockerDevelopmentEnvironment.md index 163bcd28b3..6beb1ef683 100644 --- a/umd_docs/DockerDevelopmentEnvironment.md +++ b/umd_docs/DockerDevelopmentEnvironment.md @@ -17,6 +17,12 @@ of Avalon using Docker. 127.0.0.1 av-local ``` +3) Install the Minio Client + + ```zsh + brew install minio-mc + ``` + ### Setup Instructions 1) Checkout the application and switch to the directory: @@ -93,6 +99,16 @@ of Avalon using Docker. Avalon should be available at: [http://av-local:3000](http://av-local:3000) + The MinIO browser should be accessible at + +9) In a separate terminal, run the createbuckets container to create the + necessary buckets in MinIO + + ```zsh + mc alias set minio http://localhost:9000 minio minio123; + docker-compose up createbuckets + ``` + ### Loading Sample Data Sample data for Avalon is available in the @@ -117,49 +133,44 @@ in the folder for a description of each dataset. then left-click the "Create Collection" button in the dialog. Once created, the "Manage Content" page with the collection will be displayed. Also, a - "masterfiles/dropbox/Test_Collection/" subdirectory will be created in the - Avalon project directory. + "dropbox/Test_Collection/" folder will be created in the masterfiles bucket. 3) In a terminal, add a `sample-data@example.com` admin user (which is the - email address of the submitter in the sample datasets) by executing a Bash - shell in the "avalon-avalon-1" Docker container: - - ```zsh - docker exec -it avalon-avalon-1 /bin/bash - ``` - - and running the following command: - - ```zsh - rails avalon:user:create \ - avalon_username=sample-data@example.com avalon_password=PASSWORD \ - avalon_groups=administrator - ``` - - then exit the Docker container: + email address of the submitter in the sample datasets) by running the + following command: ```zsh - exit + docker exec -it avalon-avalon-1 /bin/bash -c \ + 'rails avalon:user:create \ + avalon_username=sample-data@example.com avalon_password=PASSWORD \ + avalon_groups=administrator' ``` 4) Download the "Sample_Audio_and_Video" folder (as a Zip file) from - and place it in the - "masterfiles/dropbox/Test_Collection/" folder. In a terminal, - switch to the "masterfiles/dropbox/Test_Collection/" subdirectory: + and place it in a + temporary folder. In a terminal, switch to the directory containing + the download and extract the file. ```zsh - cd masterfiles/dropbox/Test_Collection/ + cd + unzip Sample_Audio_and_Video.zip ``` - and extract the file: + Copy the files to the masterfiles bucket dropbox folder. In the following + the "UPLOAD_DIR" environment variable contains the name of the extracted + folder from the Zip file (to simplify uploading different sample datasets). - ```zsh - unzip Sample_Audio_and_Video.zip + ```sh + export UPLOAD_DIR=Sample_Audio_and_Video + mc mirror --exclude "*.xlsx" $UPLOAD_DIR minio/masterfiles/dropbox/Test_Collection/$UPLOAD_DIR/ + mc cp -r $UPLOAD_DIR/*.xlsx minio/masterfiles/dropbox/Test_Collection/$UPLOAD_DIR/ ``` - and extract the file. + **Note:** Uploading the assets and the "xlsx" manifest file separately is a + best practice, because it ensures that the Avalon "worker" process won't + start the import before the asset files are available. -5) The "avalon-worker" container scans the "masterfiles/dropbox" directory +5) The "avalon-worker" container scans the "masterfiles/dropbox" bucket once a minute, and ingests any new items found. Depending on the size of the dataset, and the need to transcode the content, the ingest may take several minutes.