Implemented S3 client for mediaserver #39

bridge/config/config.go

@@ -216,6 +216,13 @@ type Protocol struct {
 	VerboseJoinPart        bool       // IRC
 	WebhookBindAddress     string     // mattermost, slack
 	WebhookURL             string     // mattermost, slack
+
+	S3Endpoint       string // general, mediaserver configuration
+	S3AccessKey      string // general, mediaserver configuration
+	S3SecretKey      string // general, mediaserver configuration
+	S3ForcePathStyle bool   // general, mediaserver configuration
+	S3Presign        bool   // general, mediaserver configuration
+	S3Region         string // general, mediaserver configuration
 }
 
 type ChannelOptions struct {

docs/advanced/mediaserver.md

@@ -6,10 +6,44 @@ There are 2 options to set this up:
 * You already have a webserver running
    * Matterbridge runs on the same server see [local download](#use-local-download)
    * Matterbridge runs on another server. If the webserver is using caddy, see [caddy](#use-remote-upload)
+* You already have a S3-compatible storage, then you could use [S3](#s3-minio--remote-upload-using-s3-compatible-storage)
 * You don't have a webserver running
    * See [caddy](#use-remote-upload)
 
 # Use remote upload
+
+# S3 (MinIO) — remote upload using S3-compatible storage
+
+Matterbridge can upload media to S3-compatible object stores. This is useful when you want a hosted, scalable Mediaserver and you have (or run) an S3 endpoint such as MinIO.
+
+Key points
+* Use an s3:// bucket path for uploads; Matterbridge will put objects into that bucket and return URLs based on MediaServerDownload (or the S3 public endpoint). Additional part in s3:// are treated as a prefix to a file (same behaviour as in `awscli-v2`)
+* For MinIO you normally enable path-style requests and provide the endpoint + credentials.
+
+Sample matterbridge configuration (in [general])
+```
+[general]
+# tell matterbridge to upload to the bucket
+MediaServerUpload="s3://matterbridge"
+# public URL base where objects will be served from (path style, this will be treated as a prefix to URL)
+MediaServerDownload="https://minio.example.com/matterbridge"
+
+# S3 / MinIO connection settings (common names used by many S3 clients)
+S3Endpoint="https://minio.example.com:9000"
+S3AccessKey="minioadmin"
+S3SecretKey="minioadmin"
+# MinIO typically requires path style for bucket paths
+S3ForcePathStyle=true
+# To use presigned URLs instead of public buckets. Presigned URL will be valid for 7 days.
+# when using this setting MediaServerDownload is ignored.
+# Please note that this produces awful, long links.
+S3Presign=false
+```
+
+Notes and recommendations
+* Public buckets are required for links to work, since users need to have permission to read files that were submitted by matterbridge.
+* Adjust credentials and endpoints for your environment. For MinIO on a nonstandard port or local testing, use the correct host:port in S3Endpoint and in MediaServerDownload.
+
 ## Caddy
 In this case we're using caddy for upload/downloading media. 
 Caddy has automatic https support, so I'm going to describe this for https only.

gateway/gateway.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -22,6 +23,7 @@ type Gateway struct {
 	config.Config
 
 	Router         *Router
+	mediaServer    mediaServer
 	MyConfig       *config.Gateway
 	Bridges        map[string]*bridge.Bridge
 	Channels       map[string]*config.ChannelInfo
@@ -46,15 +48,28 @@ const apiProtocol = "api"
 func New(rootLogger *logrus.Logger, cfg *config.Gateway, r *Router) *Gateway {
 	logger := rootLogger.WithFields(logrus.Fields{"prefix": "gateway"})
 
+	mediaServerInstance, err := createMediaServer(r.BridgeValues(), rootLogger.WithField("prefix", "mediaserver"))
+	if err != nil {
+		if errors.Is(err, ErrMediaConfigurationNotWanted) {
+			// media server not wanted, ignore
+			mediaServerInstance = nil
+
+			logger.Info("Media server not configured, skipping media server setup")
+		} else {
+			logger.WithError(err).Error("Failed to configure media server for gateway")
+		}
+	}
+
 	cache, _ := lru.New(5000)
 	gw := &Gateway{
-		Channels: make(map[string]*config.ChannelInfo),
-		Message:  r.Message,
-		Router:   r,
-		Bridges:  make(map[string]*bridge.Bridge),
-		Config:   r.Config,
-		Messages: cache,
-		logger:   logger,
+		Channels:    make(map[string]*config.ChannelInfo),
+		Message:     r.Message,
+		Router:      r,
+		Bridges:     make(map[string]*bridge.Bridge),
+		Config:      r.Config,
+		Messages:    cache,
+		logger:      logger,
+		mediaServer: mediaServerInstance,
 	}
 	if err := gw.AddConfig(cfg); err != nil {
 		logger.Errorf("Failed to add configuration to gateway: %#v", err)

gateway/handlers.go

@@ -1,16 +1,11 @@
 package gateway
 
 import (
-	"bytes"
 	"crypto/sha1" //nolint:gosec
 	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/matterbridge-org/matterbridge/bridge"
 	"github.com/matterbridge-org/matterbridge/bridge/config"
@@ -73,8 +68,7 @@ func (gw *Gateway) handleFiles(msg *config.Message) {
 
 	// If we don't have a attachfield or we don't have a mediaserver configured return
 	if msg.Extra == nil ||
-		(gw.BridgeValues().General.MediaServerUpload == "" &&
-			gw.BridgeValues().General.MediaDownloadPath == "") {
+		gw.mediaServer == nil {
 		return
 	}
 
@@ -92,78 +86,23 @@ func (gw *Gateway) handleFiles(msg *config.Message) {
 
 		sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
 
-		if gw.BridgeValues().General.MediaServerUpload != "" {
-			// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
-			if err := gw.handleFilesUpload(&fi); err != nil {
-				gw.logger.Error(err)
-				continue
-			}
-		} else {
-			// Use MediaServerPath. Place the file on the current filesystem.
-			if err := gw.handleFilesLocal(&fi); err != nil {
-				gw.logger.Error(err)
-				continue
-			}
+		downloadURL, err := gw.mediaServer.handleFilesUpload(&fi)
+		if err != nil {
+			gw.logger.Error(err)
+			continue
 		}
 
 		// Download URL.
-		durl := gw.BridgeValues().General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name
-
-		gw.logger.Debugf("mediaserver download URL = %s", durl)
+		gw.logger.Debugf("mediaserver download URL = %s", downloadURL)
 
 		// We uploaded/placed the file successfully. Add the SHA and URL.
 		extra := msg.Extra["file"][i].(config.FileInfo)
-		extra.URL = durl
+		extra.URL = downloadURL
 		extra.SHA = sha1sum
 		msg.Extra["file"][i] = extra
 	}
 }
 
-// handleFilesUpload uses MediaServerUpload configuration to upload the file.
-// Returns error on failure.
-func (gw *Gateway) handleFilesUpload(fi *config.FileInfo) error {
-	client := &http.Client{
-		Timeout: time.Second * 5,
-	}
-	// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
-	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
-	url := gw.BridgeValues().General.MediaServerUpload + "/" + sha1sum + "/" + fi.Name
-
-	req, err := http.NewRequest("PUT", url, bytes.NewReader(*fi.Data))
-	if err != nil {
-		return fmt.Errorf("mediaserver upload failed, could not create request: %#v", err)
-	}
-
-	gw.logger.Debugf("mediaserver upload url: %s", url)
-
-	req.Header.Set("Content-Type", "binary/octet-stream")
-	_, err = client.Do(req)
-	if err != nil {
-		return fmt.Errorf("mediaserver upload failed, could not Do request: %#v", err)
-	}
-	return nil
-}
-
-// handleFilesLocal use MediaServerPath configuration, places the file on the current filesystem.
-// Returns error on failure.
-func (gw *Gateway) handleFilesLocal(fi *config.FileInfo) error {
-	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
-	dir := gw.BridgeValues().General.MediaDownloadPath + "/" + sha1sum
-	err := os.Mkdir(dir, os.ModePerm)
-	if err != nil && !os.IsExist(err) {
-		return fmt.Errorf("mediaserver path failed, could not mkdir: %s %#v", err, err)
-	}
-
-	path := dir + "/" + fi.Name
-	gw.logger.Debugf("mediaserver path placing file: %s", path)
-
-	err = ioutil.WriteFile(path, *fi.Data, os.ModePerm)
-	if err != nil {
-		return fmt.Errorf("mediaserver path failed, could not writefile: %s %#v", err, err)
-	}
-	return nil
-}
-
 // ignoreEvent returns true if we need to ignore this event for the specified destination bridge.
 func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 	switch event {