Handle XMPP uploads

bridge/xmpp/handler.go

@@ -1,8 +1,11 @@
 package bxmpp
 
 import (
+	"net/url"
+	"path"
+
 	"github.com/matterbridge-org/matterbridge/bridge/config"
-	"github.com/matterbridge-org/matterbridge/bridge/helper"
+	"github.com/rs/xid"
 	"github.com/xmppo/go-xmpp"
 )
 
@@ -19,16 +22,131 @@ func (b *Bxmpp) handleDownloadAvatar(avatar xmpp.AvatarData) {
 		Event:    config.EventAvatarDownload,
 		Extra:    make(map[string][]interface{}),
 	}
-	if _, ok := b.avatarMap[avatar.From]; !ok {
+
+	// TODO: why do we check if the avatar is already set?
+	// Can't we change avatar once set?
+	_, ok := b.avatarMap[avatar.From]
+	if !ok {
 		b.Log.Debugf("Avatar.From: %s", avatar.From)
+		fileName := avatar.From + ".png"
 
-		err := helper.HandleDownloadSize(b.Log, &rmsg, avatar.From+".png", int64(len(avatar.Data)), b.General)
+		err := b.AddAvatarFromBytes(&rmsg, fileName, fileName, "", &avatar.Data)
 		if err != nil {
-			b.Log.Error(err)
+			b.Log.WithError(err).Warnf("Failed to save avatar for %s, ignoring.", avatar.From)
 			return
 		}
-		helper.HandleDownloadData(b.Log, &rmsg, avatar.From+".png", rmsg.Text, "", &avatar.Data, b.General)
-		b.Log.Debugf("Avatar download complete")
+
+		b.Log.Debugf("Avatar download complete: %s", avatar.From)
 		b.Remote <- rmsg
 	}
 }
+
+// handleUploadFile handles native upload of files from other bridges/channels
+//
+// Implementation notes:
+//
+//   - some clients only display a preview when the body is exactly the URL, not only contains it.
+//     https://docs.modernxmpp.org/client/protocol/#communicating-the-url (Gajim/Conversations),
+//     so we need to produce a different message with the caption
+//   - the message body may or may not be different from an attachment's caption, and should
+//     therefore be sent separately:
+//     https://github.com/matterbridge-org/matterbridge/issues/50#issuecomment-3703478547
+//
+// This method does not return an error, because it will log errors as they happen,
+// and keep trying to send the other attachments if a previous one failed.
+func (b *Bxmpp) handleUploadFile(msg *config.Message) {
+	room := msg.Channel + "@" + b.GetString("Muc")
+
+	if msg.Text != "" {
+		// There's a message body. Maybe there's also an attachment caption, but maybe not.
+		// Let's print the body and the sender first, before iterating over attachments.
+		text := msg.Username + msg.Text
+
+		_, err := b.xc.Send(xmpp.Chat{
+			Type:   "groupchat",
+			Remote: room,
+			Text:   text,
+		})
+		if err != nil {
+			b.Log.WithError(err).Warnf("Skipping file announce due to failed body announce %s", text)
+			return
+		}
+	}
+
+	for _, file := range msg.Extra["file"] {
+		fileInfo := file.(config.FileInfo) //nolint: forcetypeassert
+		if fileInfo.URL != "" {
+			// The file already has a URL, either because the origin bridge provided it,
+			// or the file was reuploaded to matterbridge's mediaserver (if enabled).
+			// In this case, no need to reupload the file.
+			b.announceUploadedFile(msg.Channel+"@"+b.GetString("Muc"), msg.Username+fileInfo.Comment, fileInfo.Comment, fileInfo.URL)
+		} else {
+			// The file received from other bridges is just a bunch of bytes in fileInfo.Data
+			// We need to upload it to the XMPP server's HTTP upload component.
+			// This is defined in XEP-0363: https://xmpp.org/extensions/xep-0363.html
+			//
+			// The steps are performed asynchronously:
+			//
+			// 1. Find the server's HTTP upload component (upon login, in HTTP_UPLOAD_DISCO steps)
+			// 2. Request an "upload slot" from the upload component (we are here)
+			// 3. Send a PUT request with the data to the remote HTTP "upload slot" (when receiving the slot)
+			//
+			// Steps 2 and 3 are commented as HTTP_UPLOAD_SLOT
+			fileId := xid.New().String()
+			go b.requestUploadSlot(fileId, &fileInfo, msg.Channel+"@"+b.GetString("Muc"), msg.Username+fileInfo.Comment, fileInfo.Comment)
+		}
+	}
+}
+
+// handleDownloadFile processes file downloads in the background.
+//
+// Returns true if the message was handled, false otherwise.
+//
+// This implements XEP-0066 https://xmpp.org/extensions/xep-0066.html
+func (b *Bxmpp) handleDownloadFile(rmsg *config.Message, v *xmpp.Chat) bool {
+	// Do we have an OOB attachment URL?
+	if v.Oob.Url != "" {
+		// Perform the download in the background
+		go b.handleDownloadFileInner(rmsg, v)
+
+		return true
+	}
+
+	return false
+}
+
+// handleDownloadFileInner is a helper to actually download a remote attachment
+// and announce it to other bridges.
+//
+// It runs in the foreground, and should only be called in a background context
+// to avoid stalling in the main thread.
+//
+// If it encounters any error, it will log the error and skip the message.
+func (b *Bxmpp) handleDownloadFileInner(rmsg *config.Message, v *xmpp.Chat) {
+	parsed_url, err := url.Parse(v.Oob.Url)
+	if err != nil {
+		b.Log.WithError(err).Warnf("Skipping message due to failed parsing of OOB URL %s", v.Oob.Url)
+		return
+	}
+	// We use the last part of the URL's path as filename. This prevents
+	// errors from extra slashes, but might not make sense if for example
+	// the URL is `/download?id=FOO`.
+	// TODO: investigate popular URL naming schemes in XMPP world, or
+	// consider naming the files after their own checksum.
+	fileName := path.Base(parsed_url.Path)
+
+	err = b.AddAttachmentFromURL(rmsg, fileName, "", "", v.Oob.Url)
+	if err != nil {
+		b.Log.WithError(err).Warnf("Skipping message due to failed OOB attachment download %s", v.Oob.Url)
+		return
+	}
+
+	// Special case: because XMPP OOB (mostly) only allows body with the OOB URL, we remove the
+	// body so we don't end up with duplicate information across bridges/channels.
+	rmsg.Text = ""
+
+	b.Log.Debugf("<= Sending message/attachment from %s on %s to gateway", rmsg.Username, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- *rmsg
+}

bridge/xmpp/helpers.go

@@ -1,9 +1,15 @@
 package bxmpp
 
 import (
+	"fmt"
+	"mime"
+	"path"
 	"regexp"
+	"strconv"
+	"time"
 
 	"github.com/matterbridge-org/matterbridge/bridge/config"
+	"github.com/xmppo/go-xmpp"
 )
 
 var pathRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
@@ -28,3 +34,131 @@ func (b *Bxmpp) cacheAvatar(msg *config.Message) string {
 	}
 	return ""
 }
+
+// This method announces a file sharer and optional caption, then advertises the URL
+// for a file attachment.
+//
+// The second argument contains the uploader nickname with the caption, while the third
+// is the raw attachment caption.
+//
+// This method does not error. Errors are logged as warnings.
+func (b *Bxmpp) announceUploadedFile(to string, text string, urlDesc string, urlStr string) {
+	b.Log.Debugf("Announcing uploaded file to %s: text `%s` desc `%s` url `%s`", to, text, urlDesc, urlStr)
+
+	// Send separate message with the username and optional file comment
+	// because we can't have an attachment comment/description.
+	_, err := b.xc.Send(xmpp.Chat{
+		Type:   "groupchat",
+		Remote: to,
+		// This contains the uploader name, and the optional caption
+		Text: text,
+	})
+	if err != nil {
+		b.Log.WithError(err).Warnf("Skipping file announce due to failed sharer announce %s", text)
+		return
+	}
+
+	_, err = b.xc.SendOOB(xmpp.Chat{
+		Type:   "groupchat",
+		Remote: to,
+		Oob: xmpp.Oob{
+			Url: urlStr,
+			// This is the raw caption, if any
+			Desc: urlDesc,
+		},
+	})
+	if err != nil {
+		b.Log.WithError(err).Warnf("Skipping file announce due to failed OOB announce %s", urlStr)
+		return
+	}
+}
+
+func (b *Bxmpp) extractMaxSizeFromX(disco_x *[]xmpp.DiscoX) int64 {
+	for _, x := range *disco_x {
+		for i, field := range x.Field {
+			if field.Var == "max-file-size" {
+				if i > 0 {
+					if x.Field[i-1].Value[0] == "urn:xmpp:http:upload:0" {
+						return b.extractMaxSizeFromXFieldValue(field.Value[0])
+					}
+				}
+			}
+		}
+	}
+
+	b.Log.Debug("No HTTP max upload size found")
+
+	return 0
+}
+
+func (b *Bxmpp) extractMaxSizeFromXFieldValue(value string) int64 {
+	maxFileSize, err := strconv.ParseInt(value, 10, 64)
+	if err != nil {
+		// If the max-file-size can't be parsed, assume it's 0
+		// and log the error.
+		b.Log.Warnf("Failed to parse HTTP max upload size: %s", value)
+		return 0
+	}
+
+	return maxFileSize
+}
+
+// HTTP_UPLOAD_SLOT step 1
+//
+// Request an upload slot from the HTTP upload component, saving the file
+// in the internal upload buffer for later processing.
+//
+// Will stall until the compoennt is advertised by the server, or until a timeout has been reached.
+// This method must therefore be called from a background thread.
+func (b *Bxmpp) requestUploadSlot(fileId string, fileInfo *config.FileInfo, to string, text string, description string) {
+	retry := 0
+
+	httpUploadComponent := ""
+	for httpUploadComponent == "" {
+		retry += 1
+		if retry > 6 {
+			// No need to keep trying, the XMPP server apparently has no HTTP upload
+			// component configured.
+			b.Log.Warn("Abandoning file upload because XMPP server still hasn't advertised an HTTP upload component.")
+			break
+		}
+
+		b.Lock()
+		httpUploadComponent = b.httpUploadComponent
+		b.Unlock()
+
+		// Wait 5 seconds before next attempt
+		time.Sleep(5 * time.Second)
+	}
+
+	reg := regexp.MustCompile(`[^a-zA-Z0-9\+\-\_\.]+`)
+	fileNameEscaped := reg.ReplaceAllString(fileInfo.Name, "_")
+
+	// Guess the mime-type
+	mimeType := mime.TypeByExtension(path.Ext(fileInfo.Name))
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	b.Log.Debugf("Requesting upload slot ID %s for %s (escaped) with mime-type %s", fileId, fileNameEscaped, mimeType)
+
+	request := fmt.Sprintf("<request xmlns='urn:xmpp:http:upload:0' filename='%s' size='%d' content-type='%s' />", fileNameEscaped, fileInfo.Size, mimeType)
+
+	_, err := b.xc.RawInformation(b.xc.JID(), httpUploadComponent, fileId, "get", request)
+	if err != nil {
+		b.Log.WithError(err).Warn("Failed to request upload slot")
+		return
+	}
+
+	// Save the FileInfo in the buffer to actually upload it later
+	// when we receive the upload slot.
+	b.Lock()
+	b.httpUploadBuffer[fileId] = &UploadBufferEntry{
+		FileInfo:    fileInfo,
+		Mime:        mimeType,
+		Text:        text,
+		To:          to,
+		Description: description,
+	}
+	b.Unlock()
+}