Add new HTTP helpers

bridge/bridge.go

@@ -1,7 +1,13 @@
 package bridge
 
 import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
 	"log"
+	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -15,6 +21,8 @@ type Bridger interface {
 	Connect() error
 	JoinChannel(channel config.ChannelInfo) error
 	Disconnect() error
+	NewHttpRequest(method, uri string, body io.Reader) (*http.Request, error)
+	NewHttpClient(proxy string) (*http.Client, error)
 }
 
 type Bridge struct {
@@ -30,6 +38,7 @@ type Bridge struct {
 	Log            *logrus.Entry
 	Config         config.Config
 	General        *config.Protocol
+	HttpClient     *http.Client // Unique HTTP settings per bridge
 }
 
 type Config struct {
@@ -41,6 +50,8 @@ type Config struct {
 // Factory is the factory function to create a bridge
 type Factory func(*Config) Bridger
 
+// New is a basic constructor. More important fields are populated
+// in gateway/gateway.go (AddBridge method).
 func New(bridge *config.Bridge) *Bridge {
 	accInfo := strings.Split(bridge.Account, ".")
 	if len(accInfo) != 2 {
@@ -133,3 +144,218 @@ func (b *Bridge) GetStringSlice2D(key string) [][]string {
 	}
 	return val
 }
+
+// NewHttpClient produces a single unified http.Client per bridge.
+//
+// This allows to have project-wide defaults (timeout) as well as
+// bridge-configurable values (`http_proxy`).
+//
+// This method is left public so that if that's needed, a bridge can
+// override this constructor.
+//
+// TODO: maybe protocols without HTTP downloads at all could override
+// this method and return nil? Or the other way around?
+func (b *Bridge) NewHttpClient(http_proxy string) (*http.Client, error) {
+	if http_proxy != "" {
+		proxyUrl, err := url.Parse(b.GetString("http_proxy"))
+		if err != nil {
+			return nil, err
+		}
+
+		b.Log.Debugf("%s using HTTP proxy %s", b.Protocol, proxyUrl)
+
+		return &http.Client{
+			Timeout:   time.Second * 15,
+			Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)},
+		}, nil
+	}
+
+	b.Log.Debugf("%s not using HTTP proxy", b.Protocol)
+
+	return &http.Client{
+		Timeout: time.Second * 5,
+	}, nil
+}
+
+var errHttpGetNotOk = errors.New("HTTP server responded non-OK code")
+
+func HttpGetNotOkError(uri string, code int) error {
+	return fmt.Errorf("%w: %s returned code %d", errHttpGetNotOk, uri, code)
+}
+
+// HttpGetBytes returns bytes from a given URI, if the request
+// succeeds and HTTP response status is 200 (OK).
+func (b *Bridge) HttpGetBytes(uri string) (*[]byte, error) {
+	req, err := b.Bridger.NewHttpRequest("GET", uri, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	b.Log.Debugf("Getting HTTP bytes with request: %#v", req)
+
+	resp, err := b.HttpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, HttpGetNotOkError(uri, resp.StatusCode)
+	}
+
+	var buf bytes.Buffer
+
+	_, err = io.Copy(&buf, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = resp.Body.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	data := buf.Bytes()
+
+	return &data, nil
+}
+
+// HttpUpload uploads data to a URI, and validates the response status code.
+//
+// Params:
+//
+// - method: specific HTTP verb
+// - uri: remote URL
+// - headers: map of headers to insert in the request
+// - data: raw bytes to upload in the body
+// - ok_status: list of HTTP status codes considered successful (default: 200, 201)
+//
+// The response body is always discarded.
+func (b *Bridge) HttpUpload(method string, uri string, headers map[string]string, data *[]byte, ok_status []int) error {
+	req, err := b.Bridger.NewHttpRequest(method, uri, bytes.NewReader(*data))
+	if err != nil {
+		return err
+	}
+
+	for header_name, header_value := range headers {
+		req.Header.Set(header_name, header_value)
+	}
+
+	b.Log.Debugf("HTTP upload request: %v", req)
+
+	resp, err := b.HttpClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	err = resp.Body.Close()
+	if err != nil {
+		return err
+	}
+
+	// By default, allow codes 200 (OK) and 201 (CREATED)
+	if len(ok_status) == 0 {
+		ok_status = []int{200, 201}
+	}
+
+	for _, expected_code := range ok_status {
+		b.Log.Debugf("Successful file upload with code %d", expected_code)
+		return nil
+	}
+
+	return HttpGetNotOkError(uri, resp.StatusCode)
+}
+
+func (b *Bridge) AddAttachmentFromURL(msg *config.Message, filename string, id string, comment string, uri string) error {
+	return b.addAttachment(msg, filename, id, comment, uri, nil, false)
+}
+
+func (b *Bridge) AddAttachmentFromBytes(msg *config.Message, filename string, id string, comment string, data *[]byte) error {
+	return b.addAttachment(msg, filename, id, comment, "", data, false)
+}
+
+func (b *Bridge) AddAvatarFromURL(msg *config.Message, filename string, id string, comment string, uri string) error {
+	return b.addAttachment(msg, filename, id, comment, uri, nil, true)
+}
+
+func (b *Bridge) AddAvatarFromBytes(msg *config.Message, filename string, id string, comment string, data *[]byte) error {
+	return b.addAttachment(msg, filename, id, comment, "", data, true)
+}
+
+// NewHttpRequest produces a new http.Request instance with bridge-specific settings.
+//
+// This is used by bridges where HTTP downloads require a cookie/token, by overriding
+// this method in the bridge struct.
+func (b *Bridge) NewHttpRequest(method, uri string, body io.Reader) (*http.Request, error) {
+	return http.NewRequest(method, uri, body)
+}
+
+// Internal method including common parts to attachment/avatar handling methods.
+//
+// This method will process received bytes. If bytes are not set, they will be downloaded from the given URL.
+// If neither data bytes nor uri is provided, this will be a hard error because there's a logic error somewhere.
+func (b *Bridge) addAttachment(msg *config.Message, filename string, id string, comment string, uri string, data *[]byte, avatar bool) error {
+	if data != nil {
+		return b.addAttachmentProcess(msg, filename, id, comment, uri, data, avatar)
+	}
+
+	if uri == "" {
+		// This should never happen
+		b.Log.Fatalf("Logic error in bridge %s: attachment should have either URL or data set, neither was provided", b.Protocol)
+	}
+
+	data, err := b.HttpGetBytes(uri)
+	if err != nil {
+		return err
+	}
+
+	return b.addAttachmentProcess(msg, filename, id, comment, uri, data, avatar)
+}
+
+type errFileTooLarge struct {
+	FileName string
+	Size     int
+	MaxSize  int
+}
+
+func (e *errFileTooLarge) Error() string {
+	return fmt.Sprintf("File %#v to large to download (%#v). MediaDownloadSize is %#v", e.FileName, e.Size, e.MaxSize)
+}
+
+type errFileBlacklisted struct {
+	FileName string
+}
+
+func (e *errFileBlacklisted) Error() string {
+	return fmt.Sprintf("File %#v matches the backlist, not downloading it", e.FileName)
+}
+
+func (b *Bridge) addAttachmentProcess(msg *config.Message, filename string, id string, comment string, uri string, data *[]byte, avatar bool) error {
+	size := len(*data)
+	if size > b.General.MediaDownloadSize {
+		return &errFileTooLarge{
+			FileName: filename,
+			Size:     size,
+			MaxSize:  b.General.MediaDownloadSize,
+		}
+	}
+
+	// Apply `MediaDownloadBlackList` regexes
+	if b.Config.IsFilenameBlacklisted(filename) {
+		return &errFileBlacklisted{
+			FileName: filename,
+		}
+	}
+
+	b.Log.Debugf("Download OK %#v %#v", filename, size)
+	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
+		Name:    filename,
+		Data:    data,
+		URL:     uri,
+		Comment: comment,
+		Avatar:  avatar,
+		// TODO: if id is not set, maybe use hash of bytes?
+		NativeID: id,
+	})
+
+	return nil
+}

bridge/config/config.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -285,14 +286,16 @@ type Config interface {
 	GetString(key string) (string, bool)
 	GetStringSlice(key string) ([]string, bool)
 	GetStringSlice2D(key string) ([][]string, bool)
+	IsFilenameBlacklisted(filename string) bool
 }
 
 type config struct {
 	sync.RWMutex
 
-	logger *logrus.Entry
-	v      *viper.Viper
-	cv     *BridgeValues
+	logger                        *logrus.Entry
+	v                             *viper.Viper
+	cv                            *BridgeValues
+	MediaDownloadBlackListRegexes *[]*regexp.Regexp
 }
 
 // NewConfig instantiates a new configuration based on the specified configuration file path.
@@ -319,6 +322,12 @@ func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
 	if mycfg.cv.General.MediaDownloadSize == 0 {
 		mycfg.cv.General.MediaDownloadSize = 1000000
 	}
+
+	// Precompile MediaBlackList regexes so we make sure they're correct,
+	// and they don't have to be compiled on every file attachment, because
+	// that's a slow operation.
+	mycfg.compileMediaDownloadBlackListRegexes()
+
 	viper.WatchConfig()
 	viper.OnConfigChange(func(e fsnotify.Event) {
 		logger.Println("Config file changed:", e.Name)
@@ -422,6 +431,44 @@ func (c *config) GetStringSlice2D(key string) ([][]string, bool) {
 	return result, true
 }
 
+// IsFilenameBlackListed checks if a given file name matches the
+// configured blacklist. This is useful to filter potentially-harmful
+// files that could be served over HTTP (eg. `.html` with XSS).
+func (c *config) IsFilenameBlacklisted(filename string) bool {
+	c.RLock()
+	defer c.RUnlock()
+
+	for _, re := range *c.MediaDownloadBlackListRegexes {
+		if re.MatchString(filename) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (c *config) compileMediaDownloadBlackListRegexes() {
+	regexes := []*regexp.Regexp{}
+
+	// TODO: apparently c.cv.General does not get updated when config reloads
+	// see https://github.com/matterbridge-org/matterbridge/issues/57
+	// for _, regex := range c.cv.General.MediaDownloadBlackList {
+	for _, regex := range c.v.GetStringSlice("general.MediaDownloadBlackList") {
+		c.logger.Debugf("Found blacklist regex %s", regex)
+
+		re, err := regexp.Compile(regex)
+		if err != nil {
+			c.logger.Errorf("incorrect regexp %s for MediaDownloadBlackList", regex)
+			continue
+		}
+
+		regexes = append(regexes, re)
+	}
+
+	c.MediaDownloadBlackListRegexes = &regexes
+	c.logger.Debug("Successfully applied new `MediaDownloadBlackList` regexes")
+}
+
 func GetIconURL(msg *Message, iconURL string) string {
 	info := strings.Split(msg.Account, ".")
 	protocol := info[0]

changelog.md

@@ -18,6 +18,7 @@
 
 - general
   - matterbridge output now colors log level for easier log reading ([#25](https://github.com/matterbridge-org/matterbridge/pull/25))
+  - new HTTP helpers are common to all bridges, and allow overriding specific settings ([#59](https://github.com/matterbridge-org/matterbridge/pull/59))
 - mastodon
   - Add new Mastodon bridge ([#14](https://github.com/matterbridge-org/matterbridge/pull/14)/[#16](https://github.com/matterbridge-org/matterbridge/pull/16), thanks @lil5)
   - Supports public messages and private messages