Added type sniffer.

modules/typesniffer/typesniffer.go

@@ -0,0 +1,81 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package typesniffer
+
+import (
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+// Use at most this many bytes to determine Content Type.
+const sniffLen = 512
+
+// SvgMimeType MIME type of SVG images.
+const SvgMimeType = "image/svg+xml"
+
+var svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!--.*?-->|<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg[\s>\/]`)
+var svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!--.*?-->|<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg[\s>\/]`)
+
+type SniffedType struct {
+	contentType string
+}
+
+// IsText etects if content format is plain text.
+func (ct SniffedType) IsText() bool {
+	return strings.Contains(ct.contentType, "text/")
+}
+
+// IsImage detects if data is an image format
+func (ct SniffedType) IsImage() bool {
+	return strings.Contains(ct.contentType, "image/")
+}
+
+// IsSvgImage detects if data is an SVG image format
+func (ct SniffedType) IsSvgImage() bool {
+	return strings.Contains(ct.contentType, SvgMimeType)
+}
+
+// IsPdf detects if data is a pdf format
+func (ct SniffedType) IsPdf() bool {
+	return strings.Contains(ct.contentType, "application/pdf")
+}
+
+// IsVideo detects if data is an video format
+func (ct SniffedType) IsVideo() bool {
+	return strings.Contains(ct.contentType, "video/")
+}
+
+// IsAudio detects if data is an video format
+func (ct SniffedType) IsAudio() bool {
+	return strings.Contains(ct.contentType, "audio/")
+}
+
+// IsRepresentableAsText returns true if file content can be represented as
+// plain text or is empty.
+func (ct SniffedType) IsRepresentableAsText() bool {
+	return ct.IsText() || ct.IsSvgImage()
+}
+
+// DetectContentType extends http.DetectContentType with more content types. Defaults to text/unknown if input is empty.
+func DetectContentType(data []byte) SniffedType {
+	if len(data) == 0 {
+		return SniffedType{"text/unknown"}
+	}
+
+	ct := http.DetectContentType(data)
+
+	if len(data) > sniffLen {
+		data = data[:sniffLen]
+	}
+
+	if (strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html")) && svgTagRegex.Match(data) ||
+		strings.Contains(ct, "text/xml") && svgTagInXMLRegex.Match(data) {
+		// SVG is unsupported. https://github.com/golang/go/issues/15888
+		ct = SvgMimeType
+	}
+
+	return SniffedType{ct}
+}

modules/typesniffer/typesniffer_test.go

@@ -0,0 +1,79 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package typesniffer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDetectContentTypeLongerThanSniffLen(t *testing.T) {
+	// Pre-condition: Shorter than sniffLen detects SVG.
+	assert.Equal(t, "image/svg+xml", DetectContentType([]byte(`<!-- Comment --><svg></svg>`)).contentType)
+	// Longer than sniffLen detects something else.
+	assert.Equal(t, "text/plain; charset=utf-8", DetectContentType([]byte(`<!--
+Comment Comment Comment Comment Comment Comment Comment Comment Comment Comment
+Comment Comment Comment Comment Comment Comment Comment Comment Comment Comment
+Comment Comment Comment Comment Comment Comment Comment Comment Comment Comment
+Comment Comment Comment Comment Comment Comment Comment Comment Comment Comment
+Comment Comment Comment Comment Comment Comment Comment Comment Comment Comment
+Comment Comment Comment Comment Comment Comment Comment Comment Comment Comment
+Comment Comment Comment --><svg></svg>`)).contentType)
+}
+
+func TestIsTextFile(t *testing.T) {
+	assert.True(t, DetectContentType([]byte{}).IsText())
+	assert.True(t, DetectContentType([]byte("lorem ipsum")).IsText())
+}
+
+func TestIsSvgImage(t *testing.T) {
+	assert.True(t, DetectContentType([]byte("<svg></svg>")).IsSvgImage())
+	assert.True(t, DetectContentType([]byte("    <svg></svg>")).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<svg width="100"></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte("<svg/>")).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?><svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<!-- Comment -->
+	<svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<!-- Multiple -->
+	<!-- Comments -->
+	<svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<!-- Multiline
+	Comment -->
+	<svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1 Basic//EN"
+	"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11-basic.dtd">
+	<svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?>
+	<!-- Comment -->
+	<svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?>
+	<!-- Multiple -->
+	<!-- Comments -->
+	<svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?>
+	<!-- Multline
+	Comment -->
+	<svg></svg>`)).IsSvgImage())
+	assert.True(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?>
+	<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+	<!-- Multline
+	Comment -->
+	<svg></svg>`)).IsSvgImage())
+	assert.False(t, DetectContentType([]byte{}).IsSvgImage())
+	assert.False(t, DetectContentType([]byte("svg")).IsSvgImage())
+	assert.False(t, DetectContentType([]byte("<svgfoo></svgfoo>")).IsSvgImage())
+	assert.False(t, DetectContentType([]byte("text<svg></svg>")).IsSvgImage())
+	assert.False(t, DetectContentType([]byte("<html><body><svg></svg></body></html>")).IsSvgImage())
+	assert.False(t, DetectContentType([]byte(`<script>"<svg></svg>"</script>`)).IsSvgImage())
+	assert.False(t, DetectContentType([]byte(`<!-- <svg></svg> inside comment -->
+	<foo></foo>`)).IsSvgImage())
+	assert.False(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?>
+	<!-- <svg></svg> inside comment -->
+	<foo></foo>`)).IsSvgImage())
+}
+
+// TODO: IsImageFile(), currently no idea how to test
+// TODO: IsPDFFile(), currently no idea how to test