working v1

This commit is contained in:
cottongin
2025-10-31 16:17:04 -04:00
parent e41402a963
commit 020daea391
71 changed files with 14793 additions and 1 deletions

287
bridge/helper/helper.go Normal file
View File

@@ -0,0 +1,287 @@
package helper
import (
"bytes"
"fmt"
"image/png"
"io"
"net/http"
"regexp"
"strings"
"time"
"unicode/utf8"
"golang.org/x/image/webp"
"github.com/42wim/matterbridge/bridge/config"
"github.com/gomarkdown/markdown"
"github.com/gomarkdown/markdown/html"
"github.com/gomarkdown/markdown/parser"
"github.com/sirupsen/logrus"
)
// DownloadFile downloads the given non-authenticated URL.
func DownloadFile(url string) (*[]byte, error) {
return DownloadFileAuth(url, "")
}
// DownloadFileAuth downloads the given URL using the specified authentication token.
func DownloadFileAuth(url string, auth string) (*[]byte, error) {
var buf bytes.Buffer
client := &http.Client{
Timeout: time.Second * 5,
}
req, err := http.NewRequest("GET", url, nil)
if auth != "" {
req.Header.Add("Authorization", auth)
}
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
io.Copy(&buf, resp.Body)
data := buf.Bytes()
return &data, nil
}
// DownloadFileAuthRocket downloads the given URL using the specified Rocket user ID and authentication token.
func DownloadFileAuthRocket(url, token, userID string) (*[]byte, error) {
var buf bytes.Buffer
client := &http.Client{
Timeout: time.Second * 5,
}
req, err := http.NewRequest("GET", url, nil)
req.Header.Add("X-Auth-Token", token)
req.Header.Add("X-User-Id", userID)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
_, err = io.Copy(&buf, resp.Body)
data := buf.Bytes()
return &data, err
}
// GetSubLines splits messages in newline-delimited lines. If maxLineLength is
// specified as non-zero GetSubLines will also clip long lines to the maximum
// length and insert a warning marker that the line was clipped.
//
// TODO: The current implementation has the inconvenient that it disregards
// word boundaries when splitting but this is hard to solve without potentially
// breaking formatting and other stylistic effects.
func GetSubLines(message string, maxLineLength int, clippingMessage string) []string {
if clippingMessage == "" {
clippingMessage = " <clipped message>"
}
var lines []string
for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
if line == "" {
// Prevent sending empty messages, so we'll skip this line
// if it has no content.
continue
}
if maxLineLength == 0 || len([]byte(line)) <= maxLineLength {
lines = append(lines, line)
continue
}
// !!! WARNING !!!
// Before touching the splitting logic below please ensure that you PROPERLY
// understand how strings, runes and range loops over strings work in Go.
// A good place to start is to read https://blog.golang.org/strings. :-)
var splitStart int
var startOfPreviousRune int
for i := range line {
if i-splitStart > maxLineLength-len([]byte(clippingMessage)) {
lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage)
splitStart = startOfPreviousRune
}
startOfPreviousRune = i
}
// This last append is safe to do without looking at the remaining byte-length
// as we assume that the byte-length of the last rune will never exceed that of
// the byte-length of the clipping message.
lines = append(lines, line[splitStart:])
}
return lines
}
// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
extra := msg.Extra
rmsg := []config.Message{}
for _, f := range extra[config.EventFileFailureSize] {
fi := f.(config.FileInfo)
text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
rmsg = append(rmsg, config.Message{
Text: text,
Username: "<system> ",
Channel: msg.Channel,
Account: msg.Account,
})
}
return rmsg
}
// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
if sha, ok := av[userid]; ok {
return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
}
return ""
}
// HandleDownloadSize checks a specified filename against the configured download blacklist
// and checks a specified file-size against the configure limit.
func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
// check blacklist here
for _, entry := range general.MediaDownloadBlackList {
if entry != "" {
re, err := regexp.Compile(entry)
if err != nil {
logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
continue
}
if re.MatchString(name) {
return fmt.Errorf("Matching blacklist %s. Not downloading %s", entry, name)
}
}
}
logger.Debugf("Trying to download %#v with size %#v", name, size)
if int(size) > general.MediaDownloadSize {
msg.Event = config.EventFileFailureSize
msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
Name: name,
Comment: msg.Text,
Size: size,
})
return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
}
return nil
}
// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
HandleDownloadData2(logger, msg, name, "", comment, url, data, general)
}
// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
func HandleDownloadData2(logger *logrus.Entry, msg *config.Message, name, id, comment, url string, data *[]byte, general *config.Protocol) {
var avatar bool
logger.Debugf("Download OK %#v %#v", name, len(*data))
if msg.Event == config.EventAvatarDownload {
avatar = true
}
msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
Name: name,
Data: data,
URL: url,
Comment: comment,
Avatar: avatar,
NativeID: id,
})
}
var emptyLineMatcher = regexp.MustCompile("\n+")
// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
// trims any preceding or trailing newline characters as well.
func RemoveEmptyNewLines(msg string) string {
return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
}
// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
// to the message in case it does so.
func ClipMessage(text string, length int, clippingMessage string) string {
if clippingMessage == "" {
clippingMessage = " <clipped message>"
}
if len(text) > length {
text = text[:length-len(clippingMessage)]
for len(text) > 0 {
if r, _ := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
text = text[:len(text)-1]
// Note: DecodeLastRuneInString only returns the constant value "1" in
// case of an error. We do not yet know whether the last rune is now
// actually valid. Example: "€" is 0xE2 0x82 0xAC. If we happen to split
// the string just before 0xAC, and go back only one byte, that would
// leave us with a string that ends in the byte 0xE2, which is not a valid
// rune, so we need to try again.
} else {
break
}
}
text += clippingMessage
}
return text
}
func ClipOrSplitMessage(text string, length int, clippingMessage string, splitMax int) []string {
var msgParts []string
remainingText := text
// Invariant of this splitting loop: No text is lost (msgParts+remainingText is the original text),
// and all parts is guaranteed to satisfy the length requirement.
for len(msgParts) < splitMax-1 && len(remainingText) > length {
// Decision: The text needs to be split (again).
var chunk string
wasted := 0
// The longest UTF-8 encoding of a valid rune is 4 bytes (0xF4 0x8F 0xBF 0xBF, encoding U+10FFFF),
// so we should never need to waste 4 or more bytes at a time.
for wasted < 4 && wasted < length {
chunk = remainingText[:length-wasted]
if r, _ := utf8.DecodeLastRuneInString(chunk); r == utf8.RuneError {
wasted += 1
} else {
break
}
}
// Note: At this point, "chunk" might still be invalid, if "text" is very broken.
msgParts = append(msgParts, chunk)
remainingText = remainingText[len(chunk):]
}
msgParts = append(msgParts, ClipMessage(remainingText, length, clippingMessage))
return msgParts
}
// ParseMarkdown takes in an input string as markdown and parses it to html
func ParseMarkdown(input string) string {
extensions := parser.HardLineBreak | parser.NoIntraEmphasis | parser.FencedCode
markdownParser := parser.NewWithExtensions(extensions)
renderer := html.NewRenderer(html.RendererOptions{
Flags: 0,
})
parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
res := string(parsedMarkdown)
res = strings.TrimPrefix(res, "<p>")
res = strings.TrimSuffix(res, "</p>\n")
return res
}
// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
func ConvertWebPToPNG(data *[]byte) error {
r := bytes.NewReader(*data)
m, err := webp.Decode(r)
if err != nil {
return err
}
var output []byte
w := bytes.NewBuffer(output)
if err := png.Encode(w, m); err != nil {
return err
}
*data = w.Bytes()
return nil
}

View File

@@ -0,0 +1,238 @@
package helper
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/assert"
)
const testLineLength = 64
var lineSplittingTestCases = map[string]struct {
input string
splitOutput []string
nonSplitOutput []string
}{
"Short single-line message": {
input: "short",
splitOutput: []string{"short"},
nonSplitOutput: []string{"short"},
},
"Long single-line message": {
input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
splitOutput: []string{
"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
" labore et dolore magna aliqua.",
},
nonSplitOutput: []string{"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."},
},
"Short multi-line message": {
input: "I\ncan't\nget\nno\nsatisfaction!",
splitOutput: []string{
"I",
"can't",
"get",
"no",
"satisfaction!",
},
nonSplitOutput: []string{
"I",
"can't",
"get",
"no",
"satisfaction!",
},
},
"Long multi-line message": {
input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n" +
"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n" +
"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n" +
"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
splitOutput: []string{
"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
" labore et dolore magna aliqua.",
"Ut enim ad minim veniam, quis nostrud exercita <clipped message>",
"tion ullamco laboris nisi ut aliquip ex ea com <clipped message>",
"modo consequat.",
"Duis aute irure dolor in reprehenderit in volu <clipped message>",
"ptate velit esse cillum dolore eu fugiat nulla <clipped message>",
" pariatur.",
"Excepteur sint occaecat cupidatat non proident <clipped message>",
", sunt in culpa qui officia deserunt mollit an <clipped message>",
"im id est laborum.",
},
nonSplitOutput: []string{
"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.",
"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.",
"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
},
},
"Message ending with new-line.": {
input: "Newline ending\n",
splitOutput: []string{"Newline ending"},
nonSplitOutput: []string{"Newline ending"},
},
"Long message containing UTF-8 multi-byte runes": {
input: "不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成?生賣的全直黑,我自我結毛分洲了世當,是政福那是東;斯說",
splitOutput: []string{
"不布人個我此而及單石業喜資富下 <clipped message>",
"我河下日沒一我臺空達的常景便物 <clipped message>",
"沒為……子大我別名解成?生賣的 <clipped message>",
"全直黑,我自我結毛分洲了世當, <clipped message>",
"是政福那是東;斯說",
},
nonSplitOutput: []string{"不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成?生賣的全直黑,我自我結毛分洲了世當,是政福那是東;斯說"},
},
"Long message, clip three-byte rune after two bytes": {
input: "x 人人生而自由,在尊嚴和權利上一律平等。 他們都具有理性和良知,應該以兄弟情誼的精神對待彼此。",
splitOutput: []string{
"x 人人生而自由,在尊嚴和權利上 <clipped message>",
"一律平等。 他們都具有理性和良知 <clipped message>",
",應該以兄弟情誼的精神對待彼此。",
},
nonSplitOutput: []string{"x 人人生而自由,在尊嚴和權利上一律平等。 他們都具有理性和良知,應該以兄弟情誼的精神對待彼此。"},
},
}
func TestGetSubLines(t *testing.T) {
for testname, testcase := range lineSplittingTestCases {
splitLines := GetSubLines(testcase.input, testLineLength, "")
assert.Equalf(t, testcase.splitOutput, splitLines, "'%s' testcase should give expected lines with splitting.", testname)
for _, splitLine := range splitLines {
byteLength := len([]byte(splitLine))
assert.True(t, byteLength <= testLineLength, "Splitted line '%s' of testcase '%s' should not exceed the maximum byte-length (%d vs. %d).", splitLine, testcase, byteLength, testLineLength)
}
nonSplitLines := GetSubLines(testcase.input, 0, "")
assert.Equalf(t, testcase.nonSplitOutput, nonSplitLines, "'%s' testcase should give expected lines without splitting.", testname)
}
}
func TestConvertWebPToPNG(t *testing.T) {
if os.Getenv("LOCAL_TEST") == "" {
t.Skip()
}
input, err := ioutil.ReadFile("test.webp")
if err != nil {
t.Fail()
}
d := &input
err = ConvertWebPToPNG(d)
if err != nil {
t.Fail()
}
err = ioutil.WriteFile("test.png", *d, 0o644) // nolint:gosec
if err != nil {
t.Fail()
}
}
var clippingOrSplittingTestCases = map[string]struct {
inputText string
clipSplitLength int
clippingMessage string
splitMax int
expectedOutput []string
}{
"Short single-line message, split 3": {
inputText: "short",
clipSplitLength: 20,
clippingMessage: "?!?!",
splitMax: 3,
expectedOutput: []string{"short"},
},
"Short single-line message, split 1": {
inputText: "short",
clipSplitLength: 20,
clippingMessage: "?!?!",
splitMax: 1,
expectedOutput: []string{"short"},
},
"Short single-line message, split 0": {
// Mainly check that we don't crash.
inputText: "short",
clipSplitLength: 20,
clippingMessage: "?!?!",
splitMax: 0,
expectedOutput: []string{"short"},
},
"Long single-line message, noclip": {
inputText: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
clipSplitLength: 50,
clippingMessage: "?!?!",
splitMax: 10,
expectedOutput: []string{
"Lorem ipsum dolor sit amet, consectetur adipiscing",
" elit, sed do eiusmod tempor incididunt ut labore ",
"et dolore magna aliqua.",
},
},
"Long single-line message, noclip tight": {
inputText: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
clipSplitLength: 50,
clippingMessage: "?!?!",
splitMax: 3,
expectedOutput: []string{
"Lorem ipsum dolor sit amet, consectetur adipiscing",
" elit, sed do eiusmod tempor incididunt ut labore ",
"et dolore magna aliqua.",
},
},
"Long single-line message, clip custom": {
inputText: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
clipSplitLength: 50,
clippingMessage: "?!?!",
splitMax: 2,
expectedOutput: []string{
"Lorem ipsum dolor sit amet, consectetur adipiscing",
" elit, sed do eiusmod tempor incididunt ut lab?!?!",
},
},
"Long single-line message, clip built-in": {
inputText: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
clipSplitLength: 50,
clippingMessage: "",
splitMax: 2,
expectedOutput: []string{
"Lorem ipsum dolor sit amet, consectetur adipiscing",
" elit, sed do eiusmod tempor inc <clipped message>",
},
},
"Short multi-line message": {
inputText: "I\ncan't\nget\nno\nsatisfaction!",
clipSplitLength: 50,
clippingMessage: "",
splitMax: 2,
expectedOutput: []string{"I\ncan't\nget\nno\nsatisfaction!"},
},
"Long message containing UTF-8 multi-byte runes": {
inputText: "人人生而自由,在尊嚴和權利上一律平等。 他們都具有理性和良知,應該以兄弟情誼的精神對待彼此。",
clipSplitLength: 50,
clippingMessage: "",
splitMax: 10,
expectedOutput: []string{
"人人生而自由,在尊嚴和權利上一律", // Note: only 48 bytes!
"平等。 他們都具有理性和良知,應該", // Note: only 49 bytes!
"以兄弟情誼的精神對待彼此。",
},
},
}
func TestClipOrSplitMessage(t *testing.T) {
for testname, testcase := range clippingOrSplittingTestCases {
actualOutput := ClipOrSplitMessage(testcase.inputText, testcase.clipSplitLength, testcase.clippingMessage, testcase.splitMax)
assert.Equalf(t, testcase.expectedOutput, actualOutput, "'%s' testcase should give expected lines with clipping+splitting.", testname)
for _, splitLine := range testcase.expectedOutput {
byteLength := len([]byte(splitLine))
assert.True(t, byteLength <= testcase.clipSplitLength, "Splitted line '%s' of testcase '%s' should not exceed the maximum byte-length (%d vs. %d).", splitLine, testname, testcase.clipSplitLength, byteLength)
}
}
}

View File

@@ -0,0 +1,35 @@
//go:build cgolottie
package helper
import (
"fmt"
"github.com/Benau/tgsconverter/libtgsconverter"
"github.com/sirupsen/logrus"
)
func CanConvertTgsToX() error {
return nil
}
// ConvertTgsToX convert input data (which should be tgs format) to any format supported by libtgsconverter
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
options := libtgsconverter.NewConverterOptions()
options.SetExtension(outputFormat)
blob, err := libtgsconverter.ImportFromData(*data, options)
if err != nil {
return fmt.Errorf("failed to run libtgsconverter.ImportFromData: %s", err.Error())
}
*data = blob
return nil
}
func SupportsFormat(format string) bool {
return libtgsconverter.SupportsExtension(format)
}
func LottieBackend() string {
return "libtgsconverter"
}

View File

@@ -0,0 +1,90 @@
//go:build !cgolottie
package helper
import (
"io/ioutil"
"os"
"os/exec"
"github.com/sirupsen/logrus"
)
// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
func CanConvertTgsToX() error {
// We depend on the fact that `lottie_convert.py --help` has exit status 0.
// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
// However, there is no alternative like `lottie_convert.py --is-properly-installed`
cmd := exec.Command("lottie_convert.py", "--help")
return cmd.Run()
}
// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
// This relies on an external command, which is ugly, but works.
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
// lottie can't handle input from a pipe, so write to a temporary file:
tmpInFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-input-*.tgs")
if err != nil {
return err
}
tmpInFileName := tmpInFile.Name()
defer func() {
if removeErr := os.Remove(tmpInFileName); removeErr != nil {
logger.Errorf("Could not delete temporary (input) file %s: %v", tmpInFileName, removeErr)
}
}()
// lottie can handle writing to a pipe, but there is no way to do that platform-independently.
// "/dev/stdout" won't work on Windows, and "-" upsets Cairo for some reason. So we need another file:
tmpOutFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-output-*.data")
if err != nil {
return err
}
tmpOutFileName := tmpOutFile.Name()
defer func() {
if removeErr := os.Remove(tmpOutFileName); removeErr != nil {
logger.Errorf("Could not delete temporary (output) file %s: %v", tmpOutFileName, removeErr)
}
}()
if _, writeErr := tmpInFile.Write(*data); writeErr != nil {
return writeErr
}
// Must close before calling lottie to avoid data races:
if closeErr := tmpInFile.Close(); closeErr != nil {
return closeErr
}
// Call lottie to transform:
cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpInFileName, tmpOutFileName)
cmd.Stdout = nil
cmd.Stderr = nil
// NB: lottie writes progress into to stderr in all cases.
_, stderr := cmd.Output()
if stderr != nil {
// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
return stderr
}
dataContents, err := ioutil.ReadFile(tmpOutFileName)
if err != nil {
return err
}
*data = dataContents
return nil
}
func SupportsFormat(format string) bool {
switch format {
case "png":
fallthrough
case "webp":
return true
default:
return false
}
return false
}
func LottieBackend() string {
return "lottie_convert.py"
}