| 1 | package helper |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "image/png" |
| 8 | "io" |
| 9 | "net/http" |
| 10 | "regexp" |
| 11 | "strings" |
| 12 | "time" |
| 13 | "unicode/utf8" |
| 14 | |
| 15 | "golang.org/x/image/webp" |
| 16 | |
| 17 | "github.com/gomarkdown/markdown" |
| 18 | "github.com/gomarkdown/markdown/html" |
| 19 | "github.com/gomarkdown/markdown/parser" |
| 20 | "github.com/matterbridge-org/matterbridge/bridge/config" |
| 21 | "github.com/sirupsen/logrus" |
| 22 | ) |
| 23 | |
| 24 | var errHttpGetNotOk = errors.New("HTTP server responded non-OK code") |
| 25 | |
| 26 | func HttpGetNotOkError(url string, code int) error { |
| 27 | return fmt.Errorf("%w: %s returned code %d", errHttpGetNotOk, url, code) |
| 28 | } |
| 29 | |
| 30 | // DownloadFile downloads the given non-authenticated URL. |
| 31 | func DownloadFile(url string) (*[]byte, error) { |
| 32 | return DownloadFileAuth(url, "") |
| 33 | } |
| 34 | |
| 35 | // DownloadFileAuth downloads the given URL using the specified authentication token. |
| 36 | func DownloadFileAuth(url string, auth string) (*[]byte, error) { |
| 37 | var buf bytes.Buffer |
| 38 | client := &http.Client{ |
| 39 | Timeout: time.Second * 5, |
| 40 | } |
| 41 | req, err := http.NewRequest("GET", url, nil) |
| 42 | if auth != "" { |
| 43 | req.Header.Add("Authorization", auth) |
| 44 | } |
| 45 | if err != nil { |
| 46 | return nil, err |
| 47 | } |
| 48 | resp, err := client.Do(req) |
| 49 | if err != nil { |
| 50 | return nil, err |
| 51 | } |
| 52 | |
| 53 | if resp.StatusCode != http.StatusOK { |
| 54 | return nil, HttpGetNotOkError(url, resp.StatusCode) |
| 55 | } |
| 56 | |
| 57 | _, err = io.Copy(&buf, resp.Body) |
| 58 | if err != nil { |
| 59 | return nil, err |
| 60 | } |
| 61 | |
| 62 | err = resp.Body.Close() |
| 63 | if err != nil { |
| 64 | return nil, err |
| 65 | } |
| 66 | |
| 67 | data := buf.Bytes() |
| 68 | return &data, nil |
| 69 | } |
| 70 | |
| 71 | // DownloadFileAuthRocket downloads the given URL using the specified Rocket user ID and authentication token. |
| 72 | func DownloadFileAuthRocket(url, token, userID string) (*[]byte, error) { |
| 73 | var buf bytes.Buffer |
| 74 | client := &http.Client{ |
| 75 | Timeout: time.Second * 5, |
| 76 | } |
| 77 | req, err := http.NewRequest("GET", url, nil) |
| 78 | |
| 79 | req.Header.Add("X-Auth-Token", token) |
| 80 | req.Header.Add("X-User-Id", userID) |
| 81 | |
| 82 | if err != nil { |
| 83 | return nil, err |
| 84 | } |
| 85 | resp, err := client.Do(req) |
| 86 | if err != nil { |
| 87 | return nil, err |
| 88 | } |
| 89 | defer resp.Body.Close() |
| 90 | _, err = io.Copy(&buf, resp.Body) |
| 91 | data := buf.Bytes() |
| 92 | return &data, err |
| 93 | } |
| 94 | |
| 95 | // GetSubLines splits messages in newline-delimited lines. If maxLineLength is |
| 96 | // specified as non-zero GetSubLines will also clip long lines to the maximum |
| 97 | // length and insert a warning marker that the line was clipped. |
| 98 | // |
| 99 | // TODO: The current implementation has the inconvenient that it disregards |
| 100 | // word boundaries when splitting but this is hard to solve without potentially |
| 101 | // breaking formatting and other stylistic effects. |
| 102 | func GetSubLines(message string, maxLineLength int, clippingMessage string) []string { |
| 103 | if clippingMessage == "" { |
| 104 | clippingMessage = " <clipped message>" |
| 105 | } |
| 106 | |
| 107 | var lines []string |
| 108 | for _, line := range strings.Split(strings.TrimSpace(message), "\n") { |
| 109 | if line == "" { |
| 110 | // Prevent sending empty messages, so we'll skip this line |
| 111 | // if it has no content. |
| 112 | continue |
| 113 | } |
| 114 | |
| 115 | if maxLineLength == 0 || len([]byte(line)) <= maxLineLength { |
| 116 | lines = append(lines, line) |
| 117 | continue |
| 118 | } |
| 119 | |
| 120 | // !!! WARNING !!! |
| 121 | // Before touching the splitting logic below please ensure that you PROPERLY |
| 122 | // understand how strings, runes and range loops over strings work in Go. |
| 123 | // A good place to start is to read https://blog.golang.org/strings. :-) |
| 124 | var splitStart int |
| 125 | var startOfPreviousRune int |
| 126 | for i := range line { |
| 127 | if i-splitStart > maxLineLength-len([]byte(clippingMessage)) { |
| 128 | lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage) |
| 129 | splitStart = startOfPreviousRune |
| 130 | } |
| 131 | startOfPreviousRune = i |
| 132 | } |
| 133 | // This last append is safe to do without looking at the remaining byte-length |
| 134 | // as we assume that the byte-length of the last rune will never exceed that of |
| 135 | // the byte-length of the clipping message. |
| 136 | lines = append(lines, line[splitStart:]) |
| 137 | } |
| 138 | return lines |
| 139 | } |
| 140 | |
| 141 | // HandleExtra manages the supplementary details stored inside a message's 'Extra' field map. |
| 142 | func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message { |
| 143 | extra := msg.Extra |
| 144 | rmsg := []config.Message{} |
| 145 | for _, f := range extra[config.EventFileFailureSize] { |
| 146 | fi := f.(config.FileInfo) |
| 147 | text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize) |
| 148 | rmsg = append(rmsg, config.Message{ |
| 149 | Text: text, |
| 150 | Username: "<system> ", |
| 151 | Channel: msg.Channel, |
| 152 | Account: msg.Account, |
| 153 | }) |
| 154 | } |
| 155 | return rmsg |
| 156 | } |
| 157 | |
| 158 | // GetAvatar constructs a URL for a given user-avatar if it is available in the cache. |
| 159 | func GetAvatar(av map[string]string, userid string, general *config.Protocol) string { |
| 160 | if sha, ok := av[userid]; ok { |
| 161 | return general.MediaServerDownload + "/" + sha + "/" + userid + ".png" |
| 162 | } |
| 163 | return "" |
| 164 | } |
| 165 | |
| 166 | // HandleDownloadSize checks a specified filename against the configured download blacklist |
| 167 | // and checks a specified file-size against the configure limit. |
| 168 | func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error { |
| 169 | // check blacklist here |
| 170 | for _, entry := range general.MediaDownloadBlackList { |
| 171 | if entry != "" { |
| 172 | re, err := regexp.Compile(entry) |
| 173 | if err != nil { |
| 174 | logger.Errorf("incorrect regexp %s for %s", entry, msg.Account) |
| 175 | continue |
| 176 | } |
| 177 | if re.MatchString(name) { |
| 178 | return fmt.Errorf("Matching blacklist %s. Not downloading %s", entry, name) |
| 179 | } |
| 180 | } |
| 181 | } |
| 182 | logger.Debugf("Trying to download %#v with size %#v", name, size) |
| 183 | if int(size) > general.MediaDownloadSize { |
| 184 | msg.Event = config.EventFileFailureSize |
| 185 | msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{ |
| 186 | Name: name, |
| 187 | Comment: msg.Text, |
| 188 | Size: size, |
| 189 | }) |
| 190 | return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize) |
| 191 | } |
| 192 | return nil |
| 193 | } |
| 194 | |
| 195 | // HandleDownloadData adds the data for a remote file into a Matterbridge gateway message. |
| 196 | func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) { |
| 197 | HandleDownloadData2(logger, msg, name, "", comment, url, data, general) |
| 198 | } |
| 199 | |
| 200 | // HandleDownloadData adds the data for a remote file into a Matterbridge gateway message. |
| 201 | func HandleDownloadData2(logger *logrus.Entry, msg *config.Message, name, id, comment, url string, data *[]byte, general *config.Protocol) { |
| 202 | var avatar bool |
| 203 | logger.Debugf("Download OK %#v %#v", name, len(*data)) |
| 204 | if msg.Event == config.EventAvatarDownload { |
| 205 | avatar = true |
| 206 | } |
| 207 | msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{ |
| 208 | Name: name, |
| 209 | Data: data, |
| 210 | URL: url, |
| 211 | Comment: comment, |
| 212 | Avatar: avatar, |
| 213 | NativeID: id, |
| 214 | }) |
| 215 | } |
| 216 | |
| 217 | var emptyLineMatcher = regexp.MustCompile("\n+") |
| 218 | |
| 219 | // RemoveEmptyNewLines collapses consecutive newline characters into a single one and |
| 220 | // trims any preceding or trailing newline characters as well. |
| 221 | func RemoveEmptyNewLines(msg string) string { |
| 222 | return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n") |
| 223 | } |
| 224 | |
| 225 | // ClipMessage trims a message to the specified length if it exceeds it and adds a warning |
| 226 | // to the message in case it does so. |
| 227 | func ClipMessage(text string, length int, clippingMessage string) string { |
| 228 | if clippingMessage == "" { |
| 229 | clippingMessage = " <clipped message>" |
| 230 | } |
| 231 | |
| 232 | if len(text) > length { |
| 233 | text = text[:length-len(clippingMessage)] |
| 234 | for len(text) > 0 { |
| 235 | if r, _ := utf8.DecodeLastRuneInString(text); r == utf8.RuneError { |
| 236 | text = text[:len(text)-1] |
| 237 | // Note: DecodeLastRuneInString only returns the constant value "1" in |
| 238 | // case of an error. We do not yet know whether the last rune is now |
| 239 | // actually valid. Example: "€" is 0xE2 0x82 0xAC. If we happen to split |
| 240 | // the string just before 0xAC, and go back only one byte, that would |
| 241 | // leave us with a string that ends in the byte 0xE2, which is not a valid |
| 242 | // rune, so we need to try again. |
| 243 | } else { |
| 244 | break |
| 245 | } |
| 246 | } |
| 247 | text += clippingMessage |
| 248 | } |
| 249 | return text |
| 250 | } |
| 251 | |
| 252 | func ClipOrSplitMessage(text string, length int, clippingMessage string, splitMax int) []string { |
| 253 | var msgParts []string |
| 254 | remainingText := text |
| 255 | // Invariant of this splitting loop: No text is lost (msgParts+remainingText is the original text), |
| 256 | // and all parts is guaranteed to satisfy the length requirement. |
| 257 | for len(msgParts) < splitMax-1 && len(remainingText) > length { |
| 258 | // Decision: The text needs to be split (again). |
| 259 | var chunk string |
| 260 | wasted := 0 |
| 261 | // The longest UTF-8 encoding of a valid rune is 4 bytes (0xF4 0x8F 0xBF 0xBF, encoding U+10FFFF), |
| 262 | // so we should never need to waste 4 or more bytes at a time. |
| 263 | for wasted < 4 && wasted < length { |
| 264 | chunk = remainingText[:length-wasted] |
| 265 | if r, _ := utf8.DecodeLastRuneInString(chunk); r == utf8.RuneError { |
| 266 | wasted += 1 |
| 267 | } else { |
| 268 | break |
| 269 | } |
| 270 | } |
| 271 | // Note: At this point, "chunk" might still be invalid, if "text" is very broken. |
| 272 | msgParts = append(msgParts, chunk) |
| 273 | remainingText = remainingText[len(chunk):] |
| 274 | } |
| 275 | msgParts = append(msgParts, ClipMessage(remainingText, length, clippingMessage)) |
| 276 | return msgParts |
| 277 | } |
| 278 | |
| 279 | // ParseMarkdown takes in an input string as markdown and parses it to html |
| 280 | func ParseMarkdown(input string) string { |
| 281 | extensions := parser.HardLineBreak | parser.NoIntraEmphasis | parser.FencedCode |
| 282 | markdownParser := parser.NewWithExtensions(extensions) |
| 283 | renderer := html.NewRenderer(html.RendererOptions{ |
| 284 | Flags: 0, |
| 285 | }) |
| 286 | parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer) |
| 287 | res := string(parsedMarkdown) |
| 288 | res = strings.TrimPrefix(res, "<p>") |
| 289 | res = strings.TrimSuffix(res, "</p>\n") |
| 290 | return res |
| 291 | } |
| 292 | |
| 293 | // ConvertWebPToPNG converts input data (which should be WebP format) to PNG format |
| 294 | func ConvertWebPToPNG(data *[]byte) error { |
| 295 | r := bytes.NewReader(*data) |
| 296 | m, err := webp.Decode(r) |
| 297 | if err != nil { |
| 298 | return err |
| 299 | } |
| 300 | var output []byte |
| 301 | w := bytes.NewBuffer(output) |
| 302 | if err := png.Encode(w, m); err != nil { |
| 303 | return err |
| 304 | } |
| 305 | *data = w.Bytes() |
| 306 | return nil |
| 307 | } |
| 308 | |
