From 41c503f00c6ac222027f555e0979acece747847e Mon Sep 17 00:00:00 2001 From: Odyssey Date: Fri, 6 Jan 2023 21:13:53 +0100 Subject: [PATCH 01/87] Make proxied requests a utility function Signed-off-by: Odyssey --- serve/serve.go | 22 +++------------------- utils/proxyrequest.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 utils/proxyrequest.go diff --git a/serve/serve.go b/serve/serve.go index e876d21..1bb7698 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -22,7 +22,6 @@ import ( "github.com/gofiber/fiber/v2/middleware/cache" "github.com/gofiber/fiber/v2/middleware/compress" "github.com/gofiber/fiber/v2/middleware/limiter" - "github.com/gofiber/fiber/v2/middleware/proxy" "github.com/gofiber/fiber/v2/middleware/recover" "github.com/gofiber/template/html" ) @@ -93,12 +92,7 @@ func Serve() { app.Get("/explore", ratelimiter, pages.HandleExplore) app.Get("/:user", ratelimiter, pages.HandleUser) app.Get("/avatar/:id", func(c *fiber.Ctx) error { - url := "https://avatars.githubusercontent.com/u/" + c.Params("id") + "?v=4" - if err := proxy.Do(c, url); err != nil { - return err - } - // Remove Server header from response - c.Response().Header.Del(fiber.HeaderServer) + utils.ProxyRequest(c, "https://avatars.githubusercontent.com/u/"+c.Params("id")+"?v=4") return nil }) app.Get("/:user/:repo", ratelimiter, pages.HandleRepo) @@ -154,21 +148,11 @@ func Serve() { }) }) app.Get("/download/:user/:repo/:branch", ratelimiter, func(c *fiber.Ctx) error { - url := "https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/archive/" + c.Params("branch") + ".zip" - if err := proxy.Do(c, url); err != nil { - return err - } - // Remove Server header from response - c.Response().Header.Del(fiber.HeaderServer) + utils.ProxyRequest(c, "https://github.com/"+c.Params("user")+"/"+c.Params("repo")+"/archive/"+c.Params("branch")+".zip") return nil }) app.Get("/raw/:user/:repo/:branch/:file", ratelimiter, func(c *fiber.Ctx) error { - url := "https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + c.Params("branch") + "/" + c.Params("file") - if err := proxy.Do(c, url); err != nil { - return err - } - // Remove Server header from response - c.Response().Header.Del(fiber.HeaderServer) + utils.ProxyRequest(c, "https://raw.githubusercontent.com/"+c.Params("user")+"/"+c.Params("repo")+"/"+c.Params("branch")+"/"+c.Params("file")) return nil }) diff --git a/utils/proxyrequest.go b/utils/proxyrequest.go new file mode 100644 index 0000000..28d0266 --- /dev/null +++ b/utils/proxyrequest.go @@ -0,0 +1,16 @@ +package utils + +import ( + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/proxy" +) + +// ProxyRequest proxies requests to GitHub media +func ProxyRequest(c *fiber.Ctx, url string) error { + if err := proxy.Do(c, url); err != nil { + return err + } + // Remove Server header from response + c.Response().Header.Del(fiber.HeaderServer) + return nil +} From 666f5d9e82a428cd1faa085d8c22a62093d03559 Mon Sep 17 00:00:00 2001 From: Odyssium Date: Fri, 13 Jan 2023 19:35:45 +0000 Subject: [PATCH 02/87] Deprecate wiki --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b5262d0..d88bebc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,24 @@ Alternative front-end for GitHub written with Go. (WIP) - Save bandwidth - no JavaScript/tracking bloat loaded! ## Instances -The instances list is in the [wiki](https://codeberg.org/gothub/gothub/wiki/Instances). +### Master Branch +These instances run the ``master`` branch of GotHub. This means that the version of GotHub they're running has been tested and approved by at least two GotHub maintainers. + +| Link | Cloudflare | Country | ISP | +| -------- | ---------- | ----------- | ----- | +| [gh.odyssey346.dev (official)](https://gh.odyssey346.dev) | No | 🇫🇷 | OVHcloud (Trolling Solutions) | +| [gh.riverside.rocks](https://gh.riverside.rocks) | No | 🇺🇸 | Comcast | +| [gh.vern.cc](https://gh.vern.cc)[^1] | No | 🇺🇸 | Hetzner | + +### Dev Branch +These instances run the ``dev`` branch of GotHub. While this might mean they are always up to date, they might not be. However, this does mean that you can get the latest and greatest, but could potentially have bugs and other nasty stuff. +| Link | Cloudflare | Country | ISP | +| -------- | ---------- | ----------- | ----- | +| [gh.dev.odyssey346.dev (official)](https://gh.dev.odyssey346.dev) | No | 🇫🇷 | OVHcloud (Trolling Solutions) | + +Have an instance you want on the list? Make an issue or PR! + +[^1]: Has some modifications: https://git.vern.cc/vern/modifications/src/branch/master/gothub ## Setup ### Docker From fc22e06a6d50df00980cb1a241cbdd1935cb611b Mon Sep 17 00:00:00 2001 From: Akis Date: Fri, 13 Jan 2023 19:40:17 +0000 Subject: [PATCH 03/87] Add gh.akisblack.dev instance --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d88bebc..63a3530 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ These instances run the ``master`` branch of GotHub. This means that the version | [gh.odyssey346.dev (official)](https://gh.odyssey346.dev) | No | 🇫🇷 | OVHcloud (Trolling Solutions) | | [gh.riverside.rocks](https://gh.riverside.rocks) | No | 🇺🇸 | Comcast | | [gh.vern.cc](https://gh.vern.cc)[^1] | No | 🇺🇸 | Hetzner | +| [gh.akisblack.dev](https://gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | ### Dev Branch These instances run the ``dev`` branch of GotHub. While this might mean they are always up to date, they might not be. However, this does mean that you can get the latest and greatest, but could potentially have bugs and other nasty stuff. From f3504e7ff199aac2b23dc8ce29199f64015f5e2e Mon Sep 17 00:00:00 2001 From: Odyssium Date: Fri, 13 Jan 2023 20:31:06 +0000 Subject: [PATCH 04/87] Add Drivet instance --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 63a3530..914f0f1 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ These instances run the ``master`` branch of GotHub. This means that the version | [gh.riverside.rocks](https://gh.riverside.rocks) | No | 🇺🇸 | Comcast | | [gh.vern.cc](https://gh.vern.cc)[^1] | No | 🇺🇸 | Hetzner | | [gh.akisblack.dev](https://gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | +| [gh.drivet.xyz](https://gh.drivet.xyz) | No | 🇫🇷, 🇸🇪 & 🇫🇮 | OVHcloud (France), Oracle Cloud (Sweden) and Elisa (Finland, hosted on a Raspberry Pi) | ### Dev Branch These instances run the ``dev`` branch of GotHub. While this might mean they are always up to date, they might not be. However, this does mean that you can get the latest and greatest, but could potentially have bugs and other nasty stuff. From 8b1188468c016749ada1c15a67d3f605b367e231 Mon Sep 17 00:00:00 2001 From: Odyssium Date: Sat, 14 Jan 2023 11:24:49 +0000 Subject: [PATCH 05/87] Add esmailelbob instance --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 914f0f1..562dd7a 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ These instances run the ``dev`` branch of GotHub. While this might mean they are | Link | Cloudflare | Country | ISP | | -------- | ---------- | ----------- | ----- | | [gh.dev.odyssey346.dev (official)](https://gh.dev.odyssey346.dev) | No | 🇫🇷 | OVHcloud (Trolling Solutions) | +| [gothub.esmailelbob.xyz](https://gothub.esmailelbob.xyz) | No | 🇨🇦 | OVHcloud | Have an instance you want on the list? Make an issue or PR! From c13376e5945b1a8959bd62326194ebc682d83b10 Mon Sep 17 00:00:00 2001 From: Odyssey Date: Sat, 14 Jan 2023 12:39:49 +0100 Subject: [PATCH 06/87] Try an issue template Signed-off-by: Odyssey --- ISSUE_TEMPLATE.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 ISSUE_TEMPLATE.md diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..20cbf92 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,39 @@ +--- + +name: "Instance Requst" +about: "Have an instance you want to add to the list? Use this template." +title: "[New Instance] " +ref: "main" +labels: + +- instance + +--- + + + +## Instance Request + + +Where is your server hosted? (e.g. OVHcloud, Hetzner, Telenor, etc.): + + +Where is your server located? (e.g. Germany, Norway, etc.): + + + +Are you using Cloudflare? (yes/no): + + +Where can we reach your instance? (e.g. https://gh.yourdoma.in): + + +Which branch of GotHub are you using? (e.g. master, dev, "latest" image on Docker): + + + +Anything else we should know? (e.g. if you are using a modified version of GotHub, etc.): + +## Contact Details + + From 366fe77002249e2e5980a74e2dbbbf654851791d Mon Sep 17 00:00:00 2001 From: Odyssey Date: Sat, 14 Jan 2023 12:41:36 +0100 Subject: [PATCH 07/87] idk Signed-off-by: Odyssey --- ISSUE_TEMPLATE.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 20cbf92..722e3fb 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,3 +1,11 @@ +--- +name: "Other" +about: "Use this template for anything else." +title: "" +ref: "other" + +--- + --- name: "Instance Requst" From faaecd2b117e2dadacf39ef257b91622a5c5a20d Mon Sep 17 00:00:00 2001 From: Odyssey Date: Sat, 14 Jan 2023 12:42:40 +0100 Subject: [PATCH 08/87] I don't know how Gitea issue templates work. Signed-off-by: Odyssey --- ISSUE_TEMPLATE.md | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 722e3fb..052bf23 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,14 +1,6 @@ ---- -name: "Other" -about: "Use this template for anything else." -title: "" -ref: "other" - --- ---- - -name: "Instance Requst" +name: "Instance Request" about: "Have an instance you want to add to the list? Use this template." title: "[New Instance] " ref: "main" @@ -18,7 +10,7 @@ labels: --- - + ## Instance Request From eececefec1439434486cae4b5c144d33d9e5281f Mon Sep 17 00:00:00 2001 From: Odyssey Date: Sat, 14 Jan 2023 12:44:12 +0100 Subject: [PATCH 09/87] Add lunar.icu instance Signed-off-by: Odyssey --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 562dd7a..f65f5e2 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ These instances run the ``master`` branch of GotHub. This means that the version | [gh.vern.cc](https://gh.vern.cc)[^1] | No | 🇺🇸 | Hetzner | | [gh.akisblack.dev](https://gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | | [gh.drivet.xyz](https://gh.drivet.xyz) | No | 🇫🇷, 🇸🇪 & 🇫🇮 | OVHcloud (France), Oracle Cloud (Sweden) and Elisa (Finland, hosted on a Raspberry Pi) | +| [gothub.lunar.icu](https://gothub.lunar.icu) | Yes | 🇩🇪 | Unesty | ### Dev Branch These instances run the ``dev`` branch of GotHub. While this might mean they are always up to date, they might not be. However, this does mean that you can get the latest and greatest, but could potentially have bugs and other nasty stuff. From 34e3fb81a07ddbabbcbbbb0687d8952192f36f11 Mon Sep 17 00:00:00 2001 From: tolstoevsky Date: Mon, 16 Jan 2023 17:16:41 +0000 Subject: [PATCH 10/87] gh.phreedom.club added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f65f5e2..ce9b901 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ These instances run the ``master`` branch of GotHub. This means that the version | [gh.akisblack.dev](https://gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | | [gh.drivet.xyz](https://gh.drivet.xyz) | No | 🇫🇷, 🇸🇪 & 🇫🇮 | OVHcloud (France), Oracle Cloud (Sweden) and Elisa (Finland, hosted on a Raspberry Pi) | | [gothub.lunar.icu](https://gothub.lunar.icu) | Yes | 🇩🇪 | Unesty | +| [gh.phreedom.club](https://gh.phreedom.club) | No | 🇸🇨 | Flokinet | ### Dev Branch These instances run the ``dev`` branch of GotHub. While this might mean they are always up to date, they might not be. However, this does mean that you can get the latest and greatest, but could potentially have bugs and other nasty stuff. From dba4708a3bc75721fff90d2935b74db1f2e7da3d Mon Sep 17 00:00:00 2001 From: Odyssium Date: Sun, 5 Feb 2023 18:23:37 +0000 Subject: [PATCH 11/87] =?UTF-8?q?=F0=9F=AB=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Odyssium --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ce9b901..b38dd8e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ These instances run the ``master`` branch of GotHub. This means that the version | [gh.riverside.rocks](https://gh.riverside.rocks) | No | 🇺🇸 | Comcast | | [gh.vern.cc](https://gh.vern.cc)[^1] | No | 🇺🇸 | Hetzner | | [gh.akisblack.dev](https://gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | -| [gh.drivet.xyz](https://gh.drivet.xyz) | No | 🇫🇷, 🇸🇪 & 🇫🇮 | OVHcloud (France), Oracle Cloud (Sweden) and Elisa (Finland, hosted on a Raspberry Pi) | | [gothub.lunar.icu](https://gothub.lunar.icu) | Yes | 🇩🇪 | Unesty | | [gh.phreedom.club](https://gh.phreedom.club) | No | 🇸🇨 | Flokinet | From d149207a3b1c0f09a62b0831b047d16fa8835db0 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 6 Feb 2023 20:53:09 +0530 Subject: [PATCH 12/87] add support for .github/profile for user README --- pages/user.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pages/user.go b/pages/user.go index a09070a..4971fb8 100644 --- a/pages/user.go +++ b/pages/user.go @@ -70,8 +70,14 @@ func HandleUser(c *fiber.Ctx) error { ToString(&readmee). Fetch(context.Background()) if err != nil { - readmee = "" - log.Println(err) + err2 := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err2 != nil { + readmee = "" + log.Println(err) + } } mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) From 7d7865b86ae79b6eabb757317ac4f797227010d3 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 19:47:06 +0530 Subject: [PATCH 13/87] add basic scraping Signed-off-by: Arya Kiran --- go.mod | 16 +++++++++++ go.sum | 40 ++++++++++++++++++++++++++ pages/user.go | 78 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 122 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 9729f25..850c943 100644 --- a/go.mod +++ b/go.mod @@ -14,17 +14,30 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.3.0 // indirect + github.com/antchfx/xmlquery v1.3.15 // indirect + github.com/antchfx/xpath v1.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/gocolly/colly v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jawher/mow.cli v1.2.0 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.15.14 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/rivo/uniseg v0.4.3 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -32,4 +45,7 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 8ba98c8..ae4e832 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw= github.com/alecthomas/chroma/v2 v2.4.0 h1:Loe2ZjT5x3q1bcWwemqyqEi8p11/IV/ncFCeLYDpWC4= github.com/alecthomas/chroma/v2 v2.4.0/go.mod h1:6kHzqF5O6FUSJzBXW7fXELjb+e+7OXW4UpoPqMO7IBQ= @@ -64,6 +66,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= +github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= +github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= +github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= +github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -136,6 +146,10 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.40.1/go.mod h1:Gko04sLksnHbzLSRBFWPFdzM9Ws9pRxvvIaohJK1dsk= github.com/gofiber/fiber/v2 v2.41.0 h1:YhNoUS/OTjEz+/WLYuQ01xI7RXgKEFnGBKMagAu5f0M= @@ -148,6 +162,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -174,6 +189,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK0Ja9a3OUa2Fo+EaN0cbLu0eKpBwPFzc8= @@ -258,6 +274,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jawher/mow.cli v1.2.0 h1:e6ViPPy+82A/NFF/cfbq3Lr6q4JHKT9tyHwTCcUQgQw= +github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -265,6 +283,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -347,6 +367,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -363,6 +385,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -370,6 +393,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -392,6 +417,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= @@ -416,6 +442,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -454,6 +481,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -496,8 +524,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= @@ -529,6 +559,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -595,12 +626,15 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -610,6 +644,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -668,6 +704,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -710,6 +747,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -815,6 +853,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pages/user.go b/pages/user.go index 4971fb8..338f372 100644 --- a/pages/user.go +++ b/pages/user.go @@ -1,15 +1,15 @@ package pages import ( + "codeberg.org/gothub/gothub/utils" "fmt" - "log" - "github.com/carlmjohnson/requests" + "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" "github.com/microcosm-cc/bluemonday" - - "codeberg.org/gothub/gothub/utils" + "log" + "strconv" "context" "regexp" @@ -99,23 +99,77 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Has no prefix") } } + // scraping + var OrgOrUser string + var BioScrape string + var LocScrape string + var WorkplaceScrape string + var AvatarUrlScrape string + var FollowingScrape string + var FollowerScrape string + var UsernameScrape string + var FullNameScrape string + sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) + sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { + OrgOrUser = e.Attr("itemtype") + }) + sc1.Visit("https://github.com/" + c.Params("user") + "/") + sc := colly.NewCollector( + colly.AllowedDomains("github.com"), + ) + if OrgOrUser == "http://schema.org/Person" { + // Bio + sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { + BioScrape = e.Attr("data-bio-text") + }) + // Avatar + sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { + AvatarUrlScrape = e.Attr("src") + }) + // Metadata (Location/Workplace/Website/Twitter etc.) + sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { + LocScrape = e.ChildText("li[itemprop*='homeLocation'] span") + WorkplaceScrape = e.ChildText("li[itemprop*='worksFor'] span") + }) + // Followers/Following + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { + FollowerScrape = e.ChildText("span") + }) + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { + FollowingScrape = e.ChildText("span") + }) + // User/Full Name + sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { + UsernameScrape = e.ChildText("span[itemprop*='additionalName']") + FullNameScrape = e.ChildText("span[itemprop*='name']") + }) + } else { + sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { + AvatarUrlScrape = e.Attr("src") + }) + log.Println("Bio and Location routes cannot be scraped for organizations") + } + sc.Visit("https://github.com/" + c.Params("user") + "/") + FollowingScrapeInt, err := strconv.ParseInt(FollowingScrape, 10, 64) + FollowerScrapeInt, err := strconv.ParseInt(FollowerScrape, 10, 64) userArray = append(userArray, User{ - Login: user.Get("login").String(), - Name: user.Get("name").String(), - Bio: user.Get("bio").String(), - AvatarUrl: "/avatar/" + user.Get("id").String(), - Location: user.Get("location").String(), - Following: user.Get("following").Int(), - Followers: user.Get("followers").Int(), + Login: UsernameScrape, + Name: FullNameScrape, + Bio: BioScrape, + AvatarUrl: AvatarUrlScrape, + Location: LocScrape, + Following: FollowingScrapeInt, + Followers: FollowerScrapeInt, Link: link, - Company: user.Get("company").String(), + Company: WorkplaceScrape, Type: user.Get("type").String(), EwTwitter: user.Get("twitter_username").String(), Readme: string(readmeOutput), }) fmt.Println(userArray) + return c.Render("user", fiber.Map{ "Title": "User " + c.Params("user"), From 652ccd0952d627d11b2ce9fc30924e729bf25e01 Mon Sep 17 00:00:00 2001 From: Odyssey Date: Tue, 7 Feb 2023 15:35:44 +0100 Subject: [PATCH 14/87] cleanup serve/serve.go a bit Signed-off-by: Odyssey --- go.mod | 1 + go.sum | 3 +++ pages/fileview.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++ serve/serve.go | 67 +++-------------------------------------------- 4 files changed, 74 insertions(+), 64 deletions(-) create mode 100644 pages/fileview.go diff --git a/go.mod b/go.mod index 9729f25..d6aaab7 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( ) require ( + github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect diff --git a/go.sum b/go.sum index 8ba98c8..844adec 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKz github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw= +github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= +github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/alecthomas/chroma/v2 v2.4.0 h1:Loe2ZjT5x3q1bcWwemqyqEi8p11/IV/ncFCeLYDpWC4= github.com/alecthomas/chroma/v2 v2.4.0/go.mod h1:6kHzqF5O6FUSJzBXW7fXELjb+e+7OXW4UpoPqMO7IBQ= github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= @@ -108,6 +110,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= diff --git a/pages/fileview.go b/pages/fileview.go new file mode 100644 index 0000000..e504726 --- /dev/null +++ b/pages/fileview.go @@ -0,0 +1,67 @@ +package pages + +import ( + "bufio" + "bytes" + "context" + "html/template" + + "github.com/alecthomas/chroma/lexers" + "github.com/alecthomas/chroma/styles" + htmlfmt "github.com/alecthomas/chroma/v2/formatters/html" + "github.com/carlmjohnson/requests" + "github.com/gofiber/fiber/v2" +) + +// FileView is the file view page +func FileView(c *fiber.Ctx) error { + var file string + url := "https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + c.Params("branch") + "/" + c.Params("file") + err := requests. + URL(url). + ToString(&file). + Fetch(context.Background()) + if err != nil { + return c.Status(404).Render("error", fiber.Map{ + "error": err, + }) + } + + lexer := lexers.Match(c.Params("file")) + if lexer == nil { + lexer = lexers.Fallback + } + + formatter := htmlfmt.New( + htmlfmt.WithClasses(true), + htmlfmt.WithLineNumbers(true), + htmlfmt.PreventSurroundingPre(false), + ) + + buf := bytes.Buffer{} + writer := bufio.NewWriter(&buf) + style := styles.Get("native") + + tokens, err := lexer.Tokenise(nil, file) + if err != nil { + return c.Status(500).Render("error", fiber.Map{ + "error": err, + }) + } + formatter.Format(writer, style, tokens) + + cssbuf := bytes.Buffer{} + cssw := bufio.NewWriter(&cssbuf) + formatter.WriteCSS(cssw, style) + + writer.Flush() + cssw.Flush() + + return c.Render("file", fiber.Map{ + "file": template.HTML(buf.String()), + "css": template.CSS(cssbuf.String()), + "fullname": c.Params("user") + "/" + c.Params("repo"), + "name": c.Params("file"), + "branch": c.Params("branch"), + }) +} diff --git a/serve/serve.go b/serve/serve.go index 1bb7698..3962ab4 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -1,9 +1,6 @@ package serve import ( - "bufio" - "bytes" - "context" "html/template" "log" "os" @@ -12,11 +9,6 @@ import ( "codeberg.org/gothub/gothub/pages" "codeberg.org/gothub/gothub/utils" - "github.com/carlmjohnson/requests" - - htmlfmt "github.com/alecthomas/chroma/v2/formatters/html" - "github.com/alecthomas/chroma/v2/lexers" - "github.com/alecthomas/chroma/v2/styles" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/cache" @@ -40,7 +32,7 @@ func Serve() { Views: engine, Prefork: false, AppName: "GotHub", - // fucked up way to fix rate limits + // kind of screwed up way to fix rate limits EnableTrustedProxyCheck: true, TrustedProxies: []string{"0.0.0.0/0"}, ProxyHeader: fiber.HeaderXForwardedFor, @@ -66,8 +58,6 @@ func Serve() { Expiration: 5 * time.Minute, })) - app.Get("/", pages.HandleIndex) - app.Use(compress.New(compress.Config{ Level: compress.LevelBestSpeed, // 1 })) @@ -84,6 +74,7 @@ func Serve() { }, }) + app.Get("/", pages.HandleIndex) app.Static("/css", "./public/css") app.Static("/fonts", "./public/fonts") app.Static("/robots.txt", "./public/robots.txt") @@ -96,57 +87,7 @@ func Serve() { return nil }) app.Get("/:user/:repo", ratelimiter, pages.HandleRepo) - app.Get("/file/:user/:repo/:branch/:file", func(c *fiber.Ctx) error { - var file string - url := "https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + c.Params("branch") + "/" + c.Params("file") - err := requests. - URL(url). - ToString(&file). - Fetch(context.Background()) - if err != nil { - return c.Status(404).Render("error", fiber.Map{ - "error": err, - }) - } - - lexer := lexers.Match(c.Params("file")) - if lexer == nil { - lexer = lexers.Fallback - } - - formatter := htmlfmt.New( - htmlfmt.WithClasses(true), - htmlfmt.WithLineNumbers(true), - htmlfmt.PreventSurroundingPre(false), - ) - - buf := bytes.Buffer{} - writer := bufio.NewWriter(&buf) - style := styles.Get("native") - - tokens, err := lexer.Tokenise(nil, file) - if err != nil { - return c.Status(500).Render("error", fiber.Map{ - "error": err, - }) - } - formatter.Format(writer, style, tokens) - - cssbuf := bytes.Buffer{} - cssw := bufio.NewWriter(&cssbuf) - formatter.WriteCSS(cssw, style) - - writer.Flush() - cssw.Flush() - - return c.Render("file", fiber.Map{ - "file": template.HTML(buf.String()), - "css": template.CSS(cssbuf.String()), - "fullname": c.Params("user") + "/" + c.Params("repo"), - "name": c.Params("file"), - "branch": c.Params("branch"), - }) - }) + app.Get("/file/:user/:repo/:branch/:file", pages.FileView) app.Get("/download/:user/:repo/:branch", ratelimiter, func(c *fiber.Ctx) error { utils.ProxyRequest(c, "https://github.com/"+c.Params("user")+"/"+c.Params("repo")+"/archive/"+c.Params("branch")+".zip") return nil @@ -169,8 +110,6 @@ func Serve() { val, ok := os.LookupEnv("GOTHUB_PORT") if !ok { val = "3000" - } else { - log.Println("Using port from env variable") } log.Fatal(app.Listen(":" + val)) } From 4db2d1188eecefa26b72b0ed85581d919660d13d Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 20:45:13 +0530 Subject: [PATCH 15/87] use user struct instead of declaring variables for scrape --- pages/user.go | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/pages/user.go b/pages/user.go index 338f372..9d73f8a 100644 --- a/pages/user.go +++ b/pages/user.go @@ -100,15 +100,8 @@ func HandleUser(c *fiber.Ctx) error { } } // scraping + var Scrape User var OrgOrUser string - var BioScrape string - var LocScrape string - var WorkplaceScrape string - var AvatarUrlScrape string - var FollowingScrape string - var FollowerScrape string - var UsernameScrape string - var FullNameScrape string sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { OrgOrUser = e.Attr("itemtype") @@ -120,49 +113,47 @@ func HandleUser(c *fiber.Ctx) error { if OrgOrUser == "http://schema.org/Person" { // Bio sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { - BioScrape = e.Attr("data-bio-text") + Scrape.Bio = e.Attr("data-bio-text") }) // Avatar sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { - AvatarUrlScrape = e.Attr("src") + Scrape.AvatarUrl = e.Attr("src") }) // Metadata (Location/Workplace/Website/Twitter etc.) sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { - LocScrape = e.ChildText("li[itemprop*='homeLocation'] span") - WorkplaceScrape = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") }) // Followers/Following sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { - FollowerScrape = e.ChildText("span") + Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) }) sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { - FollowingScrape = e.ChildText("span") + Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) }) // User/Full Name sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { - UsernameScrape = e.ChildText("span[itemprop*='additionalName']") - FullNameScrape = e.ChildText("span[itemprop*='name']") + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") }) } else { sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { - AvatarUrlScrape = e.Attr("src") + Scrape.AvatarUrl = e.Attr("src") }) log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - FollowingScrapeInt, err := strconv.ParseInt(FollowingScrape, 10, 64) - FollowerScrapeInt, err := strconv.ParseInt(FollowerScrape, 10, 64) userArray = append(userArray, User{ - Login: UsernameScrape, - Name: FullNameScrape, - Bio: BioScrape, - AvatarUrl: AvatarUrlScrape, - Location: LocScrape, - Following: FollowingScrapeInt, - Followers: FollowerScrapeInt, + Login: Scrape.Login, + Name: Scrape.Name, + Bio: Scrape.Bio, + AvatarUrl: Scrape.AvatarUrl, + Location: Scrape.Location, + Following: Scrape.Following, + Followers: Scrape.Followers, Link: link, - Company: WorkplaceScrape, + Company: Scrape.Company, Type: user.Get("type").String(), EwTwitter: user.Get("twitter_username").String(), Readme: string(readmeOutput), From 207fd7f52743f1e4aaa8d2fe677429595b559db0 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 21:12:18 +0530 Subject: [PATCH 16/87] cleanup the code a bit more --- pages/user.go | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/pages/user.go b/pages/user.go index 9d73f8a..d71efe9 100644 --- a/pages/user.go +++ b/pages/user.go @@ -100,16 +100,26 @@ func HandleUser(c *fiber.Ctx) error { } } // scraping - var Scrape User var OrgOrUser string + + Scrape := User{ + Link: link, + Type: user.Get("type").String(), + EwTwitter: user.Get("twitter_username").String(), + Readme: string(readmeOutput), + } + sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) + sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { OrgOrUser = e.Attr("itemtype") }) + sc1.Visit("https://github.com/" + c.Params("user") + "/") sc := colly.NewCollector( colly.AllowedDomains("github.com"), ) + if OrgOrUser == "http://schema.org/Person" { // Bio sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { @@ -144,20 +154,7 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - userArray = append(userArray, User{ - Login: Scrape.Login, - Name: Scrape.Name, - Bio: Scrape.Bio, - AvatarUrl: Scrape.AvatarUrl, - Location: Scrape.Location, - Following: Scrape.Following, - Followers: Scrape.Followers, - Link: link, - Company: Scrape.Company, - Type: user.Get("type").String(), - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), - }) + userArray = append(userArray, scrape) fmt.Println(userArray) From ad0c0848769d7ab9544fe7d4fa4988dc3fe64292 Mon Sep 17 00:00:00 2001 From: Odyssey Date: Tue, 7 Feb 2023 16:47:01 +0100 Subject: [PATCH 17/87] Use chroma v2 import Signed-off-by: Odyssey --- pages/fileview.go | 4 ++-- serve/serve.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pages/fileview.go b/pages/fileview.go index e504726..8601b9a 100644 --- a/pages/fileview.go +++ b/pages/fileview.go @@ -6,9 +6,9 @@ import ( "context" "html/template" - "github.com/alecthomas/chroma/lexers" - "github.com/alecthomas/chroma/styles" htmlfmt "github.com/alecthomas/chroma/v2/formatters/html" + "github.com/alecthomas/chroma/v2/lexers" + "github.com/alecthomas/chroma/v2/styles" "github.com/carlmjohnson/requests" "github.com/gofiber/fiber/v2" ) diff --git a/serve/serve.go b/serve/serve.go index 3962ab4..bd25d2a 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -22,7 +22,7 @@ func Serve() { engine := html.New("./views", ".html") engine.AddFunc( - // add unescape function + // Add unescape function. This is needed to render HTML from Markdown. "unescape", func(s string) template.HTML { return template.HTML(s) }, From 90cbacb80cf4d351e457fe76d7fcd79758c63d04 Mon Sep 17 00:00:00 2001 From: Akis Date: Tue, 7 Feb 2023 15:55:12 +0000 Subject: [PATCH 18/87] Add dev.gh.akisblack.dev instance --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b38dd8e..5ef46e5 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ These instances run the ``dev`` branch of GotHub. While this might mean they are | Link | Cloudflare | Country | ISP | | -------- | ---------- | ----------- | ----- | | [gh.dev.odyssey346.dev (official)](https://gh.dev.odyssey346.dev) | No | 🇫🇷 | OVHcloud (Trolling Solutions) | -| [gothub.esmailelbob.xyz](https://gothub.esmailelbob.xyz) | No | 🇨🇦 | OVHcloud | +| [gothub.esmailelbob.xyz](https://gothub.esmailelbob.xyz) | No | 🇨🇦 | OVHcloud | +| [dev.gh.akisblack.dev](https://dev.gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | Have an instance you want on the list? Make an issue or PR! From 9d966e1378e3db8b6fa6e6dc80a84bde35e1d839 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 21:37:28 +0530 Subject: [PATCH 19/87] fix typo --- pages/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/user.go b/pages/user.go index d71efe9..00cf947 100644 --- a/pages/user.go +++ b/pages/user.go @@ -154,7 +154,7 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - userArray = append(userArray, scrape) + userArray = append(userArray, Scrape) fmt.Println(userArray) From 4e3bb59c41d47c0fb51a8bf1b004cf848814ab1d Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 15:56:40 +0530 Subject: [PATCH 20/87] make the user page mostly work when api is disabled --- pages/user.go | 175 +++++++++++++++++++++++------------------------- views/user.html | 4 +- 2 files changed, 87 insertions(+), 92 deletions(-) diff --git a/pages/user.go b/pages/user.go index 00cf947..e45754f 100644 --- a/pages/user.go +++ b/pages/user.go @@ -2,7 +2,6 @@ package pages import ( "codeberg.org/gothub/gothub/utils" - "fmt" "github.com/carlmjohnson/requests" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" @@ -38,6 +37,10 @@ type Ratelimit struct { // HandleUser handles the user page. func HandleUser(c *fiber.Ctx) error { + // Declare Array used for displaying data + var userArray []User + + // API user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) if user.Get("message").String() == "Not Found" { return c.Status(404).Render("error", fiber.Map{ @@ -46,7 +49,7 @@ func HandleUser(c *fiber.Ctx) error { } if strings.Contains(user.Get("message").String(), "rate limit") { // dont wanna get the status code so i'll just do this instead 👍 ratelimitJSON := utils.GetRequest("https://api.github.com/rate_limit") - fmt.Println(ratelimitJSON) + log.Println(ratelimitJSON) var ratelimitArray []Ratelimit ratelimitArray = append(ratelimitArray, Ratelimit{ @@ -54,36 +57,13 @@ func HandleUser(c *fiber.Ctx) error { Limit: ratelimitJSON.Get("resources.core.limit").Int(), }) - fmt.Println(ratelimitArray) + log.Println(ratelimitArray) return c.Render("ratelimit", fiber.Map{ "Title": "GitHub API /users endpoint rate limit exceeded", "ratelimit": ratelimitArray, }) } else { - var userArray []User - - var readmee string - - err := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). - ToString(&readmee). - Fetch(context.Background()) - if err != nil { - err2 := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). - ToString(&readmee). - Fetch(context.Background()) - if err2 != nil { - readmee = "" - log.Println(err) - } - } - - mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) - - // Trust Nobody - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) var link string @@ -99,71 +79,86 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Has no prefix") } } - // scraping - var OrgOrUser string - - Scrape := User{ - Link: link, - Type: user.Get("type").String(), - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), - } - - sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) - - sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { - OrgOrUser = e.Attr("itemtype") - }) - - sc1.Visit("https://github.com/" + c.Params("user") + "/") - sc := colly.NewCollector( - colly.AllowedDomains("github.com"), - ) - - if OrgOrUser == "http://schema.org/Person" { - // Bio - sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { - Scrape.Bio = e.Attr("data-bio-text") - }) - // Avatar - sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - }) - // Metadata (Location/Workplace/Website/Twitter etc.) - sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { - Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") - Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") - }) - // Followers/Following - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { - Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { - Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - // User/Full Name - sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { - Scrape.Login = e.ChildText("span[itemprop*='additionalName']") - Scrape.Name = e.ChildText("span[itemprop*='name']") - }) - - } else { - sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - }) - log.Println("Bio and Location routes cannot be scraped for organizations") - } - sc.Visit("https://github.com/" + c.Params("user") + "/") - userArray = append(userArray, Scrape) - - fmt.Println(userArray) - - - return c.Render("user", fiber.Map{ - "Title": "User " + c.Params("user"), - "user": userArray, - }) } + // User README + var readmee string + + err := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err != nil { + err2 := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err2 != nil { + readmee = "" + log.Println(err) + } + } + mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) + // Sanitize the user README incase there is any weird shit in it + readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) + + // scraping + Scrape := User{ + //Link: link, + EwTwitter: user.Get("twitter_username").String(), + Readme: string(readmeOutput), + } + + sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) + + sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { + Scrape.Type = e.Attr("itemtype") + }) + + sc1.Visit("https://github.com/" + c.Params("user") + "/") + sc := colly.NewCollector( + colly.AllowedDomains("github.com"), + ) + + if Scrape.Type == "http://schema.org/Person" { + // Bio + sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { + Scrape.Bio = e.Attr("data-bio-text") + }) + // Avatar + sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { + Scrape.AvatarUrl = e.Attr("src") + }) + // Metadata (Location/Workplace/Website/Twitter etc.) + sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { + Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + }) + // Followers/Following + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { + Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) + }) + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { + Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) + }) + // User/Full Name + sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") + }) + + } else { + sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { + Scrape.AvatarUrl = e.Attr("src") + }) + log.Println("Bio and Location routes cannot be scraped for organizations") + } + sc.Visit("https://github.com/" + c.Params("user") + "/") + userArray = append(userArray, Scrape) + + return c.Render("user", fiber.Map{ + "Title": "User " + c.Params("user"), + "user": userArray, + }) } // copied from bluemonday's GitHub repostiory, with some adaptations diff --git a/views/user.html b/views/user.html index 127751f..ddcb207 100644 --- a/views/user.html +++ b/views/user.html @@ -12,7 +12,7 @@

{{.Name}}

{{ end }}

{{.Login}}

- {{ if eq .Type "User" }} + {{ if eq .Type "http://schema.org/Person" }}

{{.Followers}} followers - {{.Following}} following

{{ else }}

{{.Followers}} followers

@@ -48,4 +48,4 @@

User not found

That user doesn't exist.

{{ end }} - \ No newline at end of file + From dca78ff062683c81dc85efa6b9f84a07add49ded Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 16:55:50 +0530 Subject: [PATCH 21/87] add api-less 404 checking --- pages/user.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pages/user.go b/pages/user.go index e45754f..4dffd37 100644 --- a/pages/user.go +++ b/pages/user.go @@ -2,16 +2,16 @@ package pages import ( "codeberg.org/gothub/gothub/utils" + "context" "github.com/carlmjohnson/requests" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" "github.com/microcosm-cc/bluemonday" "log" - "strconv" - - "context" + "net/http" "regexp" + "strconv" "strings" ) @@ -39,7 +39,15 @@ type Ratelimit struct { func HandleUser(c *fiber.Ctx) error { // Declare Array used for displaying data var userArray []User - + resp, err := http.Get("https://github.com/" + c.Params("user")) + if err != nil { + log.Println(err) + } + if resp.StatusCode == 404 { + return c.Status(404).Render("error", fiber.Map{ + "error": "User " + c.Params("user") + " not found", + }) + } // API user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) if user.Get("message").String() == "Not Found" { @@ -83,18 +91,18 @@ func HandleUser(c *fiber.Ctx) error { // User README var readmee string - err := requests. + err0 := requests. URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). ToString(&readmee). Fetch(context.Background()) - if err != nil { + if err0 != nil { err2 := requests. URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). ToString(&readmee). Fetch(context.Background()) if err2 != nil { readmee = "" - log.Println(err) + log.Println(err0) } } mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) From d0843f28cacf4be82414cea654f6d603a9d4ef42 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 17:33:38 +0530 Subject: [PATCH 22/87] make link global var --- pages/user.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/user.go b/pages/user.go index 4dffd37..de9e5d9 100644 --- a/pages/user.go +++ b/pages/user.go @@ -50,6 +50,7 @@ func HandleUser(c *fiber.Ctx) error { } // API user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) + var link string if user.Get("message").String() == "Not Found" { return c.Status(404).Render("error", fiber.Map{ "error": "User " + c.Params("user") + " not found", @@ -73,7 +74,6 @@ func HandleUser(c *fiber.Ctx) error { }) } else { - var link string if user.Get("blog").String() == "" { link = "" @@ -111,7 +111,7 @@ func HandleUser(c *fiber.Ctx) error { // scraping Scrape := User{ - //Link: link, + Link: link, EwTwitter: user.Get("twitter_username").String(), Readme: string(readmeOutput), } From fc19decc13aba4bccc68e5a0fdadb4631131fc3c Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 18:37:13 +0530 Subject: [PATCH 23/87] scrape twitter --- pages/user.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pages/user.go b/pages/user.go index de9e5d9..d2aa65e 100644 --- a/pages/user.go +++ b/pages/user.go @@ -74,7 +74,6 @@ func HandleUser(c *fiber.Ctx) error { }) } else { - if user.Get("blog").String() == "" { link = "" } else { @@ -111,9 +110,8 @@ func HandleUser(c *fiber.Ctx) error { // scraping Scrape := User{ - Link: link, - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), + Link: link, + Readme: string(readmeOutput), } sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) @@ -140,6 +138,7 @@ func HandleUser(c *fiber.Ctx) error { sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.EwTwitter = e.ChildText("a[href*='https://twitter.com/' i]") }) // Followers/Following sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { @@ -153,7 +152,6 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Login = e.ChildText("span[itemprop*='additionalName']") Scrape.Name = e.ChildText("span[itemprop*='name']") }) - } else { sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") @@ -161,6 +159,7 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") + Scrape.EwTwitter = strings.Trim(Scrape.EwTwitter, "@") userArray = append(userArray, Scrape) return c.Render("user", fiber.Map{ From 24cefa1afaeddcf8f965cac27a81c702a9cc9256 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 18:57:45 +0530 Subject: [PATCH 24/87] User contributions --- pages/user.go | 29 +++++++++++++++++------------ views/user.html | 5 +++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pages/user.go b/pages/user.go index d2aa65e..cad1318 100644 --- a/pages/user.go +++ b/pages/user.go @@ -16,18 +16,19 @@ import ( ) type User struct { - Login string - Name string - Bio string - AvatarUrl string - Location string - Following int64 - Followers int64 - Link string - Company string - Type string - EwTwitter string - Readme string + Login string + Name string + Bio string + AvatarUrl string + Location string + Following int64 + Followers int64 + Link string + Company string + Type string + EwTwitter string + Contributions string + Readme string } type Ratelimit struct { @@ -152,6 +153,10 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Login = e.ChildText("span[itemprop*='additionalName']") Scrape.Name = e.ChildText("span[itemprop*='name']") }) + // Contributions + sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { + Scrape.Contributions = e.ChildText("h2") + }) } else { sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") diff --git a/views/user.html b/views/user.html index ddcb207..6f007ef 100644 --- a/views/user.html +++ b/views/user.html @@ -20,6 +20,8 @@ {{ if .Location }}

🌍 {{.Location}}

{{ end }} + {{ if .Timezone }} +

🕑️ {{.Timezone}}

{{ if .Link }}

🔗 {{.Link}}

{{ end }} @@ -29,6 +31,9 @@ {{ if .EwTwitter }}

🐦 {{.EwTwitter}}

{{ end }} + {{ if eq .Type "http://schema.org/Person" }} +

✏️ {{ if .Name }} {{.Name}} {{ else }} {{.Login}} {{ end }} has made {{.Contributions}}

+ {{ end }} {{ if .Bio }} From 7a130bf8cca3ffb2e0d219d76ee9beeb72d658ad Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 19:25:03 +0530 Subject: [PATCH 25/87] add localtime --- pages/user.go | 2 ++ views/user.html | 1 + 2 files changed, 3 insertions(+) diff --git a/pages/user.go b/pages/user.go index cad1318..d1fdc40 100644 --- a/pages/user.go +++ b/pages/user.go @@ -21,6 +21,7 @@ type User struct { Bio string AvatarUrl string Location string + Timezone string Following int64 Followers int64 Link string @@ -138,6 +139,7 @@ func HandleUser(c *fiber.Ctx) error { // Metadata (Location/Workplace/Website/Twitter etc.) sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") Scrape.EwTwitter = e.ChildText("a[href*='https://twitter.com/' i]") }) diff --git a/views/user.html b/views/user.html index 6f007ef..e8d6fd0 100644 --- a/views/user.html +++ b/views/user.html @@ -22,6 +22,7 @@ {{ end }} {{ if .Timezone }}

🕑️ {{.Timezone}}

+ {{ end }} {{ if .Link }}

🔗 {{.Link}}

{{ end }} From 1e0afded3132c35a828e311849cc43389fa995e9 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 19:47:30 +0530 Subject: [PATCH 26/87] add social links support --- pages/user.go | 23 +++++++++++++++++++---- views/user.html | 17 +++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pages/user.go b/pages/user.go index d1fdc40..418e149 100644 --- a/pages/user.go +++ b/pages/user.go @@ -25,9 +25,12 @@ type User struct { Following int64 Followers int64 Link string + Link1 string + Link2 string + Link3 string + Link4 string Company string Type string - EwTwitter string Contributions string Readme string } @@ -136,12 +139,25 @@ func HandleUser(c *fiber.Ctx) error { sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") }) - // Metadata (Location/Workplace/Website/Twitter etc.) + // Metadata (Location/Workplace/Website/Social etc.) sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") - Scrape.EwTwitter = e.ChildText("a[href*='https://twitter.com/' i]") + // I know this is hacky, I'll get around to fixing it later -arya + e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { + switch i { + case 0: + Scrape.Link1 = el.ChildText("a.Link--primary") + case 1: + Scrape.Link2 = el.ChildText("a.Link--primary") + case 2: + Scrape.Link3 = el.ChildText("a.Link--primary") + case 3: + Scrape.Link4 = el.ChildText("a.Link--primary") + } + }) + }) // Followers/Following sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { @@ -166,7 +182,6 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - Scrape.EwTwitter = strings.Trim(Scrape.EwTwitter, "@") userArray = append(userArray, Scrape) return c.Render("user", fiber.Map{ diff --git a/views/user.html b/views/user.html index e8d6fd0..8600dde 100644 --- a/views/user.html +++ b/views/user.html @@ -24,13 +24,22 @@

🕑️ {{.Timezone}}

{{ end }} {{ if .Link }} -

🔗 {{.Link}}

+

🌐 {{.Link}}

+ {{ end }} + {{ if .Link1 }} +

🔗 {{.Link1}}

+ {{ end }} + {{ if .Link2 }} +

🔗 {{.Link2}}

+ {{ end }} + {{ if .Link3 }} +

🔗 {{.Link3}}

+ {{ end }} + {{ if .Link4 }} +

🔗 {{.Link4}}

{{ end }} {{ if .Company }}

🏢 {{.Company}}

- {{ end }} - {{ if .EwTwitter }} -

🐦 {{.EwTwitter}}

{{ end }} {{ if eq .Type "http://schema.org/Person" }}

✏️ {{ if .Name }} {{.Name}} {{ else }} {{.Login}} {{ end }} has made {{.Contributions}}

From a41e5ac009fd54ef3bc06b2fb0b9a6ce45ca427f Mon Sep 17 00:00:00 2001 From: Odyssey Date: Wed, 8 Feb 2023 17:32:22 +0100 Subject: [PATCH 27/87] Add caching headers to static files, and fix explicit width and height (closes #52) Signed-off-by: Odyssey --- .gitignore | 5 ++++- public/css/global.css | 4 ---- serve/serve.go | 16 +++++++++++----- views/header.html | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 2bfce47..61891e2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,7 @@ go.work tmp/ # Gothub binary. -gothub \ No newline at end of file +gothub + +# Fiber compressed files +*.fiber.gz \ No newline at end of file diff --git a/public/css/global.css b/public/css/global.css index ef3249b..005d91d 100644 --- a/public/css/global.css +++ b/public/css/global.css @@ -42,10 +42,6 @@ main { padding: 8px; } -.navbarImg { - height: 30px; -} - .indexGitHub { background: #252525; color: #fff; diff --git a/serve/serve.go b/serve/serve.go index bd25d2a..4d97bea 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -74,12 +74,18 @@ func Serve() { }, }) + staticConfig := fiber.Static{ + Compress: true, + // Cache-Control: max-age=31536000 + MaxAge: 31536000, + } + app.Get("/", pages.HandleIndex) - app.Static("/css", "./public/css") - app.Static("/fonts", "./public/fonts") - app.Static("/robots.txt", "./public/robots.txt") - app.Static("/favicon.ico", "./public/assets/favicon.ico") - app.Static("/logo.svg", "./public/assets/logo.svg") + app.Static("/css", "./public/css", staticConfig) + app.Static("/fonts", "./public/fonts", staticConfig) + app.Static("/robots.txt", "./public/robots.txt", staticConfig) + app.Static("/favicon.ico", "./public/assets/favicon.ico", staticConfig) + app.Static("/logo.svg", "./public/assets/logo.svg", staticConfig) app.Get("/explore", ratelimiter, pages.HandleExplore) app.Get("/:user", ratelimiter, pages.HandleUser) app.Get("/avatar/:id", func(c *fiber.Ctx) error { diff --git a/views/header.html b/views/header.html index bf876bd..833732f 100644 --- a/views/header.html +++ b/views/header.html @@ -13,7 +13,7 @@