From e89c298c5dc9d51201e15419c5757decf025d34c Mon Sep 17 00:00:00 2001 From: medanisjbara Date: Tue, 28 Mar 2023 13:55:37 +0100 Subject: [PATCH 01/25] add basic gist support --- pages/gist.go | 90 +++++++++++++++++++++++++++++++++++++++++++++++++ serve/serve.go | 15 ++++++++- views/gist.html | 57 +++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 pages/gist.go create mode 100644 views/gist.html diff --git a/pages/gist.go b/pages/gist.go new file mode 100644 index 0000000..be20487 --- /dev/null +++ b/pages/gist.go @@ -0,0 +1,90 @@ +package pages + +import ( + "github.com/gocolly/colly" + "github.com/gofiber/fiber/v2" + "log" + "net/http" + "os" +) + +type Gist struct { + Fullname string + Refname string + Description string + DownloadLink string + Parent string + Stars string + Forks string + Language []string + License string + DefaultBranch string + Readme string +} + +type GistFiles struct { + Name string +} + +func HandleGist(c *fiber.Ctx) error { + var gistArray []Gist + var gistFilesArray []GistFiles + + resp, statusErr := http.Get("https://gists.github.com/" + c.Params("user") + "/" + c.Params("repo")) + if statusErr != nil { + log.Println(statusErr) + } + if resp.StatusCode == 404 { + // I need a better way to do this + return c.Status(404).Render("error", fiber.Map{ + "title": "Error", + "error": "Repository " + c.Params("user") + "/" + c.Params("repo") + " not found", + }) + } + + // Scraping + Scrape := Gist{} + + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + + sc := colly.NewCollector(colly.AllowedDomains("gist.github.com"), colly.UserAgent(UserAgent)) + + sc.OnHTML("div.gisthead", func(e *colly.HTMLElement) { + Scrape.Refname = c.Params("user") + "/" + c.Params("repo") + Scrape.Fullname = c.Params("user") + "/" + e.ChildText("a[href='/"+c.Params("user")+"/"+c.Params("repo")+"' i]") + Scrape.DownloadLink = e.ChildAttr("a[href$='.zip']", "href") + Scrape.Stars = e.ChildText("ul li a:first") + Scrape.Stars = e.ChildText("li a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/stargazers' i]") + Scrape.Parent = e.ChildText("span.note a") + Scrape.Forks = e.ChildText("li span.social-count") + // I need a better way to do this + if Scrape.Forks == "" { + Scrape.Forks = e.ChildText("li a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/forks' i]") + } + }) + + sc.OnHTML("div[itemprop='about']", func(e *colly.HTMLElement) { + Scrape.Description = e.Text + }) + + sc.OnHTML("div.repository-content.gist-content", func(e *colly.HTMLElement) { + e.ForEach("strong.user-select-contain.gist-blob-name.css-truncate-target", func(i int, el *colly.HTMLElement) { + gistFilesArray = append(gistFilesArray, GistFiles{ + Name: el.Text, + }) + }) + }) + + sc.Visit("https://gist.github.com/" + c.Params("user") + "/" + c.Params("repo") + "/") + // Add scrape-based info to gistArray + gistArray = append(gistArray, Scrape) + + return c.Render("gist", fiber.Map{ + "title": "Repository " + c.Params("user") + "/" + c.Params("repo"), + "gist": gistArray, + "files": gistFilesArray, + }) +} diff --git a/serve/serve.go b/serve/serve.go index 3c9cc18..62fdb40 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -3,6 +3,7 @@ package serve import ( "html/template" "log" + "net/url" "os" "regexp" "runtime" @@ -108,7 +109,19 @@ func Serve(port string) { utils.ProxyRequest(c, "https://avatars.githubusercontent.com/u/"+c.Params("id")+"?v=4") return nil }) - app.Get("/:user/:repo", pages.HandleRepo) + app.Get("/:user/:repo", ratelimiter, func(c *fiber.Ctx) error { + u, err := url.Parse(c.Request().URI().String()) + if err != nil { + return err + } + params, _ := url.ParseQuery(u.RawQuery) + typ := params.Get("type") + + if typ == "gist" { + return pages.HandleGist(c) + } + return pages.HandleRepo(c) + }) app.Get("/:user/:repo/blob/:branch/+", pages.FileView) app.Get("/:user/:repo/tree/:branch/+", pages.DirView) app.Get("/:user/:repo/tree/:branch", pages.HandleRepo) diff --git a/views/gist.html b/views/gist.html new file mode 100644 index 0000000..2b61a02 --- /dev/null +++ b/views/gist.html @@ -0,0 +1,57 @@ +{{ template "header" .}} + +
+ {{ if .gist }} + {{ range $key, $value := .gist}} +
+ Download (zip) + View on GitHub +
+ +
+

{{.Fullname}}

+ {{ if .Parent }} +

This repository is a fork of {{.Parent}}.

+ {{ end }} + {{ if .Description }} +

{{.Description}}

+ {{ end }} +

⭐ {{.Stars}} 🍴 {{.Forks}}

+ {{ if .Language }} +

🗒️ + {{range .Language}} + {{.}} + {{end}} +

+ {{end}} +
+ {{end}} + {{ if .files}} +
+

Files

+
+
    + {{ range $key, $value := .files}} +
  • {{.Name}}
    • + {{ end }} +
+
+
+ {{ end }} + {{ if .readme}} +
+ {{ if .repo }} + {{ range $key, $value := .repo}} +

{{.Readme}}

+ {{end}} + {{end}} +
+ {{ unescape .readme}} +
+
+ {{ end }} + {{ else }} +

Repository not found

+

That repository doesn't exist.

+ {{ end }} +
From b9db8fb9d59be4a97ca717b00b83de733afa16cb Mon Sep 17 00:00:00 2001 From: medanisjbara Date: Tue, 28 Mar 2023 14:05:33 +0100 Subject: [PATCH 02/25] add forkof to gists --- serve/serve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serve/serve.go b/serve/serve.go index 62fdb40..7342622 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -109,7 +109,7 @@ func Serve(port string) { utils.ProxyRequest(c, "https://avatars.githubusercontent.com/u/"+c.Params("id")+"?v=4") return nil }) - app.Get("/:user/:repo", ratelimiter, func(c *fiber.Ctx) error { + app.Get("/:user/:repo", func(c *fiber.Ctx) error { u, err := url.Parse(c.Request().URI().String()) if err != nil { return err From 877b8e0e1086331dfda335353064f2f31bcafb92 Mon Sep 17 00:00:00 2001 From: medanisjbara Date: Tue, 28 Mar 2023 14:22:38 +0100 Subject: [PATCH 03/25] beatify views --- views/dir.html | 7 +++++-- views/gist.html | 4 ++-- views/repo.html | 4 +++- views/user.html | 4 +++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/views/dir.html b/views/dir.html index 61e23ce..151210f 100644 --- a/views/dir.html +++ b/views/dir.html @@ -3,10 +3,13 @@
{{ if .dir }} {{ range $key, $value := .dir}}
- View on GitHub - Back to {{.Fullname}}
diff --git a/views/gist.html b/views/gist.html index 2b61a02..f44082b 100644 --- a/views/gist.html +++ b/views/gist.html @@ -41,10 +41,10 @@ {{ if .readme}}
{{ if .repo }} - {{ range $key, $value := .repo}} + {{ range $key, $value := .repo}}

{{.Readme}}

{{end}} - {{end}} + {{end}}
{{ unescape .readme}}
diff --git a/views/repo.html b/views/repo.html index b735162..5c4a852 100644 --- a/views/repo.html +++ b/views/repo.html @@ -6,7 +6,9 @@ Download (zip) - View on GitHub + View on GitHub