From 8aef8830563e798d673d5b09b095420d18c292cd Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 14 Feb 2023 20:24:33 +0530 Subject: [PATCH 1/5] add basic metadata scraping for repos --- pages/repo.go | 83 ++++++++++++++++++++++++++++++++----------------- pages/user.go | 11 +++---- views/repo.html | 17 +++++----- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/pages/repo.go b/pages/repo.go index 987b6f3..1becc83 100644 --- a/pages/repo.go +++ b/pages/repo.go @@ -1,27 +1,28 @@ package pages import ( - "context" - "log" - "codeberg.org/gothub/gothub/utils" + "context" "github.com/carlmjohnson/requests" + "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" + "log" + "net/http" + "os" ) type Repo struct { Fullname string Description string - HtmlUrl string - Fork bool Parent string - Stars int64 - Forks int64 - Watchers int64 + Stars string + Forks string + Watchers string Language string License string DefaultBranch string + Readme string } type RepoFiles struct { @@ -35,14 +36,20 @@ type RepoFiles struct { func HandleRepo(c *fiber.Ctx) error { var repoArray []Repo var repoFilesArray []RepoFiles - // get repo - repo := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo")) - if repo.Get("message").String() == "Not Found" { + + resp, statusErr := http.Get("https://github.com/" + c.Params("user") + "/" + c.Params("repo")) + if statusErr != nil { + log.Println(statusErr) + } + if resp.StatusCode == 404 { + // I need a better way to do this return c.Status(404).Render("error", fiber.Map{ "title": "Error", "error": "Repository " + c.Params("user") + "/" + c.Params("repo") + " not found", }) } + // API + repo := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo")) repoFiles := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo") + "/contents") bruh := repoFiles.Get("#.@pretty").Array() for _, item := range bruh { @@ -55,36 +62,54 @@ func HandleRepo(c *fiber.Ctx) error { }) } - var readmee string + // Scraping + Scrape := Repo{ + Language: repo.Get("language").String(), + } + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + + sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) + sc.OnHTML("div.Layout-sidebar", func(e *colly.HTMLElement) { + Scrape.Fullname = c.Params("user") + "/" + c.Params("repo") + Scrape.Description = e.ChildText("p.f4") + Scrape.Stars = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/stargazers' i] strong") + Scrape.Watchers = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/watchers' i] strong") + Scrape.Forks = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/network/members' i] strong") + Scrape.License = e.ChildText("a[data-analytics-event*='{\"category\":\"Repository Overview\",\"action\":\"click\",\"label\":\"location:sidebar;file:license\"}']") + }) + sc.OnHTML("div#readme", func(e *colly.HTMLElement) { + Scrape.Readme = e.ChildText("a[href*='#readme']") + }) + sc.OnHTML("div#repository-container-header", func(e *colly.HTMLElement) { + Scrape.Parent = e.ChildText("span.text-small a") + }) + sc.OnHTML("summary[title*='Switch branches or tags']", func(e *colly.HTMLElement) { + Scrape.DefaultBranch = e.ChildText("span.css-truncate-target") + }) + + sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/") + + // Add scrape-based info to repoArray + repoArray = append(repoArray, Scrape) + + // README + var readmee string err := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + repo.Get("default_branch").String() + "/README.md"). + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + Scrape.DefaultBranch + "/" + Scrape.Readme). ToString(&readmee). Fetch(context.Background()) if err != nil { readmee = "" log.Println(err) } - mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) - // Trust Nobody readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) - repoArray = append(repoArray, Repo{ - Fullname: repo.Get("full_name").String(), - Description: repo.Get("description").String(), - HtmlUrl: repo.Get("html_url").String(), - Fork: repo.Get("fork").Bool(), - Stars: repo.Get("stargazers_count").Int(), - Forks: repo.Get("forks_count").Int(), - Watchers: repo.Get("watchers_count").Int(), - Language: repo.Get("language").String(), - License: repo.Get("license").Get("name").String(), - Parent: repo.Get("parent").Get("full_name").String(), - DefaultBranch: repo.Get("default_branch").String(), - }) - return c.Render("repo", fiber.Map{ "title": "Repository " + c.Params("user") + "/" + c.Params("repo"), "repo": repoArray, diff --git a/pages/user.go b/pages/user.go index c2c19ab..8269d46 100644 --- a/pages/user.go +++ b/pages/user.go @@ -5,7 +5,6 @@ import ( "log" "net/http" "os" - "strconv" "strings" "codeberg.org/gothub/gothub/utils" @@ -26,8 +25,8 @@ type User struct { Location string Email string Timezone string - Following int64 - Followers int64 + Following string + Followers string Link string Social []string Organizations []string @@ -108,8 +107,8 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary")) }) // Followers/Following - Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i] span"), 10, 64) - Scrape.Following, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i] span"), 10, 64) + Scrape.Followers = e.ChildText("a[href*='https://github.com/" + c.Params("user") + "?tab=followers' i] span") + Scrape.Following = e.ChildText("a[href*='https://github.com/" + c.Params("user") + "?tab=following' i] span") // Organizations e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) @@ -135,7 +134,7 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Social = append(Scrape.Social, el.Attr("href")) }) // Followers - Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64) + Scrape.Followers = e.ChildText("a[href*='/orgs/" + c.Params("user") + "/followers' i] span") }) sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") diff --git a/views/repo.html b/views/repo.html index 72d62da..8e145d9 100644 --- a/views/repo.html +++ b/views/repo.html @@ -10,20 +10,17 @@

{{.Fullname}}

- {{ if .Fork }} + {{ if .Parent }}

This repository is a fork of {{.Parent}}.

{{ end }} {{ if .Description }}

{{.Description}}

{{ end }} - {{ if .Language}} {{ if .License }} -

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ {{.License}} 🗒️ {{.Language}}

+

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ {{.License}} {{ if .Language }} 🗒️ {{.Language}} {{end}} 🌿 {{.DefaultBranch}}

{{ else }} -

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ No license 🗒️ {{.Language}}

+

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ No license {{ if .Language }} 🗒️ {{.Language}} {{end}} 🌿 {{.DefaultBranch}}

{{ end }} - {{ else }} - {{ end }}
{{end}} {{ if .files}} @@ -44,7 +41,11 @@ {{ end }} {{ if .readme}}
-

README.md

+ {{ if .repo }} + {{ range $key, $value := .repo}} +

{{.Readme}}

+ {{end}} + {{end}}
{{ unescape .readme}}
@@ -54,4 +55,4 @@

Repository not found

That repository doesn't exist.

{{ end }} - \ No newline at end of file + From 270e2d92f90646237aa4f9e2292ec779f8dcc9b9 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 14 Feb 2023 20:51:44 +0530 Subject: [PATCH 2/5] add languages used in repo --- pages/repo.go | 12 +++++++----- views/repo.html | 11 +++++++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pages/repo.go b/pages/repo.go index 1becc83..46ee460 100644 --- a/pages/repo.go +++ b/pages/repo.go @@ -19,7 +19,7 @@ type Repo struct { Stars string Forks string Watchers string - Language string + Language []string License string DefaultBranch string Readme string @@ -63,9 +63,7 @@ func HandleRepo(c *fiber.Ctx) error { } // Scraping - Scrape := Repo{ - Language: repo.Get("language").String(), - } + Scrape := Repo{} UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") if !ok { @@ -84,6 +82,11 @@ func HandleRepo(c *fiber.Ctx) error { sc.OnHTML("div#readme", func(e *colly.HTMLElement) { Scrape.Readme = e.ChildText("a[href*='#readme']") }) + sc.OnHTML("div.BorderGrid-cell ul.list-style-none", func(e *colly.HTMLElement) { + e.ForEach("li.d-inline .d-inline-flex", func(i int, el *colly.HTMLElement) { + Scrape.Language = append(Scrape.Language, el.ChildText("span.text-bold")+" "+el.ChildText("span:contains('%')")) + }) + }) sc.OnHTML("div#repository-container-header", func(e *colly.HTMLElement) { Scrape.Parent = e.ChildText("span.text-small a") }) @@ -92,7 +95,6 @@ func HandleRepo(c *fiber.Ctx) error { }) sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/") - // Add scrape-based info to repoArray repoArray = append(repoArray, Scrape) diff --git a/views/repo.html b/views/repo.html index 8e145d9..66188e0 100644 --- a/views/repo.html +++ b/views/repo.html @@ -17,10 +17,17 @@

{{.Description}}

{{ end }} {{ if .License }} -

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ {{.License}} {{ if .Language }} 🗒️ {{.Language}} {{end}} 🌿 {{.DefaultBranch}}

+

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ {{.License}} 🌿 {{.DefaultBranch}}

{{ else }} -

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ No license {{ if .Language }} 🗒️ {{.Language}} {{end}} 🌿 {{.DefaultBranch}}

+

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ No license 🌿 {{.DefaultBranch}}

{{ end }} + {{ if .Language }} +

🗒️ + {{range .Language}} + {{.}} + {{end}} +

+ {{end}}
{{end}} {{ if .files}} From bbf3964b61607c582477e83642f925ebb2ab5ae2 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 15 Feb 2023 17:29:19 +0530 Subject: [PATCH 3/5] fix html errors for index page --- views/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/views/index.html b/views/index.html index 6ceb8da..ad7dbe8 100644 --- a/views/index.html +++ b/views/index.html @@ -11,7 +11,7 @@
  • Save bandwidth - no JavaScript/tracking bloat loaded!

    • Privacy

    -

    GitHub is owned by Microsoft, a big tech company known to track it's users. If you care, you can read GitHub's privacy policy here or if you don't care enough to read all that and want it condensed. GotHub solves this by proxying all GitHub requests for you!

    +

    GitHub is owned by Microsoft, a big tech company known to track it's users. If you care, you can read GitHub's privacy policy here or if you don't care enough to read all that and want it condensed. GotHub solves this by proxying all GitHub requests for you!

    Usage

    Just replace github.com with {{.host}}!

    DMCA/Copyright Notice

    @@ -28,4 +28,4 @@ GotHub version: Date: Wed, 15 Feb 2023 18:02:48 +0530 Subject: [PATCH 5/5] implement file retrive with scraping --- pages/repo.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pages/repo.go b/pages/repo.go index 3c19492..84705ed 100644 --- a/pages/repo.go +++ b/pages/repo.go @@ -48,7 +48,6 @@ func HandleRepo(c *fiber.Ctx) error { "error": "Repository " + c.Params("user") + "/" + c.Params("repo") + " not found", }) } - // API // Scraping Scrape := Repo{} @@ -81,24 +80,28 @@ func HandleRepo(c *fiber.Ctx) error { sc.OnHTML("summary[title*='Switch branches or tags']", func(e *colly.HTMLElement) { Scrape.DefaultBranch = e.ChildText("span.css-truncate-target") }) + sc.OnHTML("div.js-details-container div.Details-content--hidden-not-important", func(e *colly.HTMLElement) { + e.ForEach("div.js-navigation-item", func(i int, el *colly.HTMLElement) { + var FileType string + if el.ChildAttr("div.flex-shrink-0 svg", "aria-label") == "Directory" { + FileType = "dir" + } else { + FileType = "file" + } + repoFilesArray = append(repoFilesArray, RepoFiles{ + Name: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"), + Path: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"), + Type: FileType, + Fullname: Scrape.Fullname, + DefaultBranch: Scrape.DefaultBranch, + }) + }) + }) sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/") // Add scrape-based info to repoArray repoArray = append(repoArray, Scrape) - // API - repoFiles := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo") + "/contents") - bruh := repoFiles.Get("#.@pretty").Array() - for _, item := range bruh { - repoFilesArray = append(repoFilesArray, RepoFiles{ - Name: item.Get("path").String(), - Path: item.Get("path").String(), - Type: item.Get("type").String(), - Fullname: Scrape.Fullname, - DefaultBranch: Scrape.DefaultBranch, - }) - } - // README var readmee string err := requests.