GotHub/gothub
1
0
Fork 0
mirror of https://codeberg.org/gothub/gothub synced 2024-12-06 19:16:24 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request 'Add scraping for repos' (#67) from aryak/gothub:scraping-repo into dev

Browse Source 
Reviewed-on: https://codeberg.org/gothub/gothub/pulls/67
This commit is contained in:
Odyssium
2023-02-25 12:38:01 +00:00
parent 149d3aaaa6 86f5f72f7c
commit 1b20eb79ef
4 changed files with 94 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pages/repo.go
+71 -40
View File
@@ -1,27 +1,28 @@
package pages
import (
"context"
"log"
"codeberg.org/gothub/gothub/utils"
"context"
"github.com/carlmjohnson/requests"
"github.com/gocolly/colly"
"github.com/gofiber/fiber/v2"
"github.com/gomarkdown/markdown"
"log"
"net/http"
"os"
)
type Repo struct {
Fullname string
Description string
HtmlUrl string
Fork bool
Parent string
Stars int64
Forks int64
Watchers int64
Language string
Stars string
Forks string
Watchers string
Language []string
License string
DefaultBranch string
Readme string
}
type RepoFiles struct {
@@ -35,56 +36,86 @@ type RepoFiles struct {
func HandleRepo(c *fiber.Ctx) error {
var repoArray []Repo
var repoFilesArray []RepoFiles
// get repo
repo := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo"))
if repo.Get("message").String() == "Not Found" {
resp, statusErr := http.Get("https://github.com/" + c.Params("user") + "/" + c.Params("repo"))
if statusErr != nil {
log.Println(statusErr)
}
if resp.StatusCode == 404 {
// I need a better way to do this
return c.Status(404).Render("error", fiber.Map{
"title": "Error",
"error": "Repository " + c.Params("user") + "/" + c.Params("repo") + " not found",
})
}
repoFiles := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo") + "/contents")
bruh := repoFiles.Get("#.@pretty").Array()
for _, item := range bruh {
repoFilesArray = append(repoFilesArray, RepoFiles{
Name: item.Get("path").String(),
Path: item.Get("path").String(),
Type: item.Get("type").String(),
Fullname: repo.Get("full_name").String(),
DefaultBranch: repo.Get("default_branch").String(),
})
// Scraping
Scrape := Repo{}
UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT")
if !ok {
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
var readmee string
sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent))
sc.OnHTML("div.Layout-sidebar", func(e *colly.HTMLElement) {
Scrape.Fullname = c.Params("user") + "/" + c.Params("repo")
Scrape.Description = e.ChildText("p.f4")
Scrape.Stars = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/stargazers' i] strong")
Scrape.Watchers = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/watchers' i] strong")
Scrape.Forks = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/network/members' i] strong")
Scrape.License = e.ChildText("a[data-analytics-event*='{\"category\":\"Repository Overview\",\"action\":\"click\",\"label\":\"location:sidebar;file:license\"}']")
})
sc.OnHTML("div#readme", func(e *colly.HTMLElement) {
Scrape.Readme = e.ChildText("a[href*='#readme']")
})
sc.OnHTML("div.BorderGrid-cell ul.list-style-none", func(e *colly.HTMLElement) {
e.ForEach("li.d-inline .d-inline-flex", func(i int, el *colly.HTMLElement) {
Scrape.Language = append(Scrape.Language, el.ChildText("span.text-bold")+" "+el.ChildText("span:contains('%')"))
})
})
sc.OnHTML("div#repository-container-header", func(e *colly.HTMLElement) {
Scrape.Parent = e.ChildText("span.text-small a")
})
sc.OnHTML("summary[title*='Switch branches or tags']", func(e *colly.HTMLElement) {
Scrape.DefaultBranch = e.ChildText("span.css-truncate-target")
})
sc.OnHTML("div.js-details-container div.Details-content--hidden-not-important", func(e *colly.HTMLElement) {
e.ForEach("div.js-navigation-item", func(i int, el *colly.HTMLElement) {
var FileType string
if el.ChildAttr("div.flex-shrink-0 svg", "aria-label") == "Directory" {
FileType = "dir"
} else {
FileType = "file"
}
repoFilesArray = append(repoFilesArray, RepoFiles{
Name: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"),
Path: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"),
Type: FileType,
Fullname: Scrape.Fullname,
DefaultBranch: Scrape.DefaultBranch,
})
})
})
sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/")
// Add scrape-based info to repoArray
repoArray = append(repoArray, Scrape)
// README
var readmee string
err := requests.
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + repo.Get("default_branch").String() + "/README.md").
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + Scrape.DefaultBranch + "/" + Scrape.Readme).
ToString(&readmee).
Fetch(context.Background())
if err != nil {
readmee = ""
log.Println(err)
}
mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil)
// Trust Nobody
readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe)
repoArray = append(repoArray, Repo{
Fullname: repo.Get("full_name").String(),
Description: repo.Get("description").String(),
HtmlUrl: repo.Get("html_url").String(),
Fork: repo.Get("fork").Bool(),
Stars: repo.Get("stargazers_count").Int(),
Forks: repo.Get("forks_count").Int(),
Watchers: repo.Get("watchers_count").Int(),
Language: repo.Get("language").String(),
License: repo.Get("license").Get("name").String(),
Parent: repo.Get("parent").Get("full_name").String(),
DefaultBranch: repo.Get("default_branch").String(),
})
return c.Render("repo", fiber.Map{
"title": "Repository " + c.Params("user") + "/" + c.Params("repo"),
"repo": repoArray,
pages/user.go
+5 -6
View File
@@ -5,7 +5,6 @@ import (
"log"
"net/http"
"os"
"strconv"
"strings"
"codeberg.org/gothub/gothub/utils"
@@ -26,8 +25,8 @@ type User struct {
Location string
Email string
Timezone string
Following int64
Followers int64
Following string
Followers string
Link string
Social []string
Organizations []string
@@ -108,8 +107,8 @@ func HandleUser(c *fiber.Ctx) error {
Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary"))
})
// Followers/Following
Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i] span"), 10, 64)
Scrape.Following, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i] span"), 10, 64)
Scrape.Followers = e.ChildText("a[href*='https://github.com/" + c.Params("user") + "?tab=followers' i] span")
Scrape.Following = e.ChildText("a[href*='https://github.com/" + c.Params("user") + "?tab=following' i] span")
// Organizations
e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) {
Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label"))
@@ -135,7 +134,7 @@ func HandleUser(c *fiber.Ctx) error {
Scrape.Social = append(Scrape.Social, el.Attr("href"))
})
// Followers
Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64)
Scrape.Followers = e.ChildText("a[href*='/orgs/" + c.Params("user") + "/followers' i] span")
})
sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) {
Scrape.AvatarUrl = e.Attr("src")
views/index.html
+2 -2
View File
@@ -11,7 +11,7 @@
<li>Save bandwidth - no JavaScript/tracking bloat loaded!</li>
</ul>
<h3>Privacy</h3>
<p>GitHub is owned by Microsoft, a big tech company known to track it's users. <a href="https://docs.github.com/en/site-policy/privacy-policies/github-privacy-statement" target="_blank">If you care, you can read GitHub's privacy policy here</a> or <a href="https://tosdr.org/en/service/297" target="_blank">if you don't care enough to read all that and want it condensed</a>. GotHub solves this by proxying all GitHub requests for you!</a></p>
<p>GitHub is owned by Microsoft, a big tech company known to track it's users. <a href="https://docs.github.com/en/site-policy/privacy-policies/github-privacy-statement" target="_blank">If you care, you can read GitHub's privacy policy here</a> or <a href="https://tosdr.org/en/service/297" target="_blank">if you don't care enough to read all that and want it condensed</a>. GotHub solves this by proxying all GitHub requests for you!</p>
<h3>Usage</h3>
<p>Just replace <code class="indexGitHub">github.com</code> with <code class="indexGitHub">{{.host}}</code>!</p>
<h3>DMCA/Copyright Notice</h3>
@@ -28,4 +28,4 @@ GotHub version: <a href="https://codeberg.org/gothub/gothub/commit/{{ .version}}
{{ end }}
<br>
Fiber version: {{ .fiberversion}}, running on {{ .goversion}}
</main>
</main>
views/repo.html
+16 -8
View File
@@ -10,20 +10,24 @@
<div class="userProfile">
<h1>{{.Fullname}}</h1>
{{ if .Fork }}
{{ if .Parent }}
<p>This repository is a fork of <a href="/{{.Parent}}">{{.Parent}}</a>.</p>
{{ end }}
{{ if .Description }}
<p>{{.Description}}</p>
{{ end }}
{{ if .Language}}
{{ if .License }}
<p>⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ {{.License}} 🗒️ {{.Language}}</p>
<p>⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ {{.License}} 🌿 {{.DefaultBranch}}</p>
{{ else }}
<p>⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ No license 🗒️ {{.Language}}</p>
<p>⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ No license 🌿 {{.DefaultBranch}}</p>
{{ end }}
{{ else }}
{{ end }}
{{ if .Language }}
<p>🗒️
{{range .Language}}
{{.}}
{{end}}
</p>
{{end}}
</div>
{{end}}
{{ if .files}}
@@ -44,7 +48,11 @@
{{ end }}
{{ if .readme}}
<div class="userReadme">
<h3>README.md</h3>
{{ if .repo }}
{{ range $key, $value := .repo}}
<h3>{{.Readme}}</h3>
{{end}}
{{end}}
<div class="userReadmeText">
{{ unescape .readme}}
</div>
@@ -54,4 +62,4 @@
<h2>Repository not found</h2>
<p>That repository doesn't exist.</p>
{{ end }}
</main>
</main>