mirror of
https://codeberg.org/gothub/gothub
synced 2024-12-06 19:16:24 +01:00
Arya Kiran
362 lines
10 KiB
Go
362 lines
10 KiB
Go
package pages
|
|
|
|
import (
|
|
"codeberg.org/gothub/gothub/utils"
|
|
"context"
|
|
"github.com/carlmjohnson/requests"
|
|
"github.com/gocolly/colly"
|
|
"github.com/gofiber/fiber/v2"
|
|
"github.com/gomarkdown/markdown"
|
|
"github.com/microcosm-cc/bluemonday"
|
|
"log"
|
|
"net/http"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
type User struct {
|
|
Login string
|
|
Name string
|
|
Bio string
|
|
AvatarUrl string
|
|
Location string
|
|
Following int64
|
|
Followers int64
|
|
Link string
|
|
Company string
|
|
Type string
|
|
EwTwitter string
|
|
Readme string
|
|
}
|
|
|
|
type Ratelimit struct {
|
|
Remaining int64
|
|
Limit int64
|
|
}
|
|
|
|
// HandleUser handles the user page.
|
|
func HandleUser(c *fiber.Ctx) error {
|
|
// Declare Array used for displaying data
|
|
var userArray []User
|
|
resp, err := http.Get("https://github.com/" + c.Params("user"))
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
if resp.StatusCode == 404 {
|
|
return c.Status(404).Render("error", fiber.Map{
|
|
"error": "User " + c.Params("user") + " not found",
|
|
})
|
|
}
|
|
// API
|
|
user := utils.GetRequest("https://api.github.com/users/" + c.Params("user"))
|
|
var link string
|
|
if user.Get("message").String() == "Not Found" {
|
|
return c.Status(404).Render("error", fiber.Map{
|
|
"error": "User " + c.Params("user") + " not found",
|
|
})
|
|
}
|
|
if strings.Contains(user.Get("message").String(), "rate limit") { // dont wanna get the status code so i'll just do this instead 👍
|
|
ratelimitJSON := utils.GetRequest("https://api.github.com/rate_limit")
|
|
log.Println(ratelimitJSON)
|
|
var ratelimitArray []Ratelimit
|
|
|
|
ratelimitArray = append(ratelimitArray, Ratelimit{
|
|
Remaining: ratelimitJSON.Get("resources.core.remaining").Int(),
|
|
Limit: ratelimitJSON.Get("resources.core.limit").Int(),
|
|
})
|
|
|
|
log.Println(ratelimitArray)
|
|
|
|
return c.Render("ratelimit", fiber.Map{
|
|
"Title": "GitHub API /users endpoint rate limit exceeded",
|
|
"ratelimit": ratelimitArray,
|
|
})
|
|
} else {
|
|
|
|
if user.Get("blog").String() == "" {
|
|
link = ""
|
|
} else {
|
|
link = user.Get("blog").String()
|
|
if strings.HasPrefix(link, "https://") {
|
|
link = strings.TrimPrefix(link, "https://")
|
|
} else if strings.HasPrefix(link, "http://") {
|
|
link = strings.TrimPrefix(link, "http://")
|
|
} else {
|
|
log.Println("Has no prefix")
|
|
}
|
|
}
|
|
}
|
|
// User README
|
|
var readmee string
|
|
|
|
err0 := requests.
|
|
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md").
|
|
ToString(&readmee).
|
|
Fetch(context.Background())
|
|
if err0 != nil {
|
|
err2 := requests.
|
|
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md").
|
|
ToString(&readmee).
|
|
Fetch(context.Background())
|
|
if err2 != nil {
|
|
readmee = ""
|
|
log.Println(err0)
|
|
}
|
|
}
|
|
mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil)
|
|
// Sanitize the user README incase there is any weird shit in it
|
|
readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe)
|
|
|
|
// scraping
|
|
Scrape := User{
|
|
Link: link,
|
|
Readme: string(readmeOutput),
|
|
}
|
|
|
|
sc1 := colly.NewCollector(colly.AllowedDomains("github.com"))
|
|
|
|
sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) {
|
|
Scrape.Type = e.Attr("itemtype")
|
|
})
|
|
|
|
sc1.Visit("https://github.com/" + c.Params("user") + "/")
|
|
sc := colly.NewCollector(
|
|
colly.AllowedDomains("github.com"),
|
|
)
|
|
|
|
if Scrape.Type == "http://schema.org/Person" {
|
|
// Bio
|
|
sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) {
|
|
Scrape.Bio = e.Attr("data-bio-text")
|
|
})
|
|
// Avatar
|
|
sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) {
|
|
Scrape.AvatarUrl = e.Attr("src")
|
|
})
|
|
// Metadata (Location/Workplace/Website/Twitter etc.)
|
|
sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) {
|
|
Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span")
|
|
Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span")
|
|
Scrape.EwTwitter = e.ChildText("a[href*='https://twitter.com/' i]")
|
|
})
|
|
// Followers/Following
|
|
sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) {
|
|
Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64)
|
|
})
|
|
sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) {
|
|
Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64)
|
|
})
|
|
// User/Full Name
|
|
sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) {
|
|
Scrape.Login = e.ChildText("span[itemprop*='additionalName']")
|
|
Scrape.Name = e.ChildText("span[itemprop*='name']")
|
|
})
|
|
} else {
|
|
sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) {
|
|
Scrape.AvatarUrl = e.Attr("src")
|
|
})
|
|
log.Println("Bio and Location routes cannot be scraped for organizations")
|
|
}
|
|
sc.Visit("https://github.com/" + c.Params("user") + "/")
|
|
Scrape.EwTwitter = strings.Trim(Scrape.EwTwitter, "@")
|
|
userArray = append(userArray, Scrape)
|
|
|
|
return c.Render("user", fiber.Map{
|
|
"Title": "User " + c.Params("user"),
|
|
"user": userArray,
|
|
})
|
|
}
|
|
|
|
// copied from bluemonday's GitHub repostiory, with some adaptations
|
|
func UGCPolicy() *bluemonday.Policy {
|
|
|
|
p := bluemonday.NewPolicy()
|
|
|
|
///////////////////////
|
|
// Global attributes //
|
|
///////////////////////
|
|
|
|
// "class" is not permitted as we are not allowing users to style their own
|
|
// content
|
|
|
|
p.AllowStandardAttributes()
|
|
|
|
//////////////////////////////
|
|
// Global URL format policy //
|
|
//////////////////////////////
|
|
|
|
p.AllowStandardURLs()
|
|
|
|
////////////////////////////////
|
|
// Declarations and structure //
|
|
////////////////////////////////
|
|
|
|
// "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are
|
|
// expecting user generated content to be a fragment of HTML and not a full
|
|
// document.
|
|
|
|
//////////////////////////
|
|
// Sectioning root tags //
|
|
//////////////////////////
|
|
|
|
// "article" and "aside" are permitted and takes no attributes
|
|
p.AllowElements("article", "aside")
|
|
|
|
// "body" is not permitted as we are expecting user generated content to be a fragment
|
|
// of HTML and not a full document.
|
|
|
|
// "details" is permitted, including the "open" attribute which can either
|
|
// be blank or the value "open".
|
|
p.AllowAttrs(
|
|
"open",
|
|
).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details")
|
|
|
|
// "fieldset" is not permitted as we are not allowing forms to be created.
|
|
|
|
// "figure" is permitted and takes no attributes
|
|
p.AllowElements("figure")
|
|
|
|
// "nav" is not permitted as it is assumed that the site (and not the user)
|
|
// has defined navigation elements
|
|
|
|
// "section" is permitted and takes no attributes
|
|
p.AllowElements("section")
|
|
|
|
// "summary" is permitted and takes no attributes
|
|
p.AllowElements("summary")
|
|
|
|
//////////////////////////
|
|
// Headings and footers //
|
|
//////////////////////////
|
|
|
|
// "footer" is not permitted as we expect user content to be a fragment and
|
|
// not structural to this extent
|
|
|
|
// "h1" through "h6" are permitted and take no attributes
|
|
p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6")
|
|
|
|
// "header" is not permitted as we expect user content to be a fragment and
|
|
// not structural to this extent
|
|
|
|
// "hgroup" is permitted and takes no attributes
|
|
p.AllowElements("hgroup")
|
|
|
|
/////////////////////////////////////
|
|
// Content grouping and separating //
|
|
/////////////////////////////////////
|
|
|
|
// "blockquote" is permitted, including the "cite" attribute which must be
|
|
// a standard URL.
|
|
p.AllowAttrs("cite").OnElements("blockquote")
|
|
|
|
// "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes
|
|
p.AllowElements("br", "div", "hr", "p", "span", "wbr")
|
|
|
|
///////////
|
|
// Links //
|
|
///////////
|
|
|
|
// "a" is permitted
|
|
p.AllowAttrs("href").OnElements("a")
|
|
|
|
// "area" is permitted along with the attributes that map image maps work
|
|
p.AllowAttrs("name").Matching(
|
|
regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`),
|
|
).OnElements("map")
|
|
p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area")
|
|
p.AllowAttrs("coords").Matching(
|
|
regexp.MustCompile(`^([0-9]+,)+[0-9]+$`),
|
|
).OnElements("area")
|
|
p.AllowAttrs("href").OnElements("area")
|
|
p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area")
|
|
p.AllowAttrs("shape").Matching(
|
|
regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`),
|
|
).OnElements("area")
|
|
p.AllowAttrs("usemap").Matching(
|
|
regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`),
|
|
).OnElements("img")
|
|
|
|
// "link" is not permitted
|
|
|
|
/////////////////////
|
|
// Phrase elements //
|
|
/////////////////////
|
|
|
|
// The following are all inline phrasing elements
|
|
p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em",
|
|
"figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var")
|
|
|
|
// "q" is permitted and "cite" is a URL and handled by URL policies
|
|
p.AllowAttrs("cite").OnElements("q")
|
|
|
|
// "time" is permitted
|
|
p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time")
|
|
|
|
////////////////////
|
|
// Style elements //
|
|
////////////////////
|
|
|
|
// block and inline elements that impart no semantic meaning but style the
|
|
// document
|
|
p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u")
|
|
|
|
// "style" is not permitted as we are not yet sanitising CSS and it is an
|
|
// XSS attack vector
|
|
|
|
//////////////////////
|
|
// HTML5 Formatting //
|
|
//////////////////////
|
|
|
|
// "bdi" "bdo" are permitted
|
|
p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo")
|
|
|
|
// "rp" "rt" "ruby" are permitted
|
|
p.AllowElements("rp", "rt", "ruby")
|
|
|
|
///////////////////////////
|
|
// HTML5 Change tracking //
|
|
///////////////////////////
|
|
|
|
// "del" "ins" are permitted
|
|
p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins")
|
|
p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins")
|
|
|
|
///////////
|
|
// Lists //
|
|
///////////
|
|
|
|
p.AllowLists()
|
|
|
|
////////////
|
|
// Tables //
|
|
////////////
|
|
|
|
p.AllowTables()
|
|
|
|
///////////
|
|
// Forms //
|
|
///////////
|
|
|
|
// By and large, forms are not permitted. However there are some form
|
|
// elements that can be used to present data, and we do permit those
|
|
//
|
|
// "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist"
|
|
// "textarea" "optgroup" "option" are all not permitted
|
|
|
|
// "meter" is permitted
|
|
p.AllowAttrs(
|
|
"value",
|
|
"min",
|
|
"max",
|
|
"low",
|
|
"high",
|
|
"optimum",
|
|
).Matching(bluemonday.Number).OnElements("meter")
|
|
|
|
// "progress" is permitted
|
|
p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress")
|
|
|
|
return p
|
|
}
|