GotHub/gothub
1
0
Fork 0
mirror of https://codeberg.org/gothub/gothub synced 2024-12-06 19:16:24 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

simplify scraping and remove un-needed OnHTML calls

Browse Source
This commit is contained in:
Arya Kiran
2023-02-13 20:08:36 +05:30
parent 05793d776a
commit e215d628e4
1 changed files with 27 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pages/user.go
+27 -45
View File
@@ -133,71 +133,53 @@ func HandleUser(c *fiber.Ctx) error {
sc1.Visit("https://github.com/" + c.Params("user") + "/")
sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent))
if Scrape.Type == "http://schema.org/Person" {
// Bio
sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) {
Scrape.Bio = e.Attr("data-bio-text")
})
// Avatar
sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) {
Scrape.AvatarUrl = e.Attr("src")
})
// Metadata (Location/Workplace/Website/Social etc.)
sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) {
sc.OnHTML("div.js-profile-editable-replace", func(e *colly.HTMLElement) {
// Main info
Scrape.Login = e.ChildText("span[itemprop*='additionalName']")
Scrape.Name = e.ChildText("span[itemprop*='name']")
Scrape.Bio = e.ChildText("div[data-bio-text] div")
Scrape.AvatarUrl = e.ChildAttr("img[alt*='Avatar']", "src")
// Metadata
Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span")
Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span")
Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span")
Scrape.Link = e.ChildText("a[itemprop*='url']")
e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) {
Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary"))
})
Scrape.Link = e.ChildText("a[itemprop*='url']")
})
// Followers/Following
sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) {
Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64)
})
sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) {
Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64)
})
// User/Full Name
sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) {
Scrape.Login = e.ChildText("span[itemprop*='additionalName']")
Scrape.Name = e.ChildText("span[itemprop*='name']")
})
// User Status
sc.OnHTML("div.user-status-circle-badge", func(e *colly.HTMLElement) {
// Followers/Following
Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i] span"), 10, 64)
Scrape.Following, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i] span"), 10, 64)
// Organizations
e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) {
Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label"))
})
// User Status
Scrape.Status = e.ChildText("div.user-status-message-wrapper div")
})
// User Status Emoji
sc.OnHTML("div.user-status-circle-badge div.user-status-emoji-container g-emoji", func(e *colly.HTMLElement) {
Scrape.StatusEmoji = e.Attr("alias")
Scrape.StatusEmoji = e.ChildAttr("div.user-status-circle-badge div.user-status-emoji-container g-emoji" ,"alias")
})
// Contributions
sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) {
Scrape.Contributions = e.ChildText("h2")
})
// Organizations
sc.OnHTML("div.mt-3", func(e *colly.HTMLElement) {
e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) {
Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label"))
})
})
} else {
sc.OnHTML("div.container-xl div.flex-md-items-center div.flex-1", func(e *colly.HTMLElement) {
Scrape.Bio = e.ChildText("div.color-fg-muted div")
Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64)
sc.OnHTML("div.container-xl div.flex-md-items-center", func(e *colly.HTMLElement) {
// Main info
Scrape.Login = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "alt") // Hacky
Scrape.Name = e.ChildText("h1.h2")
Scrape.Bio = e.ChildText("div.color-fg-muted div")
Scrape.AvatarUrl = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "src")
// Metadata
Scrape.Location = e.ChildText("span[itemprop*='location']")
Scrape.Link = e.ChildText("a[itemprop*='url']")
Scrape.Email = e.ChildText("a[itemprop*='email']")
Scrape.Link = e.ChildText("a[itemprop*='url']")
e.ForEach("a.Link--primary", func(i int, el *colly.HTMLElement) {
Scrape.Social = append(Scrape.Social, el.Attr("href"))
})
// Followers
Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64)
})
sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) {
Scrape.AvatarUrl = e.Attr("src")
Scrape.Login = e.Attr("alt")
})
// Org Members
sc.OnHTML("div.clearfix", func(e *colly.HTMLElement) {
e.ForEach("a[data-hovercard-type*='user'] img", func(i int, el *colly.HTMLElement) {
Scrape.OrgMembers = append(Scrape.OrgMembers, strings.TrimPrefix(el.Attr("alt"), "@"))