From e215d628e4138b589bb1793ec0090c63c2f47fea Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 20:08:36 +0530 Subject: [PATCH] simplify scraping and remove un-needed OnHTML calls --- pages/user.go | 72 +++++++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/pages/user.go b/pages/user.go index e7fa079..4049649 100644 --- a/pages/user.go +++ b/pages/user.go @@ -133,71 +133,53 @@ func HandleUser(c *fiber.Ctx) error { sc1.Visit("https://github.com/" + c.Params("user") + "/") sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) if Scrape.Type == "http://schema.org/Person" { - // Bio - sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { - Scrape.Bio = e.Attr("data-bio-text") - }) - // Avatar - sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - }) - // Metadata (Location/Workplace/Website/Social etc.) - sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { + sc.OnHTML("div.js-profile-editable-replace", func(e *colly.HTMLElement) { + // Main info + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") + Scrape.Bio = e.ChildText("div[data-bio-text] div") + Scrape.AvatarUrl = e.ChildAttr("img[alt*='Avatar']", "src") + // Metadata Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.Link = e.ChildText("a[itemprop*='url']") e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary")) }) - Scrape.Link = e.ChildText("a[itemprop*='url']") - }) - // Followers/Following - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { - Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { - Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - // User/Full Name - sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { - Scrape.Login = e.ChildText("span[itemprop*='additionalName']") - Scrape.Name = e.ChildText("span[itemprop*='name']") - }) - // User Status - sc.OnHTML("div.user-status-circle-badge", func(e *colly.HTMLElement) { + // Followers/Following + Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i] span"), 10, 64) + Scrape.Following, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i] span"), 10, 64) + // Organizations + e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { + Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) + }) + // User Status Scrape.Status = e.ChildText("div.user-status-message-wrapper div") - }) - // User Status Emoji - sc.OnHTML("div.user-status-circle-badge div.user-status-emoji-container g-emoji", func(e *colly.HTMLElement) { - Scrape.StatusEmoji = e.Attr("alias") + Scrape.StatusEmoji = e.ChildAttr("div.user-status-circle-badge div.user-status-emoji-container g-emoji" ,"alias") }) // Contributions sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { Scrape.Contributions = e.ChildText("h2") }) - // Organizations - sc.OnHTML("div.mt-3", func(e *colly.HTMLElement) { - e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { - Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) - }) - }) - } else { - sc.OnHTML("div.container-xl div.flex-md-items-center div.flex-1", func(e *colly.HTMLElement) { - Scrape.Bio = e.ChildText("div.color-fg-muted div") - Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64) + sc.OnHTML("div.container-xl div.flex-md-items-center", func(e *colly.HTMLElement) { + // Main info + Scrape.Login = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "alt") // Hacky Scrape.Name = e.ChildText("h1.h2") + Scrape.Bio = e.ChildText("div.color-fg-muted div") + Scrape.AvatarUrl = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "src") + // Metadata Scrape.Location = e.ChildText("span[itemprop*='location']") - Scrape.Link = e.ChildText("a[itemprop*='url']") Scrape.Email = e.ChildText("a[itemprop*='email']") + Scrape.Link = e.ChildText("a[itemprop*='url']") e.ForEach("a.Link--primary", func(i int, el *colly.HTMLElement) { Scrape.Social = append(Scrape.Social, el.Attr("href")) }) + // Followers + Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64) }) - sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - Scrape.Login = e.Attr("alt") - }) + // Org Members sc.OnHTML("div.clearfix", func(e *colly.HTMLElement) { e.ForEach("a[data-hovercard-type*='user'] img", func(i int, el *colly.HTMLElement) { Scrape.OrgMembers = append(Scrape.OrgMembers, strings.TrimPrefix(el.Attr("alt"), "@"))