GotHub/gothub
1
0
Fork 0
mirror of https://codeberg.org/gothub/gothub synced 2024-12-06 19:16:24 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

use user struct instead of declaring variables for scrape

Browse Source
This commit is contained in:
Arya Kiran
2023-02-07 20:45:13 +05:30
parent 7d7865b86a
commit 4db2d1188e
1 changed files with 18 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pages/user.go
+18 -27
View File
@@ -100,15 +100,8 @@ func HandleUser(c *fiber.Ctx) error {
}
}
// scraping
var Scrape User
var OrgOrUser string
var BioScrape string
var LocScrape string
var WorkplaceScrape string
var AvatarUrlScrape string
var FollowingScrape string
var FollowerScrape string
var UsernameScrape string
var FullNameScrape string
sc1 := colly.NewCollector(colly.AllowedDomains("github.com"))
sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) {
OrgOrUser = e.Attr("itemtype")
@@ -120,49 +113,47 @@ func HandleUser(c *fiber.Ctx) error {
if OrgOrUser == "http://schema.org/Person" {
// Bio
sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) {
BioScrape = e.Attr("data-bio-text")
Scrape.Bio = e.Attr("data-bio-text")
})
// Avatar
sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) {
AvatarUrlScrape = e.Attr("src")
Scrape.AvatarUrl = e.Attr("src")
})
// Metadata (Location/Workplace/Website/Twitter etc.)
sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) {
LocScrape = e.ChildText("li[itemprop*='homeLocation'] span")
WorkplaceScrape = e.ChildText("li[itemprop*='worksFor'] span")
Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span")
Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span")
})
// Followers/Following
sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) {
FollowerScrape = e.ChildText("span")
Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64)
})
sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) {
FollowingScrape = e.ChildText("span")
Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64)
})
// User/Full Name
sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) {
UsernameScrape = e.ChildText("span[itemprop*='additionalName']")
FullNameScrape = e.ChildText("span[itemprop*='name']")
Scrape.Login = e.ChildText("span[itemprop*='additionalName']")
Scrape.Name = e.ChildText("span[itemprop*='name']")
})
} else {
sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) {
AvatarUrlScrape = e.Attr("src")
Scrape.AvatarUrl = e.Attr("src")
})
log.Println("Bio and Location routes cannot be scraped for organizations")
}
sc.Visit("https://github.com/" + c.Params("user") + "/")
FollowingScrapeInt, err := strconv.ParseInt(FollowingScrape, 10, 64)
FollowerScrapeInt, err := strconv.ParseInt(FollowerScrape, 10, 64)
userArray = append(userArray, User{
Login: UsernameScrape,
Name: FullNameScrape,
Bio: BioScrape,
AvatarUrl: AvatarUrlScrape,
Location: LocScrape,
Following: FollowingScrapeInt,
Followers: FollowerScrapeInt,
Login: Scrape.Login,
Name: Scrape.Name,
Bio: Scrape.Bio,
AvatarUrl: Scrape.AvatarUrl,
Location: Scrape.Location,
Following: Scrape.Following,
Followers: Scrape.Followers,
Link: link,
Company: WorkplaceScrape,
Company: Scrape.Company,
Type: user.Get("type").String(),
EwTwitter: user.Get("twitter_username").String(),
Readme: string(readmeOutput),