mirror of
https://codeberg.org/gothub/gothub
synced 2024-12-06 19:16:24 +01:00
Arya Kiran
88 lines
2.9 KiB
Go
88 lines
2.9 KiB
Go
package pages
|
|
|
|
import (
|
|
"codeberg.org/gothub/gothub/utils"
|
|
"github.com/gocolly/colly"
|
|
"github.com/gofiber/fiber/v2"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
type Wiki struct {
|
|
RepoUser string
|
|
RepoName string
|
|
Title string
|
|
LastEdit string
|
|
LastEditor string
|
|
RevisionNum string
|
|
PageNum string
|
|
PageContent string
|
|
Sidebar []Sidebar
|
|
}
|
|
type Sidebar struct {
|
|
Name string
|
|
Link string
|
|
}
|
|
|
|
func WikiView(c *fiber.Ctx) error {
|
|
var wikiArray []Wiki
|
|
|
|
resp, statusErr := http.Get("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page"))
|
|
if statusErr != nil {
|
|
log.Println(statusErr)
|
|
}
|
|
if resp.StatusCode == 404 {
|
|
// I need a better way to do this
|
|
return c.Status(404).Render("error", fiber.Map{
|
|
"title": "Error",
|
|
"error": "Wiki" + c.Params("page") + "not found",
|
|
})
|
|
}
|
|
|
|
// Scraping
|
|
Scrape := Wiki{}
|
|
|
|
UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT")
|
|
if !ok {
|
|
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent))
|
|
sc.OnHTML("div#repository-container-header", func(e *colly.HTMLElement) {
|
|
Scrape.RepoUser = e.ChildText("span.author a")
|
|
Scrape.RepoName = e.ChildText("strong a")
|
|
})
|
|
sc.OnHTML("div#wiki-wrapper", func(e *colly.HTMLElement) {
|
|
Scrape.Title = e.ChildText("div.d-flex h1")
|
|
Scrape.LastEdit = e.ChildText("div.gh-header-meta relative-time")
|
|
re := regexp.MustCompile(`\ edited.*`)
|
|
Scrape.LastEditor = re.ReplaceAllString(strings.Replace(e.ChildText("div.gh-header-meta"), "\n", "", -1), "${1}")
|
|
Scrape.RevisionNum = strings.TrimSuffix(e.ChildText("a.Link--muted"), " revisions")
|
|
})
|
|
sc.OnHTML("div.wiki-pages-box div.Box--condensed", func(e *colly.HTMLElement) {
|
|
Scrape.PageNum = e.ChildText("span.Counter--primary")
|
|
e.ForEach("ul.list-style-none li.Box-row", func(i int, el *colly.HTMLElement) {
|
|
Scrape.Sidebar = append(Scrape.Sidebar, Sidebar{
|
|
Name: el.ChildText("summary span a.Truncate-text"),
|
|
Link: el.ChildAttr("summary span a.Truncate-text", "href"),
|
|
})
|
|
})
|
|
})
|
|
sc.OnHTML("div#wiki-body > div.markdown-body", func(e *colly.HTMLElement) {
|
|
Content, _ := e.DOM.Html()
|
|
Scrape.PageContent = strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(string(utils.UGCPolicy().SanitizeBytes([]byte(Content))), "https://github.com", "", -1), "user-content-", "", -1), "https://camo.githubusercontent.com", "/camo", -1), "https://raw.githubusercontent.com", "/raw", -1), "https://user-images.githubusercontent.com", "/user-images", -1)
|
|
|
|
})
|
|
sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page"))
|
|
// Add scrape-based info to wikiArray
|
|
wikiArray = append(wikiArray, Scrape)
|
|
|
|
return c.Render("wiki", fiber.Map{
|
|
"title": "Wiki page " + c.Params("page") + " | " + c.Params("user") + "/" + c.Params("repo"),
|
|
"wiki": wikiArray,
|
|
})
|
|
}
|