package pages import ( "codeberg.org/gothub/gothub/utils" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "log" "net/http" "os" "regexp" "strings" ) type Wiki struct { RepoUser string RepoName string Title string LastEdit string LastEditor string RevisionNum string PageNum string PageContent string Sidebar []Sidebar } type Sidebar struct { Name string Link string } func WikiView(c *fiber.Ctx) error { var wikiArray []Wiki resp, statusErr := http.Get("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page")) if statusErr != nil { log.Println(statusErr) } if resp.StatusCode == 404 { // I need a better way to do this return c.Status(404).Render("error", fiber.Map{ "title": "Error", "error": "Wiki" + c.Params("page") + "not found", }) } // Scraping Scrape := Wiki{} UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") if !ok { UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" } sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) sc.OnHTML("div#repository-container-header", func(e *colly.HTMLElement) { Scrape.RepoUser = e.ChildText("span.author a") Scrape.RepoName = e.ChildText("strong a") }) sc.OnHTML("div#wiki-wrapper", func(e *colly.HTMLElement) { Scrape.Title = e.ChildText("div.d-flex h1") Scrape.LastEdit = e.ChildText("div.gh-header-meta relative-time") re := regexp.MustCompile(`\ edited.*`) Scrape.LastEditor = re.ReplaceAllString(strings.Replace(e.ChildText("div.gh-header-meta"), "\n", "", -1), "${1}") Scrape.RevisionNum = strings.TrimSuffix(e.ChildText("a.Link--muted"), " revisions") }) sc.OnHTML("div.wiki-pages-box div.Box--condensed", func(e *colly.HTMLElement) { Scrape.PageNum = e.ChildText("span.Counter--primary") e.ForEach("ul.list-style-none li.Box-row", func(i int, el *colly.HTMLElement) { Scrape.Sidebar = append(Scrape.Sidebar, Sidebar{ Name: el.ChildText("summary span a.Truncate-text"), Link: el.ChildAttr("summary span a.Truncate-text", "href"), }) }) }) sc.OnHTML("div#wiki-body > div.markdown-body", func(e *colly.HTMLElement) { Content, _ := e.DOM.Html() Scrape.PageContent = strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(string(utils.UGCPolicy().SanitizeBytes([]byte(Content))), "https://github.com", "", -1), "user-content-", "", -1), "https://camo.githubusercontent.com", "/camo", -1), "https://raw.githubusercontent.com", "/raw", -1), "https://user-images.githubusercontent.com", "/user-images", -1) }) sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page")) // Add scrape-based info to wikiArray wikiArray = append(wikiArray, Scrape) return c.Render("wiki", fiber.Map{ "title": "Wiki page " + c.Params("page") + " | " + c.Params("user") + "/" + c.Params("repo"), "wiki": wikiArray, }) }