package pages import ( "log" "net/http" "os" "github.com/tidwall/gjson" "codeberg.org/gothub/gothub/utils" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" ) type Dir struct { Readme string Fullname string DirName string Branch string // TruncationWarning string } // TODO: Handle repository truncation warning. // Example text: "Sorry, we had to truncate this directory to 1,000 files. // 718 entries were omitted from the list. Latest commit info may be omitted." type DirFiles struct { Name string Path string Type string Branch string Fullname string } // Given a gjson object, as well as branch name and fullname ('user/repository' format), // return an array of directory files func return_dir_files_array(dirList gjson.Result, branch string, fullname string) []DirFiles { var dirFilesArray []DirFiles dirList.ForEach(func(i, e gjson.Result) bool { var FileType string if e.Get("contentType").Str == "directory" { FileType = "dir" } else { FileType = "file" } dirFilesArray = append(dirFilesArray, DirFiles{ Name: e.Get("name").Str, // Path is the path to the file/folder from the repository root Path: e.Get("path").Str, Type: FileType, Branch: branch, Fullname: fullname, }) return true //keep iterating }) return dirFilesArray } func DirView(c *fiber.Ctx) error { var dirArray []Dir var dirFilesArray []DirFiles var readmeOutput string resp, statusErr := http.Get("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/tree/" + c.Params("branch") + "/" + c.Params("+")) if statusErr != nil { log.Println(statusErr) } if resp.StatusCode == 404 { // I need a better way to do this return c.Status(404).Render("error", fiber.Map{ "title": "Error", "error": "Directory " + c.Params("+") + " not found", }) } // Scraping Scrape := Dir{} Scrape.Fullname = c.Params("user") + "/" + c.Params("repo") Scrape.DirName = c.Params("+") Scrape.Branch = c.Params("branch") UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") if !ok { UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" } sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) sc.OnHTML("react-app script[data-target]", func(e *colly.HTMLElement) { jsonMess := e.Text dirList := gjson.Get(jsonMess, "payload.tree.items") dirFilesArray = return_dir_files_array(dirList, Scrape.Branch, Scrape.Fullname) }) // Get repository truncation warning. This only works with js enabled, so it does not work with colly /* sc.OnHTML("[data-testid='repo-truncation-warning']", func(e *colly.HTMLElement) { fmt.Printf("Truncation warning: %s", e.Text) Scrape.TruncationWarning = e.Text }) */ sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/tree/" + c.Params("branch") + "/" + c.Params("+")) // Add scrape-based info to dirArray dirArray = append(dirArray, Scrape) return c.Render("dir", fiber.Map{ "title": "Directory " + c.Params("+") + " | " + c.Params("user") + "/" + c.Params("repo"), "branch": utils.Branch, "dir": dirArray, "files": dirFilesArray, "readme": string(readmeOutput), }) }