From 16fa0305bdd222f1ccb36a7ee1bc320df073e85b Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 25 Apr 2023 23:20:12 +0530 Subject: [PATCH 1/7] add initial wiki support in backend and a test sample for frontend --- pages/wiki.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ serve/serve.go | 5 +++ views/wiki.html | 17 +++++++++++ 3 files changed, 103 insertions(+) create mode 100644 pages/wiki.go create mode 100644 views/wiki.html diff --git a/pages/wiki.go b/pages/wiki.go new file mode 100644 index 0000000..623794f --- /dev/null +++ b/pages/wiki.go @@ -0,0 +1,81 @@ +package pages + +import ( + "github.com/gocolly/colly" + "github.com/gofiber/fiber/v2" + "log" + "net/http" + "os" + "regexp" + "strings" +) + +type Wiki struct { + RepoUser string + RepoName string + Title string + LastEdit string + LastEditor string + RevisionNum string + PageNum string + PageContent string + Sidebar []Sidebar +} +type Sidebar struct { + Name string + Link string +} + +func WikiView(c *fiber.Ctx) error { + var wikiArray []Wiki + + resp, statusErr := http.Get("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page")) + if statusErr != nil { + log.Println(statusErr) + } + if resp.StatusCode == 404 { + // I need a better way to do this + return c.Status(404).Render("error", fiber.Map{ + "title": "Error", + "error": "Wiki" + c.Params("page") + "not found", + }) + } + + // Scraping + Scrape := Wiki{} + + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + + sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) + sc.OnHTML("div#repository-container-header", func(e *colly.HTMLElement) { + Scrape.RepoUser = e.ChildText("span.author a") + Scrape.RepoName = e.ChildText("strong a") + }) + sc.OnHTML("div#wiki-wrapper", func(e *colly.HTMLElement) { + Scrape.Title = e.ChildText("div.d-flex h1") + Scrape.LastEdit = e.ChildText("div.gh-header-meta relative-time") + re := regexp.MustCompile(`\ edited.*`) + Scrape.LastEditor = re.ReplaceAllString(strings.Replace(e.ChildText("div.gh-header-meta"), "\n", "", -1), "${1}") + Scrape.RevisionNum = strings.TrimSuffix(e.ChildText("a.Link--muted"), " revisions") + }) + sc.OnHTML("div.Box--condensed", func(e *colly.HTMLElement) { + Scrape.PageNum = e.ChildText("span.Counter--primary") + e.ForEach("ul.list-style-none li.Box-row", func(i int, el *colly.HTMLElement) { + Scrape.Sidebar = append(Scrape.Sidebar, Sidebar{ + Name: el.ChildText("summary span a.Truncate-text"), + Link: el.ChildAttr("summary span a.Truncate-text", "href"), + }) + }) + }) + sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page")) + // Add scrape-based info to wikiArray + wikiArray = append(wikiArray, Scrape) + + return c.Render("wiki", fiber.Map{ + "title": "Wiki page " + c.Params("page") + " | " + c.Params("user") + "/" + c.Params("repo"), + "wiki": wikiArray, + }) +} diff --git a/serve/serve.go b/serve/serve.go index 6ee4a3c..fa6fda3 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -111,6 +111,11 @@ func Serve(port string) { app.Get("/:user/:repo", pages.HandleRepo) app.Get("/:user/:repo/blob/:branch/+", pages.FileView) app.Get("/:user/:repo/tree/:branch/+", pages.DirView) + app.Get("/:user/:repo/wiki", func(c *fiber.Ctx) error { + c.Redirect("/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/Home") + return nil + }) + app.Get("/:user/:repo/wiki/:page", pages.WikiView) app.Get("/:user/:repo/tree/:branch", pages.HandleRepo) app.Get("/download/:user/:repo/:branch", func(c *fiber.Ctx) error { utils.ProxyRequest(c, "https://github.com/"+c.Params("user")+"/"+c.Params("repo")+"/archive/"+c.Params("branch")+".zip") diff --git a/views/wiki.html b/views/wiki.html new file mode 100644 index 0000000..4a6a8a8 --- /dev/null +++ b/views/wiki.html @@ -0,0 +1,17 @@ +{{ template "header" .}} + +
+ {{ if .wiki }} {{ range $key, $value := .wiki}} +

RepoUser: {{.RepoUser}}

+

RepoName: {{.RepoName}}

+

Title: {{.Title}}

+

LastEdit: {{.LastEdit}}

+

LastEditor: {{.LastEditor}}

+

RevisionNum: {{.RevisionNum}}

+

Sidebar: {{.Sidebar}}

+ {{ end }} {{ else }} +

Wiki not found

+

That wiki doesn't exist.

+ {{ end }} +
+{{ template "footer" .}} From fa049697e73fa06d37316f10e6110ad3c2db935a Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 26 Apr 2023 20:21:23 +0530 Subject: [PATCH 2/7] add support for page content and number of pages --- pages/wiki.go | 7 ++++++- views/wiki.html | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pages/wiki.go b/pages/wiki.go index 623794f..3fa4208 100644 --- a/pages/wiki.go +++ b/pages/wiki.go @@ -1,6 +1,7 @@ package pages import ( + "codeberg.org/gothub/gothub/utils" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "log" @@ -61,7 +62,7 @@ func WikiView(c *fiber.Ctx) error { Scrape.LastEditor = re.ReplaceAllString(strings.Replace(e.ChildText("div.gh-header-meta"), "\n", "", -1), "${1}") Scrape.RevisionNum = strings.TrimSuffix(e.ChildText("a.Link--muted"), " revisions") }) - sc.OnHTML("div.Box--condensed", func(e *colly.HTMLElement) { + sc.OnHTML("div.wiki-pages-box div.Box--condensed", func(e *colly.HTMLElement) { Scrape.PageNum = e.ChildText("span.Counter--primary") e.ForEach("ul.list-style-none li.Box-row", func(i int, el *colly.HTMLElement) { Scrape.Sidebar = append(Scrape.Sidebar, Sidebar{ @@ -70,6 +71,10 @@ func WikiView(c *fiber.Ctx) error { }) }) }) + sc.OnHTML("div#wiki-body > div.markdown-body", func(e *colly.HTMLElement) { + Content, _ := e.DOM.Html() + Scrape.PageContent = string(utils.UGCPolicy().SanitizeBytes([]byte(Content))) + }) sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page")) // Add scrape-based info to wikiArray wikiArray = append(wikiArray, Scrape) diff --git a/views/wiki.html b/views/wiki.html index 4a6a8a8..9485def 100644 --- a/views/wiki.html +++ b/views/wiki.html @@ -8,7 +8,10 @@

LastEdit: {{.LastEdit}}

LastEditor: {{.LastEditor}}

RevisionNum: {{.RevisionNum}}

+

PageNum: {{.PageNum}}

Sidebar: {{.Sidebar}}

+

Content:

+ {{ unescape .PageContent }} {{ end }} {{ else }}

Wiki not found

That wiki doesn't exist.

From 703c0ee63e20da96960bef3ec571dde1e8243551 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Thu, 27 Apr 2023 12:31:51 +0530 Subject: [PATCH 3/7] replace https://github.com with / --- pages/wiki.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pages/wiki.go b/pages/wiki.go index 3fa4208..c35b564 100644 --- a/pages/wiki.go +++ b/pages/wiki.go @@ -73,7 +73,8 @@ func WikiView(c *fiber.Ctx) error { }) sc.OnHTML("div#wiki-body > div.markdown-body", func(e *colly.HTMLElement) { Content, _ := e.DOM.Html() - Scrape.PageContent = string(utils.UGCPolicy().SanitizeBytes([]byte(Content))) + Scrape.PageContent = strings.Replace(string(utils.UGCPolicy().SanitizeBytes([]byte(Content))), "https://github.com", "", -1) + }) sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/wiki/" + c.Params("page")) // Add scrape-based info to wikiArray From 5ec6b2f9a9eeaa352cbfb1af9ff46e9aea8a46e4 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Fri, 28 Apr 2023 16:46:36 +0530 Subject: [PATCH 4/7] wrap repo name if it overflows --- public/css/global.css | 1 + 1 file changed, 1 insertion(+) diff --git a/public/css/global.css b/public/css/global.css index b01effb..e4fd442 100644 --- a/public/css/global.css +++ b/public/css/global.css @@ -147,6 +147,7 @@ a:hover { margin: 4px; word-wrap: nowrap; text-overflow: ellipsis; + overflow-wrap: anywhere; } .user-profile h2 { From 3183a3616b8afdb8eda2a5be11bb70786c74b98f Mon Sep 17 00:00:00 2001 From: Midou Date: Thu, 15 Jun 2023 17:28:21 +0100 Subject: [PATCH 5/7] Initial CSS work. This is somewhat different but it should be close to GitHub's layout. --- public/css/global.css | 51 +++++++++++++++++++++++++++++++++++++------ views/wiki.html | 38 ++++++++++++++++++++++---------- 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/public/css/global.css b/public/css/global.css index e4fd442..9fea7cf 100644 --- a/public/css/global.css +++ b/public/css/global.css @@ -6,12 +6,14 @@ --secondary-background: #353535; --background-darker: #151515; --accent: #00b7c3; - --yellow: #8B8000; + --yellow: #8b8000; color-scheme: dark; } body { - font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, arial, sans-serif; + font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, + helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, arial, + sans-serif; color: var(--text); background-color: var(--background-darker); transition: ease-in-out 0.25s; @@ -32,6 +34,9 @@ main { margin: 0 24vw; } +#wrap { + display: flex; +} .navbar { display: flex; align-items: center; @@ -73,7 +78,9 @@ main { background-color: var(--background); padding: 8px; border-radius: 4px; - font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, arial, sans-serif; + font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, + helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, arial, + sans-serif; color: var(--text); } @@ -95,7 +102,9 @@ a:hover { background-color: var(--yellow); padding: 8px; border-radius: 4px; - font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, arial, sans-serif; + font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, + helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, arial, + sans-serif; color: var(--text); } @@ -111,7 +120,8 @@ a:hover { /* URI: /explore */ -.explore-card, .user-repo-card { +.explore-card, +.user-repo-card { background-color: var(--background); padding: 8px; border-radius: 4px; @@ -125,7 +135,8 @@ a:hover { transition: ease-in-out 0.25s; } -.explore-card:hover, .user-repo-card:hover { +.explore-card:hover, +.user-repo-card:hover { background-color: var(--secondary-background); color: var(--text); } @@ -267,6 +278,23 @@ a:hover { background-color: var(--secondary-background); } +/* Overwrite */ + +.no-margin { + margin: 0; +} + +.no-padding { + padding: 0; +} + +.float-right { + float: right; + display: grid; + flex: none; + align-self: flex-start; +} + @media screen and (prefers-color-scheme: light) { :root { --text: #000; @@ -294,6 +322,15 @@ a:hover { margin: 8px; } + #wrap { + display: inline-grid; + } + + .float-right { + float: none; + display: inline-grid; + } + .navbar-slogan { display: none; } @@ -313,5 +350,5 @@ a:hover { } } .cl { - padding-left: 8px + padding-left: 8px; } diff --git a/views/wiki.html b/views/wiki.html index 9485def..5a26d91 100644 --- a/views/wiki.html +++ b/views/wiki.html @@ -1,18 +1,32 @@ {{ template "header" .}} -
+ +
{{ if .wiki }} {{ range $key, $value := .wiki}} -

RepoUser: {{.RepoUser}}

-

RepoName: {{.RepoName}}

-

Title: {{.Title}}

-

LastEdit: {{.LastEdit}}

-

LastEditor: {{.LastEditor}}

-

RevisionNum: {{.RevisionNum}}

-

PageNum: {{.PageNum}}

-

Sidebar: {{.Sidebar}}

-

Content:

- {{ unescape .PageContent }} - {{ end }} {{ else }} +
+
+

{{.Title}}

+

{{.RepoUser}}/{{.RepoName}}

+

+ {{.LastEditor}} edited this page on {{.LastEdit}} · {{.RevisionNum}} revisions +

+
+
+ {{ unescape .PageContent }} +
+
+
+

{{.PageNum}} pages

+
+ {{ range $key, $value := .Sidebar }} +
+

{{.Name}}

 + {{end}} +
+
+
+ {{ end }} + {{ else }}

Wiki not found

That wiki doesn't exist.

{{ end }} From bd8353a36b8d062cce0f95d0837e0bfb97d1e1c8 Mon Sep 17 00:00:00 2001 From: Midou Date: Thu, 15 Jun 2023 17:33:20 +0100 Subject: [PATCH 6/7] Whoops --- views/wiki.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/wiki.html b/views/wiki.html index 5a26d91..e366923 100644 --- a/views/wiki.html +++ b/views/wiki.html @@ -13,7 +13,7 @@
{{ unescape .PageContent }} -
+

{{.PageNum}} pages

From 7d1fdeb456a30bd9479b978a65604db19b0170cf Mon Sep 17 00:00:00 2001 From: Midou Date: Sun, 18 Jun 2023 10:56:55 +0100 Subject: [PATCH 7/7] Base styling for wiki is done and ready to be merged. --- public/css/global.css | 14 +++++++++++++- views/wiki.html | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/public/css/global.css b/public/css/global.css index 9fea7cf..88b4889 100644 --- a/public/css/global.css +++ b/public/css/global.css @@ -34,6 +34,17 @@ main { margin: 0 24vw; } +.margin-ow { + margin: 0 12vw; +} + +pre { + overflow: scroll; + background: var(--secondary-background); + border-radius: 4px; + padding: 4px; +} + #wrap { display: flex; } @@ -323,7 +334,8 @@ a:hover { } #wrap { - display: inline-grid; + display: flex; + flex-direction: column; } .float-right { diff --git a/views/wiki.html b/views/wiki.html index e366923..5e60e22 100644 --- a/views/wiki.html +++ b/views/wiki.html @@ -1,7 +1,7 @@ {{ template "header" .}} -
+
{{ if .wiki }} {{ range $key, $value := .wiki}}
@@ -11,7 +11,7 @@ {{.LastEditor}} edited this page on {{.LastEdit}} · {{.RevisionNum}} revisions

-
+
{{ unescape .PageContent }}