mirror of
https://github.com/httpjamesm/AnonymousOverflow.git
synced 2024-12-06 19:16:43 +01:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a232a33b72 | |||
| 42ad68fe34 | |||
| 634c7f1ad0 | |||
| 4db7b4795b | |||
| 13418054b4 | |||
| ea455f9317 | |||
| aa74bbc5fc | |||
| 56c9e4b196 | |||
| 16828585ac | |||
| cf89be2a48 | |||
| dcf0805334 |
@@ -23,7 +23,7 @@ tmp_dir = "tmp"
|
||||
rerun = false
|
||||
rerun_delay = 500
|
||||
send_interrupt = false
|
||||
stop_on_error = false
|
||||
stop_on_error = true
|
||||
|
||||
[color]
|
||||
app = ""
|
||||
|
||||
@@ -14,26 +14,30 @@ This project is super lightweight by design. The UI is simple and the frontend i
|
||||
|
||||
## Clearnet Instances
|
||||
|
||||
| Instance URL | Region | Notes |
|
||||
| ------------------------------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------ |
|
||||
| [code.whatever.social](https://code.whatever.social) | United States & The Netherlands | Operated by [Whatever Social](https://whatever.social) and [http.james](https://httpjames.space) |
|
||||
| [ao.vern.cc](https://ao.vern.cc) | United States | Operated by [vern.cc](https://vern.cc) |
|
||||
| [overflow.smnz.de](https://overflow.smnz.de) | Germany | Operated by [smnz.de](https://smnz.de) |
|
||||
| [overflow.lunar.icu](https://overflow.lunar.icu) | Germany | Operated by [lunar.icu](https://lunar.icu/) |
|
||||
| [overflow.adminforge.de](https://overflow.adminforge.de/) | Germany | Operated by [adminForge](https://adminforge.de/) |
|
||||
| [overflow.hostux.net](https://overflow.hostux.net/) | France | Operated by [Hostux](https://hostux.net/) |
|
||||
| [overflow.projectsegfau.lt](https://overflow.projectsegfau.lt/) | United States, France, India | Operated by [Project Segfault](https://projectsegfau.lt/) |
|
||||
| [code.xbdm.fun](https://code.xbdm.fun) | Germany | Operated by [xbdm.fun](https://xbdm.fun) |
|
||||
| [overflow.fascinated.cc](https://overflow.fascinated.cc/) | Germany | Operated by [fascinated.cc](https://fascinated.cc/) |
|
||||
| [ao.bloatcat.tk](https://ao.bloatcat.tk) | Iceland | Operated by [bloatcat.tk](https://bloatcat.tk) |
|
||||
| [anonoverflow.frontendfriendly.xyz](https://anonoverflow.frontendfriendly.xyz/) | United States | Operated by [frontendfriendly.xyz](https://frontendfriendly.xyz/) |
|
||||
| [ao.owo.si](https://ao.owo.si/) | Germany | Operated by [owo.si](https://owo.si/) |
|
||||
| [overflow.datura.network](https://overflow.datura.network/) | Germany | Operated by [datura.network](https://datura.network) |
|
||||
| [overflow.freedit.eu](overflow.freedit.eu) | United States | Operated by [freedit.eu](https://freedit.eu) |
|
||||
| [ao.ftw.lol](https://ao.ftw.lol) | Germany | Operated by [ftw.lol](https://ftw.lol) |
|
||||
| [anonoverflow.hyperreal.coffee](https://anonoverflow.hyperreal.coffee) | United States | Operated by [hyperreal.coffee](https://hyperreal.coffee) |
|
||||
| [a.opnxng.com](a.opnxng.com) | Singapore | Operated by [opnxng.com](https://about.opnxng.com) |
|
||||
| [overflow.sudovanilla.com](https://overflow.sudovanilla.com) | United States | Operated by [SudoVanilla](https://sudovanilla.com) |
|
||||
| Instance URL | Region | Notes |
|
||||
| ----------------------------------------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------ |
|
||||
| [code.whatever.social](https://code.whatever.social) | United States & The Netherlands | Operated by [Whatever Social](https://whatever.social) and [http.james](https://httpjames.space) |
|
||||
| [ao.vern.cc](https://ao.vern.cc) | United States | Operated by [vern.cc](https://vern.cc) |
|
||||
| [overflow.smnz.de](https://overflow.smnz.de) | Germany | Operated by [smnz.de](https://smnz.de) |
|
||||
| [overflow.lunar.icu](https://overflow.lunar.icu) | Germany | Operated by [lunar.icu](https://lunar.icu/) |
|
||||
| [overflow.adminforge.de](https://overflow.adminforge.de/) | Germany | Operated by [adminForge](https://adminforge.de/) |
|
||||
| [overflow.hostux.net](https://overflow.hostux.net/) | France | Operated by [Hostux](https://hostux.net/) |
|
||||
| [overflow.projectsegfau.lt](https://overflow.projectsegfau.lt/) | United States, France, India | Operated by [Project Segfault](https://projectsegfau.lt/) |
|
||||
| [code.xbdm.fun](https://code.xbdm.fun) | Germany | Operated by [xbdm.fun](https://xbdm.fun) |
|
||||
| [overflow.fascinated.cc](https://overflow.fascinated.cc/) | Germany | Operated by [fascinated.cc](https://fascinated.cc/) |
|
||||
| [ao.bloatcat.tk](https://ao.bloatcat.tk) | Iceland | Operated by [bloatcat.tk](https://bloatcat.tk) |
|
||||
| [anonoverflow.frontendfriendly.xyz](https://anonoverflow.frontendfriendly.xyz/) | United States | Operated by [frontendfriendly.xyz](https://frontendfriendly.xyz/) |
|
||||
| [ao.owo.si](https://ao.owo.si/) | Germany | Operated by [owo.si](https://owo.si/) |
|
||||
| [overflow.datura.network](https://overflow.datura.network/) | Germany | Operated by [datura.network](https://datura.network) |
|
||||
| [overflow.freedit.eu](overflow.freedit.eu) | United States | Operated by [freedit.eu](https://freedit.eu) |
|
||||
| [ao.ftw.lol](https://ao.ftw.lol) | Germany | Operated by [ftw.lol](https://ftw.lol) |
|
||||
| [anonoverflow.hyperreal.coffee](https://anonoverflow.hyperreal.coffee) | United States | Operated by [hyperreal.coffee](https://hyperreal.coffee) |
|
||||
| [a.opnxng.com](a.opnxng.com) | Singapore | Operated by [opnxng.com](https://about.opnxng.com) |
|
||||
| [overflow.sudovanilla.com](https://overflow.sudovanilla.com) | United States | Operated by [SudoVanilla](https://sudovanilla.com) |
|
||||
| [anonymousoverflow.privacyfucking.rocks](https://anonymousoverflow.privacyfucking.rocks/) | Germany | Operated by [privacyfucking.rocks](https://privacyfucking.rocks) |
|
||||
| [exchange.seitan-ayoub.lol](https://exchange.seitan-ayoub.lol) | Germany | Operated by [Seitan Ayoub](https://seitan-ayoub.lol) |
|
||||
| [overflow.r4fo.com](https://overflow.r4fo.com) | The Netherlands | Operated by [r4fo.com](https://r4fo.com) |
|
||||
| [overflow.ducks.party](https://overflow.ducks.party) | The Netherlands | Operated by [ducks.party](https://ducks.party) |
|
||||
|
||||
## Other Instances
|
||||
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
package config
|
||||
|
||||
var Version = "1.10.0"
|
||||
var Version = "1.10.1"
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
anonymousoverflow:
|
||||
container_name: 'app'
|
||||
build:
|
||||
context: .
|
||||
network: 'host'
|
||||
environment:
|
||||
- APP_URL=https://domain.com
|
||||
- JWT_SIGNING_SECRET=secret
|
||||
ports:
|
||||
- '80:8080'
|
||||
restart: 'always'
|
||||
anonymousoverflow:
|
||||
container_name: 'app'
|
||||
image: 'ghcr.io/httpjamesm/anonymousoverflow:release'
|
||||
environment:
|
||||
- APP_URL=https://domain.com
|
||||
- JWT_SIGNING_SECRET=secret
|
||||
ports:
|
||||
- '80:8080'
|
||||
restart: 'always'
|
||||
|
||||
@@ -51,6 +51,7 @@ func main() {
|
||||
r.POST("/", routes.PostHome)
|
||||
|
||||
r.GET("/a/:id", routes.RedirectShortenedOverflowURL)
|
||||
r.GET("/q/:id", routes.RedirectShortenedOverflowURL)
|
||||
|
||||
r.GET("/questions/:id", func(c *gin.Context) {
|
||||
// redirect user to the question with the title
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ func PostHome(c *gin.Context) {
|
||||
|
||||
// validate URL
|
||||
isStackOverflow := strings.HasPrefix(soLink, "https://stackoverflow.com/questions/")
|
||||
isShortenedStackOverflow := strings.HasPrefix(soLink, "https://stackoverflow.com/a/")
|
||||
isShortenedStackOverflow := strings.HasPrefix(soLink, "https://stackoverflow.com/a/") || strings.HasPrefix(soLink, "https://stackoverflow.com/q/")
|
||||
isStackExchange := stackExchangeRegex.MatchString(soLink)
|
||||
if !isStackExchange && !isStackOverflow && !isShortenedStackOverflow {
|
||||
c.HTML(400, "home.html", gin.H{
|
||||
|
||||
+182
-195
@@ -29,7 +29,6 @@ var soSortValues = map[string]string{
|
||||
}
|
||||
|
||||
func ViewQuestion(c *gin.Context) {
|
||||
client := resty.New()
|
||||
|
||||
questionId := c.Param("id")
|
||||
if _, err := strconv.Atoi(questionId); err != nil {
|
||||
@@ -41,42 +40,24 @@ func ViewQuestion(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
questionTitle := c.Param("title")
|
||||
|
||||
sortValue := c.Query("sort_by")
|
||||
if sortValue == "" {
|
||||
sortValue = "votes"
|
||||
params, err := parseAndValidateParameters(c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
soSortValue, ok := soSortValues[sortValue]
|
||||
if !ok {
|
||||
soSortValue = soSortValues["votes"]
|
||||
}
|
||||
|
||||
sub := c.Param("sub")
|
||||
|
||||
domain := "stackoverflow.com"
|
||||
|
||||
if sub != "" {
|
||||
domain = fmt.Sprintf("%s.stackexchange.com", sub)
|
||||
if params.Sub != "" {
|
||||
domain = fmt.Sprintf("%s.stackexchange.com", params.Sub)
|
||||
}
|
||||
|
||||
soLink := fmt.Sprintf("https://%s/questions/%s/%s?answertab=%s", domain, questionId, questionTitle, soSortValue)
|
||||
soLink := fmt.Sprintf("https://%s/questions/%s/%s?answertab=%s", domain, questionId, params.QuestionTitle, params.SoSortValue)
|
||||
|
||||
resp, err := client.R().Get(soLink)
|
||||
if err != nil {
|
||||
c.HTML(500, "home.html", gin.H{
|
||||
"errorMessage": "Unable to fetch question data",
|
||||
"theme": c.MustGet("theme").(string),
|
||||
"version": config.Version,
|
||||
})
|
||||
return
|
||||
}
|
||||
defer resp.RawResponse.Body.Close()
|
||||
resp, err := fetchQuestionData(soLink)
|
||||
|
||||
if resp.StatusCode() != 200 {
|
||||
c.HTML(500, "home.html", gin.H{
|
||||
"errorMessage": "Received a non-OK status code",
|
||||
"errorMessage": fmt.Sprintf("Received a non-OK status code %d", resp.StatusCode()),
|
||||
"theme": c.MustGet("theme").(string),
|
||||
"version": config.Version,
|
||||
})
|
||||
@@ -97,181 +78,25 @@ func ViewQuestion(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
newFilteredQuestion := types.FilteredQuestion{}
|
||||
|
||||
questionTextParent := doc.Find("h1.fs-headline1")
|
||||
|
||||
questionText := questionTextParent.Children().First().Text()
|
||||
|
||||
newFilteredQuestion.Title = questionText
|
||||
|
||||
questionPostLayout := doc.Find("div.post-layout").First()
|
||||
|
||||
questionTags := utils.GetPostTags(questionPostLayout)
|
||||
newFilteredQuestion.Tags = questionTags
|
||||
|
||||
questionBodyParent := doc.Find("div.s-prose")
|
||||
|
||||
questionBodyParentHTML, err := questionBodyParent.Html()
|
||||
newFilteredQuestion, err := extractQuestionData(doc, domain)
|
||||
if err != nil {
|
||||
c.HTML(500, "home.html", gin.H{
|
||||
"errorMessage": "Unable to parse question body",
|
||||
"errorMessage": "Failed to extract question data",
|
||||
"theme": c.MustGet("theme").(string),
|
||||
"version": config.Version,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
newFilteredQuestion.Body = template.HTML(utils.ReplaceImgTags(questionBodyParentHTML))
|
||||
|
||||
questionBodyText := questionBodyParent.Text()
|
||||
|
||||
// remove all whitespace to create the shortened body desc
|
||||
shortenedBody := strings.TrimSpace(questionBodyText)
|
||||
|
||||
// remove all newlines
|
||||
shortenedBody = strings.ReplaceAll(shortenedBody, "\n", " ")
|
||||
|
||||
// get the first 50 chars
|
||||
shortenedBody = shortenedBody[:50]
|
||||
|
||||
newFilteredQuestion.ShortenedBody = shortenedBody
|
||||
|
||||
comments := utils.FindAndReturnComments(questionBodyParentHTML, domain, questionPostLayout)
|
||||
newFilteredQuestion.Comments = comments
|
||||
|
||||
// parse any code blocks and highlight them
|
||||
answerCodeBlocks := questionCodeBlockRegex.FindAllString(questionBodyParentHTML, -1)
|
||||
for _, codeBlock := range answerCodeBlocks {
|
||||
codeBlock = utils.StripBlockTags(codeBlock)
|
||||
|
||||
// syntax highlight
|
||||
highlightedCodeBlock := utils.HighlightSyntaxViaContent(codeBlock)
|
||||
|
||||
// replace the code block with the highlighted code block
|
||||
questionBodyParentHTML = strings.Replace(questionBodyParentHTML, codeBlock, highlightedCodeBlock, 1)
|
||||
}
|
||||
|
||||
questionCard := doc.Find("div.postcell")
|
||||
|
||||
questionMetadata := questionCard.Find("div.user-info")
|
||||
questionTimestamp := ""
|
||||
questionMetadata.Find("span.relativetime").Each(func(i int, s *goquery.Selection) {
|
||||
// get the second
|
||||
if i == 0 {
|
||||
if s.Text() != "" {
|
||||
// if it's not been edited, it means it's the first
|
||||
questionTimestamp = s.Text()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// otherwise it's the second element
|
||||
if i == 1 {
|
||||
questionTimestamp = s.Text()
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
newFilteredQuestion.Timestamp = questionTimestamp
|
||||
|
||||
userDetails := questionMetadata.Find("div.user-details")
|
||||
|
||||
questionAuthor := ""
|
||||
questionAuthorURL := fmt.Sprintf("https://%s", domain)
|
||||
|
||||
// check if the question has been edited
|
||||
isQuestionEdited := false
|
||||
questionMetadata.Find("a.js-gps-track").Each(func(i int, s *goquery.Selection) {
|
||||
if strings.Contains(s.Text(), "edited") {
|
||||
isQuestionEdited = true
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
userDetails.Find("a").Each(func(i int, s *goquery.Selection) {
|
||||
// if question has been edited, the author is the second element.
|
||||
|
||||
if isQuestionEdited {
|
||||
if i == 1 {
|
||||
questionAuthor = s.Text()
|
||||
questionAuthorURL += s.AttrOr("href", "")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// otherwise it's the first element
|
||||
if i == 0 {
|
||||
questionAuthor = s.Text()
|
||||
questionAuthorURL += s.AttrOr("href", "")
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
newFilteredQuestion.AuthorName = questionAuthor
|
||||
newFilteredQuestion.AuthorURL = questionAuthorURL
|
||||
|
||||
answers := []types.FilteredAnswer{}
|
||||
|
||||
doc.Find("div.answer").Each(func(i int, s *goquery.Selection) {
|
||||
|
||||
newFilteredAnswer := types.FilteredAnswer{}
|
||||
|
||||
postLayout := s.Find("div.post-layout")
|
||||
voteCell := postLayout.Find("div.votecell")
|
||||
answerCell := postLayout.Find("div.answercell")
|
||||
answerBody := answerCell.Find("div.s-prose")
|
||||
answerBodyHTML, _ := answerBody.Html()
|
||||
|
||||
voteCount := html.EscapeString(voteCell.Find("div.js-vote-count").Text())
|
||||
|
||||
newFilteredAnswer.Upvotes = voteCount
|
||||
newFilteredAnswer.IsAccepted = s.HasClass("accepted-answer")
|
||||
|
||||
answerFooter := s.Find("div.mt24")
|
||||
|
||||
answerAuthorURL := fmt.Sprintf("https://%s", domain)
|
||||
answerAuthorName := ""
|
||||
answerTimestamp := ""
|
||||
|
||||
answerFooter.Find("div.post-signature").Each(func(i int, s *goquery.Selection) {
|
||||
answerAuthorDetails := s.Find("div.user-details")
|
||||
|
||||
if answerAuthorDetails.Length() > 0 {
|
||||
questionAuthor := answerAuthorDetails.Find("a").First()
|
||||
answerAuthorName = html.EscapeString(questionAuthor.Text())
|
||||
answerAuthorURL += html.EscapeString(questionAuthor.AttrOr("href", ""))
|
||||
}
|
||||
|
||||
answerTimestamp = html.EscapeString(s.Find("span.relativetime").Text())
|
||||
answers, err := extractAnswersData(doc, domain)
|
||||
if err != nil {
|
||||
c.HTML(500, "home.html", gin.H{
|
||||
"errorMessage": "Failed to extract answer data",
|
||||
"theme": c.MustGet("theme").(string),
|
||||
"version": config.Version,
|
||||
})
|
||||
|
||||
answerId, _ := s.Attr("data-answerid")
|
||||
newFilteredAnswer.ID = answerId
|
||||
|
||||
newFilteredAnswer.AuthorName = answerAuthorName
|
||||
newFilteredAnswer.AuthorURL = answerAuthorURL
|
||||
newFilteredAnswer.Timestamp = answerTimestamp
|
||||
|
||||
// parse any code blocks and highlight them
|
||||
answerCodeBlocks := codeBlockRegex.FindAllString(answerBodyHTML, -1)
|
||||
for _, codeBlock := range answerCodeBlocks {
|
||||
codeBlock = utils.StripBlockTags(codeBlock)
|
||||
|
||||
// syntax highlight
|
||||
highlightedCodeBlock := utils.HighlightSyntaxViaContent(codeBlock)
|
||||
|
||||
// replace the code block with the highlighted code block
|
||||
answerBodyHTML = strings.Replace(answerBodyHTML, codeBlock, highlightedCodeBlock, 1)
|
||||
}
|
||||
|
||||
comments = utils.FindAndReturnComments(answerBodyHTML, domain, postLayout)
|
||||
|
||||
newFilteredAnswer.Comments = comments
|
||||
newFilteredAnswer.Body = template.HTML(utils.ReplaceImgTags(answerBodyHTML))
|
||||
|
||||
answers = append(answers, newFilteredAnswer)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
imagePolicy := "'self' https:"
|
||||
|
||||
@@ -284,9 +109,171 @@ func ViewQuestion(c *gin.Context) {
|
||||
"answers": answers,
|
||||
"imagePolicy": imagePolicy,
|
||||
"theme": c.MustGet("theme").(string),
|
||||
"currentUrl": fmt.Sprintf("%s/questions/%s/%s", os.Getenv("APP_URL"), questionId, questionTitle),
|
||||
"sortValue": sortValue,
|
||||
"currentUrl": fmt.Sprintf("%s%s", os.Getenv("APP_URL"), c.Request.URL.Path),
|
||||
"sortValue": params.SoSortValue,
|
||||
"domain": domain,
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
type viewQuestionInputs struct {
|
||||
QuestionID string
|
||||
QuestionTitle string
|
||||
SoSortValue string
|
||||
Sub string
|
||||
}
|
||||
|
||||
// parseAndValidateParameters consolidates the URL and query parameters into an easily-accessible struct.
|
||||
func parseAndValidateParameters(c *gin.Context) (inputs viewQuestionInputs, err error) {
|
||||
|
||||
questionId := c.Param("id")
|
||||
if _, err = strconv.Atoi(questionId); err != nil {
|
||||
c.HTML(400, "home.html", gin.H{
|
||||
"errorMessage": "Invalid question ID",
|
||||
"theme": c.MustGet("theme").(string),
|
||||
"version": config.Version,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
inputs.QuestionID = questionId
|
||||
|
||||
sortValue := c.Query("sort_by")
|
||||
if sortValue == "" {
|
||||
sortValue = "votes"
|
||||
}
|
||||
|
||||
soSortValue, ok := soSortValues[sortValue]
|
||||
if !ok {
|
||||
soSortValue = soSortValues["votes"]
|
||||
}
|
||||
|
||||
inputs.SoSortValue = soSortValue
|
||||
|
||||
sub := c.Param("sub")
|
||||
|
||||
inputs.Sub = sub
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// fetchQuestionData sends the request to StackOverflow.
|
||||
func fetchQuestionData(soLink string) (resp *resty.Response, err error) {
|
||||
client := resty.New()
|
||||
resp, err = client.R().Get(soLink)
|
||||
return
|
||||
}
|
||||
|
||||
// extractQuestionData parses the HTML document and extracts question data.
|
||||
func extractQuestionData(doc *goquery.Document, domain string) (question types.FilteredQuestion, err error) {
|
||||
// Extract the question title.
|
||||
questionTextParent := doc.Find("h1.fs-headline1").First()
|
||||
question.Title = strings.TrimSpace(questionTextParent.Children().First().Text())
|
||||
|
||||
// Extract question tags.
|
||||
questionTags := utils.GetPostTags(doc.Find("div.post-layout").First())
|
||||
question.Tags = questionTags
|
||||
|
||||
// Extract and process the question body.
|
||||
questionBodyParent := doc.Find("div.s-prose").First()
|
||||
questionBodyParentHTML, err := questionBodyParent.Html()
|
||||
if err != nil {
|
||||
return question, err
|
||||
}
|
||||
question.Body = template.HTML(processHTMLBody(questionBodyParentHTML))
|
||||
|
||||
// Extract the shortened body description.
|
||||
shortenedBody := strings.TrimSpace(questionBodyParent.Text())
|
||||
shortenedBody = strings.ReplaceAll(shortenedBody, "\n", " ")
|
||||
if len(shortenedBody) > 50 {
|
||||
shortenedBody = shortenedBody[:50]
|
||||
}
|
||||
question.ShortenedBody = shortenedBody
|
||||
|
||||
// Extract question comments.
|
||||
comments := utils.FindAndReturnComments(questionBodyParentHTML, domain, doc.Find("div.post-layout").First())
|
||||
question.Comments = comments
|
||||
|
||||
// Extract question timestamp and author information.
|
||||
questionCard := doc.Find("div.postcell").First()
|
||||
extractMetadata(questionCard, &question, domain)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// extractMetadata extracts author and timestamp information from a given selection.
|
||||
func extractMetadata(selection *goquery.Selection, question *types.FilteredQuestion, domain string) {
|
||||
questionMetadata := selection.Find("div.user-info").First()
|
||||
question.Timestamp = questionMetadata.Find("span.relativetime").First().Text()
|
||||
|
||||
questionAuthorURL := "https://" + domain
|
||||
questionAuthor := selection.Find("div.post-signature.owner div.user-info div.user-details a").First()
|
||||
question.AuthorName = questionAuthor.Text()
|
||||
questionAuthorURL += questionAuthor.AttrOr("href", "")
|
||||
question.AuthorURL = questionAuthorURL
|
||||
|
||||
// Determine if the question has been edited and update author details accordingly.
|
||||
isQuestionEdited := selection.Find("a.js-gps-track").Text() == "edited"
|
||||
if isQuestionEdited {
|
||||
editedAuthor := questionMetadata.Find("a").Last()
|
||||
question.AuthorName = editedAuthor.Text()
|
||||
question.AuthorURL = "https://" + domain + editedAuthor.AttrOr("href", "")
|
||||
}
|
||||
}
|
||||
|
||||
// extractAnswersData parses the HTML document and extracts answers data.
|
||||
func extractAnswersData(doc *goquery.Document, domain string) ([]types.FilteredAnswer, error) {
|
||||
var answers []types.FilteredAnswer
|
||||
|
||||
// Iterate over each answer block.
|
||||
doc.Find("div.answer").Each(func(i int, s *goquery.Selection) {
|
||||
var answer types.FilteredAnswer
|
||||
|
||||
postLayout := s.Find("div.post-layout").First()
|
||||
|
||||
// Extract upvotes.
|
||||
voteCell := postLayout.Find("div.votecell").First()
|
||||
voteCount := html.EscapeString(voteCell.Find("div.js-vote-count").Text())
|
||||
answer.Upvotes = voteCount
|
||||
|
||||
// Check if the answer is accepted.
|
||||
answer.IsAccepted = s.HasClass("accepted-answer")
|
||||
|
||||
// Extract answer body and process it.
|
||||
answerCell := postLayout.Find("div.answercell").First()
|
||||
answerBody := answerCell.Find("div.s-prose").First()
|
||||
answerBodyHTML, _ := answerBody.Html()
|
||||
|
||||
// Process code blocks within the answer.
|
||||
processedAnswerBody := processHTMLBody(answerBodyHTML)
|
||||
answer.Body = template.HTML(html.UnescapeString(processedAnswerBody))
|
||||
|
||||
// Extract author information and timestamp.
|
||||
extractAnswerAuthorInfo(s, &answer, domain)
|
||||
|
||||
answers = append(answers, answer)
|
||||
})
|
||||
|
||||
return answers, nil
|
||||
}
|
||||
|
||||
// processHTMLBody highlights syntax and replaces images with proxied versions.
|
||||
func processHTMLBody(bodyHTML string) string {
|
||||
highlightedBody := utils.HighlightCodeBlocks(bodyHTML)
|
||||
imageProxiedBody := utils.ReplaceImgTags(highlightedBody)
|
||||
return imageProxiedBody
|
||||
}
|
||||
|
||||
// extractAnswerAuthorInfo extracts the author name, URL, and timestamp from an answer block.
|
||||
// It directly mutates the answer.
|
||||
func extractAnswerAuthorInfo(selection *goquery.Selection, answer *types.FilteredAnswer, domain string) {
|
||||
authorDetails := selection.Find("div.post-signature").Last()
|
||||
|
||||
authorName := html.EscapeString(authorDetails.Find("div.user-details a").First().Text())
|
||||
authorURL := "https://" + domain + authorDetails.Find("div.user-details a").AttrOr("href", "")
|
||||
timestamp := html.EscapeString(authorDetails.Find("span.relativetime").Text())
|
||||
|
||||
answer.AuthorName = authorName
|
||||
answer.AuthorURL = authorURL
|
||||
answer.Timestamp = timestamp
|
||||
}
|
||||
|
||||
+30
-5
@@ -12,7 +12,9 @@ import (
|
||||
"github.com/alecthomas/chroma/styles"
|
||||
)
|
||||
|
||||
func HighlightSyntaxViaContent(content string) (htmlOut string) {
|
||||
// highlightSyntaxViaContent uses Chroma to lex code content and apply the appropriate tokenizer engine.
|
||||
// If it can't find one, it defaults to JavaScript syntax highlighting.
|
||||
func highlightSyntaxViaContent(content string) (htmlOut string) {
|
||||
content = html.UnescapeString(content)
|
||||
|
||||
fallbackOut := html.EscapeString(content)
|
||||
@@ -20,9 +22,7 @@ func HighlightSyntaxViaContent(content string) (htmlOut string) {
|
||||
// identify the language
|
||||
lexer := lexers.Analyse(content)
|
||||
if lexer == nil {
|
||||
// unable to identify, so just return the wrapped content
|
||||
htmlOut = fallbackOut
|
||||
return
|
||||
lexer = lexers.Get(".js")
|
||||
}
|
||||
|
||||
style := styles.Get("xcode")
|
||||
@@ -54,7 +54,9 @@ func HighlightSyntaxViaContent(content string) (htmlOut string) {
|
||||
|
||||
var preClassRegex = regexp.MustCompile(`(?s)<pre class=".+">`)
|
||||
|
||||
func StripBlockTags(content string) (result string) {
|
||||
// stripBlockTags takes an extracted code block from HTML and strips it of its pre and code tags.
|
||||
// What's returned is just the code.
|
||||
func stripBlockTags(content string) (result string) {
|
||||
// strip all "<code>" tags
|
||||
content = strings.Replace(content, "<code>", "", -1)
|
||||
content = strings.Replace(content, "</code>", "", -1)
|
||||
@@ -68,3 +70,26 @@ func StripBlockTags(content string) (result string) {
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var codeBlockRegex = regexp.MustCompile(`(?s)<pre><code>(.*?)<\/code><\/pre>`)
|
||||
|
||||
// HighlightCodeBlocks uses both highlightSyntaxViaContent stripCodeBlocks and returns the newly highlighted code HTML.
|
||||
func HighlightCodeBlocks(html string) string {
|
||||
// Replace each code block with the highlighted version
|
||||
highlightedHTML := codeBlockRegex.ReplaceAllStringFunc(html, func(codeBlock string) string {
|
||||
// Extract the code content from the code block
|
||||
codeContent := codeBlockRegex.FindStringSubmatch(codeBlock)[1]
|
||||
|
||||
codeContent = stripBlockTags(codeContent)
|
||||
|
||||
// Highlight the code content
|
||||
highlightedCode := highlightSyntaxViaContent(codeContent)
|
||||
|
||||
// Replace the original code block with the highlighted version
|
||||
highlightedCodeBlock := "<pre>" + highlightedCode + "</pre>"
|
||||
|
||||
return highlightedCodeBlock
|
||||
})
|
||||
|
||||
return highlightedHTML
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user