diff --git a/core/pixivision.go b/core/pixivision.go index 4cf4123..02f12c7 100644 --- a/core/pixivision.go +++ b/core/pixivision.go @@ -13,18 +13,23 @@ import ( "codeberg.org/vnpower/pixivfe/v2/i18n" ) +// PixivDatetimeLayout defines the date format used by Pixiv const PixivDatetimeLayout = "2006-01-02" +// re_lang is a regular expression to extract the language code from a URL var re_lang = regexp.MustCompile(`.*\/\/.*?\/(.*?)\/`) +// generateRequest creates a new HTTP request with appropriate headers for Pixiv func generateRequest(r *http.Request, link, method string, body io.Reader) *http.Request { req, err := http.NewRequestWithContext(r.Context(), method, link, body) if err != nil { panic(err) } + // Extract language from the URL lang := re_lang.FindStringSubmatch(link)[1] + // Set headers to mimic a browser request req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0") req.Header.Set("Connection", "keep-alive") req.Header.Set("Accept-Language", "en-US,en;q=0.9") @@ -34,6 +39,7 @@ func generateRequest(r *http.Request, link, method string, body io.Reader) *http return req } +// executeRequest sends an HTTP request and handles common error cases func executeRequest(req *http.Request) (*http.Response, error) { client := http.Client{Timeout: 60 * time.Second} resp, err := client.Do(req) @@ -47,6 +53,7 @@ func executeRequest(req *http.Request) (*http.Response, error) { return resp, nil } +// PixivisionArticle represents a single article on Pixivision type PixivisionArticle struct { ID string Title string @@ -58,11 +65,13 @@ type PixivisionArticle struct { Tags []PixivisionEmbedTag } +// PixivisionEmbedTag represents a tag associated with a Pixivision article type PixivisionEmbedTag struct { ID string Name string } +// PixivisionArticleItem represents an item (artwork) within a Pixivision article type PixivisionArticleItem struct { Username string UserID string @@ -72,24 +81,27 @@ type PixivisionArticleItem struct { Image string } +// PixivisionTag represents a tag page on Pixivision type PixivisionTag struct { Title string Description string Articles []PixivisionArticle } +// PixivisionCategory represents a category page on Pixivision type PixivisionCategory struct { Articles []PixivisionArticle Title string Description string } +// generatePixivisionURL creates a URL for Pixivision based on the route and language func generatePixivisionURL(route string, lang []string) string { template := "https://www.pixivision.net/%s/%s" language := "en" // Default availableLangs := []string{"en", "zh", "ja", "zh-tw", "ko"} - // Validation + // Validate and set the language if provided if len(lang) > 0 { t := lang[0] @@ -98,24 +110,28 @@ func generatePixivisionURL(route string, lang []string) string { language = t } } - } return fmt.Sprintf(template, language, route) } +// re_findid is a regular expression to extract the ID from a Pixiv link var re_findid = regexp.MustCompile(`.*\/(\d+)`) +// parseIDFromPixivLink extracts the numeric ID from a Pixiv URL func parseIDFromPixivLink(link string) string { return re_findid.FindStringSubmatch(link)[1] } +// r_img is a regular expression to extract the image URL from a CSS background-image property var r_img = regexp.MustCompile(`.*\((.*)\)`) +// parseBackgroundImage extracts the image URL from a CSS background-image property func parseBackgroundImage(link string) string { return r_img.FindStringSubmatch(link)[1] } +// PixivisionGetHomepage fetches and parses the Pixivision homepage func PixivisionGetHomepage(r *http.Request, page string, lang ...string) ([]PixivisionArticle, error) { var articles []PixivisionArticle @@ -135,7 +151,7 @@ func PixivisionGetHomepage(r *http.Request, page string, lang ...string) ([]Pixi return nil, err } - // TODO: Re-use this function + // Parse each article on the homepage doc.Find("article.spotlight").Each(func(i int, s *goquery.Selection) { var article PixivisionArticle @@ -147,6 +163,7 @@ func PixivisionGetHomepage(r *http.Request, page string, lang ...string) ([]Pixi article.Thumbnail = parseBackgroundImage(s.Find("._thumbnail").AttrOr("style", "")) article.Date, _ = time.Parse(PixivDatetimeLayout, date) + // Parse tags associated with the article s.Find("._tag-list a").Each(func(i int, t *goquery.Selection) { var tag PixivisionEmbedTag tag.ID = parseIDFromPixivLink(t.AttrOr("href", "")) @@ -161,6 +178,7 @@ func PixivisionGetHomepage(r *http.Request, page string, lang ...string) ([]Pixi return articles, nil } +// PixivisionGetTag fetches and parses a tag page on Pixivision func PixivisionGetTag(r *http.Request, id string, page string, lang ...string) (PixivisionTag, error) { var tag PixivisionTag @@ -179,6 +197,7 @@ func PixivisionGetTag(r *http.Request, id string, page string, lang ...string) ( tag.Title = doc.Find(".tdc__header h1").Text() tag.Description = doc.Find(".tdc__description").Text() + // Parse each article in the tag page doc.Find("._article-card").Each(func(i int, s *goquery.Selection) { var article PixivisionArticle @@ -193,6 +212,7 @@ func PixivisionGetTag(r *http.Request, id string, page string, lang ...string) ( date := s.Find("time._date").AttrOr("datetime", "") article.Date, _ = time.Parse(PixivDatetimeLayout, date) + // Parse tags associated with the article s.Find("._tag-list a").Each(func(i int, t *goquery.Selection) { var tag PixivisionEmbedTag tag.ID = parseIDFromPixivLink(t.AttrOr("href", "")) @@ -207,6 +227,7 @@ func PixivisionGetTag(r *http.Request, id string, page string, lang ...string) ( return tag, nil } +// PixivisionGetArticle fetches and parses a single article on Pixivision func PixivisionGetArticle(r *http.Request, id string, lang ...string) (PixivisionArticle, error) { var article PixivisionArticle @@ -222,14 +243,14 @@ func PixivisionGetArticle(r *http.Request, id string, lang ...string) (Pixivisio return article, err } - // Title + // Parse article metadata date := doc.Find("time._date").AttrOr("datetime", "") article.Title = doc.Find("h1.am__title").Text() article.Category = doc.Find(".am__categoty-pr ._category-label").Text() article.Thumbnail = doc.Find(".aie__image").AttrOr("src", "") article.Date, _ = time.Parse(PixivDatetimeLayout, date) - // Description + // Parse article description doc.Find(".fab__paragraph p").Each(func(i int, s *goquery.Selection) { desc, err := s.Html() if err != nil { @@ -245,6 +266,7 @@ func PixivisionGetArticle(r *http.Request, id string, lang ...string) (Pixivisio article.Description = append(article.Description, desc) }) + // Parse artworks featured in the article doc.Find("._feature-article-body__pixiv_illust").Each(func(i int, s *goquery.Selection) { var item PixivisionArticleItem @@ -258,6 +280,7 @@ func PixivisionGetArticle(r *http.Request, id string, lang ...string) (Pixivisio article.Items = append(article.Items, item) }) + // Parse tags associated with the article doc.Find("._tag-list a").Each(func(i int, s *goquery.Selection) { var tag PixivisionEmbedTag tag.ID = parseIDFromPixivLink(s.AttrOr("href", "")) @@ -269,6 +292,7 @@ func PixivisionGetArticle(r *http.Request, id string, lang ...string) (Pixivisio return article, nil } +// PixivisionGetCategory fetches and parses a category page on Pixivision func PixivisionGetCategory(r *http.Request, id string, page string, lang ...string) (PixivisionCategory, error) { var category PixivisionCategory @@ -285,8 +309,9 @@ func PixivisionGetCategory(r *http.Request, id string, page string, lang ...stri } category.Title = doc.Find(".ssc__name").Text() - category.Description = doc.Find(".ssc__descriotion").Text() // Not a typo + category.Description = doc.Find(".ssc__descriotion").Text() // NOTE: This is a typo in the original HTML + // Parse each article in the category page doc.Find("._article-card").Each(func(i int, s *goquery.Selection) { var article PixivisionArticle