From ce60c9ca34e1da35f3a40ea78db7518ebc9dc75a Mon Sep 17 00:00:00 2001 From: VnPower Date: Thu, 18 May 2023 22:03:21 +0700 Subject: [PATCH] Breaking API update!!! --- configs/config.go | 2 +- entity/models.go | 59 ------ handler/{parser.go => _parser.go} | 0 handler/pixiv.go | 326 ++++++++++++++++++++++++++++++ main.go | 1 - models/models.go | 120 +++++++++++ template/artwork.html | 11 +- template/large-tn.html | 17 +- template/small-tn.html | 17 +- template/user.html | 7 +- views/routes.go | 82 +++++--- 11 files changed, 541 insertions(+), 101 deletions(-) delete mode 100644 entity/models.go rename handler/{parser.go => _parser.go} (100%) create mode 100644 handler/pixiv.go create mode 100644 models/models.go diff --git a/configs/config.go b/configs/config.go index f436739..b06b09b 100644 --- a/configs/config.go +++ b/configs/config.go @@ -11,7 +11,7 @@ var Configs Conf type Conf struct { PHPSESSID string `yaml:"PHPSESSID"` - userAgent int `yaml:"userAgent"` + UserAgent string `yaml:"userAgent"` } func (conf *Conf) ReadConfig() { diff --git a/entity/models.go b/entity/models.go deleted file mode 100644 index ebf9a04..0000000 --- a/entity/models.go +++ /dev/null @@ -1,59 +0,0 @@ -package entity - -import ( - "html/template" - "time" -) - -type Illust struct { - ID int `json:"id"` - Title string `json:"title"` - Caption template.HTML `json:"caption"` - Artist User `json:"user"` - Date time.Time `json:"create_date"` - Pages int `json:"page_count"` - Views int `json:"total_view"` - Bookmarks int `json:"total_bookmarks"` - Tags []Tag `json:"tags"` - Images []Image -} - -type Spotlight struct { - ID int `json:"id"` - Title string `json:"title"` - Thumbnail string `json:"thumbnail"` - URL string `json:"article_url"` - Date string `json:"publish_date"` -} - -type Tag struct { - Name string `json:"name"` - TranslatedName string `json:"translated_name"` -} - -type User struct { - ID int `json:"id"` - Name string `json:"name"` - Account string `json:"account"` - Avatar map[string]string `json:"profile_image_urls"` - Webpage string `json:"webpage"` - Gender string `json:"gender"` - Birth string `json:"birth"` - BirthDay string `json:"birth_day"` - BirthYear int `json:"birth_year"` - Region string `json:"region"` - BackgroundImage string `json:"background_image_url"` - Followers int `json:"total_follow_users"` - MyPixiv int `json:"total_mypixiv_users"` - Illusts int `json:"total_illusts"` - Mangas int `json:"total_mangas"` - TwitterURL string `json:"twitter_url"` - PawooURL string `json:"pawoo_url"` -} - -type Image struct { - Small string `json:"square_medium"` - Medium string `json:"medium"` - Large string `json:"large"` - Original string `json:"original"` -} diff --git a/handler/parser.go b/handler/_parser.go similarity index 100% rename from handler/parser.go rename to handler/_parser.go diff --git a/handler/pixiv.go b/handler/pixiv.go new file mode 100644 index 0000000..a084470 --- /dev/null +++ b/handler/pixiv.go @@ -0,0 +1,326 @@ +package models + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "pixivfe/models" + "regexp" + "sort" + "strconv" +) + +type PixivClient struct { + Client *http.Client + + Cookie map[string]string + Header map[string]string + Lang string +} + +const ( + ArtworkInformationURL = "https://www.pixiv.net/ajax/illust/%s" + ArtworkImagesURL = "https://www.pixiv.net/ajax/illust/%s/pages" + ArtworkRelatedURL = "https://www.pixiv.net/ajax/illust/%s/recommend/init?limit=30" + UserInformationURL = "https://www.pixiv.net/ajax/user/%s?full=1" + UserArtworksURL = "https://www.pixiv.net/ajax/user/%s/profile/all" + UserArtworksFullURL = "https://www.pixiv.net/ajax/user/%s/profile/illusts?work_category=illustManga&is_first_page=0&lang=en%s" +) + +func ProxyImage(url string) string { + regex := regexp.MustCompile(`i\.pximg\.net`) + proxy := "px2.rainchan.win" + + return regex.ReplaceAllString(url, proxy) +} + +func (p *PixivClient) SetHeader(header map[string]string) { + p.Header = header +} + +func (p *PixivClient) AddHeader(key, value string) { + p.Header[key] = value +} + +func (p *PixivClient) SetUserAgent(value string) { + p.AddHeader("User-Agent", value) +} + +func (p *PixivClient) SetCookie(cookie map[string]string) { + p.Cookie = cookie +} + +func (p *PixivClient) AddCookie(key, value string) { + p.Cookie[key] = value +} + +func (p *PixivClient) SetSessionID(value string) { + p.Cookie["PHPSESSID"] = value +} + +func (p *PixivClient) SetLang(lang string) { + p.Lang = lang +} + +func (p *PixivClient) Request(URL string) (*http.Response, error) { + req, _ := http.NewRequest("GET", URL, nil) + + // Add headers + for k, v := range p.Header { + req.Header.Add(k, v) + } + for k, v := range p.Cookie { + req.AddCookie(&http.Cookie{Name: k, Value: v}) + } + // Make a request + resp, err := p.Client.Do(req) + + if err != nil { + return resp, err + } + + if resp.StatusCode == http.StatusNotFound { + return resp, errors.New("404 returned") + } + + if resp.StatusCode != 200 { + return resp, errors.New(fmt.Sprintf("Server returned code: %d", resp.StatusCode)) + } + + return resp, nil +} + +func (p *PixivClient) TextRequest(URL string) (string, error) { + resp, err := p.Request(URL) + if err != nil { + return "", err + } + + // Extract the bytes from server's response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return string(body), err + } + + return string(body), nil +} + +func (p *PixivClient) GetArtworkImages(id string) ([]models.Image, error) { + s, _ := p.TextRequest(fmt.Sprintf(ArtworkImagesURL, id)) + + var pr models.PixivResponse + var resp []models.ImageResponse + var images []models.Image + + err := json.Unmarshal([]byte(s), &pr) + if err != nil { + return images, errors.New(fmt.Sprintf("Failed to extract data from JSON response from server. %s", err)) + } + if pr.Error { + return images, errors.New(fmt.Sprintf("Server refuses to return data. %s", err)) + } + + err = json.Unmarshal([]byte(pr.Body), &resp) + if err != nil { + return images, errors.New(fmt.Sprintf("Failed to extract images for illust. %s", err)) + } + + // Extract and proxy every images + for _, imageRaw := range resp { + var image models.Image + + image.Small = ProxyImage(imageRaw.Urls["thumb_mini"]) + image.Medium = ProxyImage(imageRaw.Urls["small"]) + image.Large = ProxyImage(imageRaw.Urls["regular"]) + image.Original = ProxyImage(imageRaw.Urls["original"]) + + images = append(images, image) + } + + return images, nil +} + +func (p *PixivClient) GetArtworkByID(id string) (*models.Illust, error) { + s, _ := p.TextRequest(fmt.Sprintf(ArtworkInformationURL, id)) + + var resp models.PixivResponse + var images []models.Image + + // Parse Pixiv response body + err := json.Unmarshal([]byte(s), &resp) + if err != nil { + return nil, errors.New(fmt.Sprintf("Failed to extract data from JSON response from server. %s", err)) + } + if resp.Error { + return nil, errors.New(fmt.Sprintf("Server refuses to return data. %s", err)) + } + + var illust struct { + *models.Illust + RawTags json.RawMessage `json:"tags"` + } + + // Parse basic illust information + err = json.Unmarshal([]byte(resp.Body), &illust) + + // Get illust images + images, err = p.GetArtworkImages(id) + if err != nil { + fmt.Printf("%s\n", err) + } + + illust.Images = images + + // Extract tags + var tags struct { + Tags []struct { + Tag string `json:"tag"` + Translation map[string]string `json:"translation"` + } `json:"tags"` + } + err = json.Unmarshal(illust.RawTags, &tags) + if err != nil { + fmt.Printf("%s\n", err) + return nil, err + } + + for _, tag := range tags.Tags { + var newTag models.Tag + newTag.Name = tag.Tag + newTag.TranslatedName = tag.Translation["en"] + + illust.Tags = append(illust.Tags, newTag) + } + + return illust.Illust, nil +} + +func (p *PixivClient) GetUserArtworksID(id string) (*string, error) { + s, _ := p.TextRequest(fmt.Sprintf(UserArtworksURL, id)) + + var pr models.PixivResponse + + err := json.Unmarshal([]byte(s), &pr) + + if err != nil { + return nil, errors.New(fmt.Sprintf("Failed to extract data from JSON response from server. %s", err)) + } + if pr.Error { + return nil, errors.New(pr.Message) + } + + var ids []int + var idsString string + var body struct { + Illusts map[int]string `json:"illusts"` + } + err = json.Unmarshal(pr.Body, &body) + + // Get the keys, because Pixiv only returns IDs (very evil) + for k := range body.Illusts { + ids = append(ids, k) + } + + // Reverse sort the ids + sort.Sort(sort.Reverse(sort.IntSlice(ids))) + + i := 0 + for _, k := range ids { + if i < 30 { + idsString += fmt.Sprintf("&ids[]=%d", k) + } else { + break + } + i++ + } + + return &idsString, nil +} + +func (p *PixivClient) GetRelatedArtworks(id string) ([]models.IllustShort, error) { + url := fmt.Sprintf(ArtworkRelatedURL, id) + + var pr models.PixivResponse + + s, err := p.TextRequest(url) + if err != nil { + return nil, err + } + + err = json.Unmarshal([]byte(s), &pr) + + var body struct { + Illusts []models.IllustShort `json:"illusts"` + } + + err = json.Unmarshal([]byte(pr.Body), &body) + if err != nil { + return nil, err + } + + return body.Illusts, nil +} + +func (p *PixivClient) GetUserArtworks(id string) ([]models.IllustShort, error) { + ids, err := p.GetUserArtworksID(id) + if err != nil { + return nil, err + } + url := fmt.Sprintf(UserArtworksFullURL, id, *ids) + + var pr models.PixivResponse + var works []models.IllustShort + + s, err := p.TextRequest(url) + + err = json.Unmarshal([]byte(s), &pr) + + var body struct { + Illusts map[int]json.RawMessage `json:"works"` + } + + err = json.Unmarshal(pr.Body, &body) + + for _, v := range body.Illusts { + var illust models.IllustShort + err = json.Unmarshal(v, &illust) + + works = append(works, illust) + } + + // IDK but the order got shuffled even though Pixiv sorted the IDs in the response + sort.Slice(works[:], func(i, j int) bool { + left, _ := strconv.Atoi(works[i].ID) + right, _ := strconv.Atoi(works[j].ID) + return left > right + }) + + return works, nil +} + +func (p *PixivClient) GetUserInformation(id string) (*models.User, error) { + s, _ := p.TextRequest(fmt.Sprintf(UserInformationURL, id)) + + var user *models.User + var pr models.PixivResponse + + err := json.Unmarshal([]byte(s), &pr) + + if err != nil { + return nil, errors.New(fmt.Sprintf("Failed to extract data from JSON response from server. %s", err)) + } + if pr.Error { + return nil, errors.New(pr.Message) + } + + // Basic user information + err = json.Unmarshal([]byte(pr.Body), &user) + + // Artworks + works, _ := p.GetUserArtworks(id) + user.Artworks = works + + return user, nil +} diff --git a/main.go b/main.go index b45b196..c1a3bb1 100644 --- a/main.go +++ b/main.go @@ -33,7 +33,6 @@ func setupRouter() *gin.Engine { func main() { configs.Configs.ReadConfig() - println(configs.Configs.PHPSESSID) r := setupRouter() diff --git a/models/models.go b/models/models.go new file mode 100644 index 0000000..cea8ac4 --- /dev/null +++ b/models/models.go @@ -0,0 +1,120 @@ +package models + +import ( + "html/template" + "time" + + "encoding/json" +) + +type PixivResponse struct { + Error bool + Message string + Body json.RawMessage +} + +type ImageResponse struct { + Urls map[string]string `json:"urls"` +} + +type TagResponse struct { + AuthorID string `json:"authorId"` + RawTags json.RawMessage `json:"tags"` +} + +// Pixiv returns 0, 1, 2 to filter SFW and/or NSFW artworks. +// Those values are saved in `xRestrict` +// 0: Safe +// 1: R18 +// 2: R18G +type xRestrict int + +const ( + Safe xRestrict = 0 + R18 xRestrict = 1 + R18G xRestrict = 2 +) + +var xRestrictModel = map[xRestrict]string{ + Safe: "Safe", + R18: "R18", + R18G: "R18G", +} + +// Pixiv returns 0, 1, 2 to filter SFW and/or NSFW artworks. +// Those values are saved in `aiType` +// 0: Not rated / Unknown +// 1: Not AI-generated +// 2: AI-generated + +type aiType int + +const ( + Unrated aiType = 0 + NotAI aiType = 1 + AI aiType = 2 +) + +var aiTypeModel = map[aiType]string{ + Unrated: "Unrated", + NotAI: "NotAI", + AI: "AI", +} + +// Pixiv gives us 5 types of an image. I don't need the mini one tho. +// PS: Where tf is my 360x360 image, Pixiv? :rage: +type Image struct { + Small string `json:"thumb_mini"` + Medium string `json:"small"` + Large string `json:"regular"` + Original string `json:"original"` +} + +type Tag struct { + Name string `json:"tag"` + TranslatedName string `json:"translation"` +} + +type Illust struct { + ID string `json:"id"` + Title string `json:"title"` + Description template.HTML `json:"description"` + UserID string `json:"userId"` + UserName string `json:"userName"` + UserAccount string `json:"userAccount"` + Date time.Time `json:"uploadDate"` + Images []Image `json:"images"` + Tags []Tag `json:"tags"` + Pages int `json:"pageCount"` + Bookmarks int `json:"bookmarkCount"` + Likes int `json:"likeCount"` + Comments int `json:"commentCount"` + Views int `json:"viewCount"` + XRestrict xRestrict `json:"xRestrict"` + AiType aiType `json:"aiType"` +} + +type IllustShort struct { + ID string `json:"id"` + Title string `json:"title"` + Description template.HTML `json:"description"` + ArtistID string `json:"userId"` + ArtistName string `json:"userName"` + ArtistAvatar string `json:"profileImageUrl"` + Date time.Time `json:"uploadDate"` + Thumbnail string `json:"url"` + Pages int `json:"pageCount"` + XRestrict xRestrict `json:"xRestrict"` + AiType aiType `json:"aiType"` +} + +type User struct { + ID string `json:"userId"` + Name string `json:"name"` + Avatar string `json:"image"` + BackgroundImage string `json:"background"` + Following int `json:"following"` + MyPixiv int `json:"mypixivCount"` + Comment string `json:"comment"` + Artworks []IllustShort `json:"artworks"` +} diff --git a/template/artwork.html b/template/artwork.html index 009f322..60a9874 100644 --- a/template/artwork.html +++ b/template/artwork.html @@ -12,7 +12,7 @@ {{ end }}

{{ .Title }}

-

{{ .Caption }}

+

{{ .Description }}

@@ -26,15 +26,14 @@
{{ .Date }} - {{ .Artist.Name }} - {{ .Artist.Name }} + {{ $parent.Artist.Name }} + {{ $parent.Artist.Name }}
{{ range $parent.Recent }}
- {{ (index .Images 0).Small }} + {{ .Thumbnail }}
{{ end }} diff --git a/template/large-tn.html b/template/large-tn.html index fd5ce8b..e06fe3e 100644 --- a/template/large-tn.html +++ b/template/large-tn.html @@ -2,14 +2,23 @@
{{ $i | inc}}
- {{ .Title }} + {{ .Title }}

{{ .Title }}

 - {{ .Artist.Name }} - {{ .Artist.Name }} + {{ .ArtistName }} + {{ .ArtistName }}
{{ end }} diff --git a/template/small-tn.html b/template/small-tn.html index 0b1ed6a..b0d695e 100644 --- a/template/small-tn.html +++ b/template/small-tn.html @@ -1,14 +1,23 @@ {{ range . }}
- {{ .Title }} + {{ .Title }}

{{ .Title }}

 - {{ .Artist.Name }} - {{ .Artist.Name }} + {{ .ArtistName }} + {{ .ArtistName }}
{{ end }} diff --git a/template/user.html b/template/user.html index ef177cc..f855f73 100644 --- a/template/user.html +++ b/template/user.html @@ -5,10 +5,11 @@
- avatar + avatar

{{ .User.Name }}

-

@{{ .User.Account }}

-

{{ .User.Followers }} Following | {{ .User.MyPixiv }} MyPixiv

+

+ {{ .User.Following }} Following | {{ .User.MyPixiv }} MyPixiv +

diff --git a/views/routes.go b/views/routes.go index 8dc9309..e9d8045 100644 --- a/views/routes.go +++ b/views/routes.go @@ -1,44 +1,80 @@ package views import ( - "github.com/gin-gonic/gin" "net/http" + "pixivfe/configs" "pixivfe/handler" - "strconv" + "time" + + "github.com/gin-gonic/gin" ) +var PC *models.PixivClient + func artwork_page(c *gin.Context) { - illust, _ := handler.GetIllustByID(c) - related, _ := handler.GetRelatedIllust(c) - recent_by_artist, _ := handler.GetMemberIllust(c, strconv.Itoa(illust.Artist.ID)) + id := c.Param("id") + illust, _ := PC.GetArtworkByID(id) + related, _ := PC.GetRelatedArtworks(id) + recent_by_artist, _ := PC.GetUserArtworks(illust.UserID) + artist_info, _ := PC.GetUserInformation(illust.UserID) + c.HTML(http.StatusOK, "artwork.html", gin.H{ "Illust": illust, "Related": related, "Recent": recent_by_artist, + "Artist": artist_info, }) } -func index_page(c *gin.Context) { - recommended, _ := handler.GetRecommendedIllust(c) - ranking, _ := handler.GetRankingIllust(c, "day") - spotlight := handler.GetSpotlightArticle(c) - newest, _ := handler.GetNewestIllust(c) - c.HTML(http.StatusOK, "index.html", gin.H{ - "Recommended": recommended, - "Rankings": ranking, - "Spotlights": spotlight, - "Newest": newest, - }) -} +// func index_page(c *gin.Context) { +// recommended, _ := handler.GetRecommendedIllust(c) +// ranking, _ := handler.GetRankingIllust(c, "day") +// spotlight := handler.GetSpotlightArticle(c) +// newest, _ := handler.GetNewestIllust(c) +// c.HTML(http.StatusOK, "index.html", gin.H{ +// "Recommended": recommended, +// "Rankings": ranking, +// "Spotlights": spotlight, +// "Newest": newest, +// }) +// } func user_page(c *gin.Context) { - user, _ := handler.GetUserInfo(c) - recent, _ := handler.GetMemberIllust(c, c.Param("id")) + id := c.Param("id") + user, _ := PC.GetUserInformation(id) + recent, _ := PC.GetUserArtworks(id) c.HTML(http.StatusOK, "user.html", gin.H{"User": user, "Recent": recent}) } -func SetupRoutes(r *gin.Engine) { - r.GET("/", index_page) - r.GET("artworks/:id", artwork_page) - r.GET("user/:id", user_page) +func getUserInformation(c *gin.Context) { + id := c.Param("id") + data, _ := PC.GetUserInformation(id) + + c.IndentedJSON(http.StatusOK, data) +} + +func NewPixivClient(timeout int) *models.PixivClient { + transport := &http.Transport{Proxy: http.ProxyFromEnvironment} + client := &http.Client{ + Timeout: time.Duration(timeout) * time.Millisecond, + Transport: transport, + } + + pc := &models.PixivClient{ + Client: client, + Header: make(map[string]string), + Cookie: make(map[string]string), + Lang: "en", + } + + return pc +} + +func SetupRoutes(r *gin.Engine) { + PC = NewPixivClient(5000) + PC.SetSessionID(configs.Configs.PHPSESSID) + PC.SetUserAgent(configs.Configs.UserAgent) + // r.GET("/", index_page) + r.GET("artworks/:id", artwork_page) + r.GET("users/:id", user_page) }