From 52fdb0fd511ca1d65047e860198749f158997ac9 Mon Sep 17 00:00:00 2001 From: perennial Date: Fri, 25 Oct 2024 19:32:09 +1100 Subject: [PATCH] tag search by popular, experimental update frontend for popular search add PIXIVFE_POPULAR_SEARCH_ENABLED option --- .env.example | 5 +- assets/views/tag.jet.html | 40 ++++---- config/config.go | 12 +++ core/artwork.go | 23 ++--- core/endpoints.go | 50 +++++++++- core/popular_search.go | 138 +++++++++++++++++++++++++++ core/tag.go | 34 +++++++ doc/hosting/environment-variables.md | 15 +++ i18n/locale/en/code.json | 1 + i18n/locale/en/template.json | 7 ++ server/routes/tag.go | 24 ++--- server/routes/types.go | 23 ++--- 12 files changed, 320 insertions(+), 52 deletions(-) create mode 100644 core/popular_search.go diff --git a/.env.example b/.env.example index f88ca3b..c807ad9 100644 --- a/.env.example +++ b/.env.example @@ -29,7 +29,10 @@ PIXIVFE_HOST='127.0.0.1' # PIXIVFE_CACHE_ENABLED= # PIXIVFE_CACHE_SIZE= # PIXIVFE_CACHE_TTL= -# PIXIVFE_CACHE_SHORT_TTL +# PIXIVFE_CACHE_SHORT_TTL= + +### Feature configuration +# PIXIVFE_POPULAR_SEARCH_ENABLED= ### Development options # PIXIVFE_DEV= diff --git a/assets/views/tag.jet.html b/assets/views/tag.jet.html index 9deb344..71468be 100644 --- a/assets/views/tag.jet.html +++ b/assets/views/tag.jet.html @@ -166,10 +166,30 @@
-
- -
+
+

Order

+ {{- if .PopularSearchEnabled == true }} + {{- _Order := isset(.Queries.order) ? .Queries.order : "date_d" }} + {{- URL := unfinishedQuery(.QueriesC, "order") }} + {{- path := slice("date_d", "date", "popular") }} + {{- name := slice("Newest", "Oldest", "Popular") }} + {{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveOrder) }} +
+ The "Popular" sorting uses users入り bookmark count tags to show the most-liked works, but is limited to one page and specific milestones. If no results appear, the tag likely lacks bookmark data. Try broader tags like #アイマス instead of #アイドルマスターシャイニーカラーズ for better results. +
+ {{- else }} + {{- _Order := isset(.Queries.order) ? .Queries.order : "date_d" }} + {{- URL := unfinishedQuery(.QueriesC, "order") }} + {{- path := slice("date_d", "date") }} + {{- name := slice("Newest", "Oldest") }} + {{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveOrder) }} + {{- end }} +
+
+ +
+

Ratio

{{- _Ratio := isset(.Queries.ratio) ? .Queries.ratio : "" }} {{- URL := unfinishedQuery(.QueriesC, "ratio") }} @@ -177,22 +197,10 @@ {{- name := slice("All", "Portrait", "Square", "Landscape") }} {{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveRatio) }}
- - -
-

Order

- {{- _Order := isset(.Queries.order) ? .Queries.order : "date_d" }} - {{- URL := unfinishedQuery(.QueriesC, "order") }} - {{- path := slice("date_d", "date") }} - {{- name := slice("Newest", "Oldest") }} - {{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveOrder) }} -
-
- -
+

Search mode

{{- SearchMode := isset(.Queries.smode) ? .Queries.smode : "" }} {{- URL := unfinishedQuery(.QueriesC, "smode") }} diff --git a/config/config.go b/config/config.go index cab6031..15d6d95 100644 --- a/config/config.go +++ b/config/config.go @@ -45,6 +45,7 @@ const ( defaultCacheSize = 100 defaultCacheTTL = 60 * time.Minute defaultCacheShortTTL = 10 * time.Second + defaultPopularSearchEnabled = false defaultResponseSaveLocation = "/tmp/pixivfe/responses" defaultLogLevel = "info" defaultLogFormat = "console" @@ -91,6 +92,9 @@ type ServerConfig struct { CacheTTL time.Duration `env:"PIXIVFE_CACHE_TTL,overwrite"` CacheShortTTL time.Duration `env:"PIXIVFE_CACHE_SHORT_TTL,overwrite"` + // Feature configuration + PopularSearchEnabled bool `env:"PIXIVFE_POPULAR_SEARCH_ENABLED,overwrite"` + // Development options InDevelopment bool `env:"PIXIVFE_DEV"` ResponseSaveLocation string `env:"PIXIVFE_RESPONSE_SAVE_LOCATION,overwrite"` @@ -161,6 +165,8 @@ func (s *ServerConfig) LoadConfig() error { s.CacheTTL = defaultCacheTTL s.CacheShortTTL = defaultCacheShortTTL + s.PopularSearchEnabled = defaultPopularSearchEnabled + s.ResponseSaveLocation = defaultResponseSaveLocation s.LogLevel = defaultLogLevel @@ -240,6 +246,12 @@ func (s *ServerConfig) LoadConfig() error { log.Println("API response cache is disabled") } + if s.PopularSearchEnabled { + log.Println("Tag search by popularity is enabled") + } else { + log.Println("Tag search by popularity is disabled") + } + // Only print ResponseSaveLocation if InDevelopment is set if s.InDevelopment { log.Printf("Response save location: %s\n", s.ResponseSaveLocation) diff --git a/core/artwork.go b/core/artwork.go index 1999f09..0254cc1 100644 --- a/core/artwork.go +++ b/core/artwork.go @@ -107,17 +107,18 @@ type UserBrief struct { } type ArtworkBrief struct { - ID string `json:"id"` - Title string `json:"title"` - ArtistID string `json:"userId"` - ArtistName string `json:"userName"` - ArtistAvatar string `json:"profileImageUrl"` - Thumbnail string `json:"url"` - Pages int `json:"pageCount"` - XRestrict int `json:"xRestrict"` - AiType int `json:"aiType"` - Bookmarked any `json:"bookmarkData"` - IllustType int `json:"illustType"` + ID string `json:"id"` + Title string `json:"title"` + ArtistID string `json:"userId"` + ArtistName string `json:"userName"` + ArtistAvatar string `json:"profileImageUrl"` + Thumbnail string `json:"url"` + Pages int `json:"pageCount"` + XRestrict int `json:"xRestrict"` + AiType int `json:"aiType"` + Bookmarked any `json:"bookmarkData"` + IllustType int `json:"illustType"` + Tags []string `json:"tags"` // used by core/popular_search } type Illust struct { diff --git a/core/endpoints.go b/core/endpoints.go index b21a395..e448441 100644 --- a/core/endpoints.go +++ b/core/endpoints.go @@ -118,9 +118,55 @@ func GetTagDetailURL(unescapedTag string) string { func GetSearchArtworksURL(s map[string]string) string { // Long. - base := "https://www.pixiv.net/ajax/search/%s/%s?order=%s&mode=%s&ratio=%s&s_mode=%s&wlt=%s&wgt=%s&hlt=%s&hgt=%s&tool=%s&scd=%s&ecd=%s&p=%s" + base := "https://www.pixiv.net/ajax/search/%s/%s" - return fmt.Sprintf(base, s["Category"], s["Name"], s["Order"], s["Mode"], s["Ratio"], s["Smode"], s["Wlt"], s["Wgt"], s["Hlt"], s["Hgt"], s["Tool"], s["Scd"], s["Ecd"], s["Page"]) + // URL-encode the category and name + category := url.PathEscape(s["Category"]) + name := url.PathEscape(s["Name"]) + + // Base URL + baseURL := fmt.Sprintf(base, category, name) + + // Build the query parameters + params := url.Values{} + if s["Order"] != "" { + params.Add("order", s["Order"]) + } + if s["Mode"] != "" { + params.Add("mode", s["Mode"]) + } + if s["Ratio"] != "" { + params.Add("ratio", s["Ratio"]) + } + if s["Smode"] != "" { + params.Add("s_mode", s["Smode"]) + } + if s["Wlt"] != "" { + params.Add("wlt", s["Wlt"]) + } + if s["Wgt"] != "" { + params.Add("wgt", s["Wgt"]) + } + if s["Hlt"] != "" { + params.Add("hlt", s["Hlt"]) + } + if s["Hgt"] != "" { + params.Add("hgt", s["Hgt"]) + } + if s["Tool"] != "" { + params.Add("tool", s["Tool"]) + } + if s["Scd"] != "" { + params.Add("scd", s["Scd"]) + } + if s["Ecd"] != "" { + params.Add("ecd", s["Ecd"]) + } + if s["Page"] != "" { + params.Add("p", s["Page"]) + } + + return baseURL + "?" + params.Encode() } func GetLandingURL(mode string) string { diff --git a/core/popular_search.go b/core/popular_search.go new file mode 100644 index 0000000..d047bf2 --- /dev/null +++ b/core/popular_search.go @@ -0,0 +1,138 @@ +package core + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sort" + "strconv" + "strings" + + "codeberg.org/vnpower/pixivfe/v2/audit" + "codeberg.org/vnpower/pixivfe/v2/server/session" +) + +// Constants for result limits +const ( + resultsLimit = 100 +) + +// Popularity thresholds +var suffixes = []string{"100000", "50000", "30000", "10000", "5000", "1000", "500", "100", "50"} + +// SearchResponse represents the JSON structure returned by the Pixiv API for the search endpoint +type SearchResponse struct { + IllustManga struct { + Data []ArtworkBrief `json:"data"` + } `json:"illustManga"` +} + +// isValidResult checks if an artwork meets the search criteria +func isValidResult(item ArtworkBrief, category string) bool { + hasFakeTag := strings.Contains(strings.Join(item.Tags, " "), "虚偽users入りタグ") + isCorrectMode := (category == "artworks" && (item.IllustType == 0 || item.IllustType == 1)) || + (category == "illustrations" && item.IllustType == 0) || + (category == "manga" && item.IllustType == 1) + return !hasFakeTag && isCorrectMode +} + +// processSearchResults filters and adds valid results to the artworks slice +func processSearchResults(items []ArtworkBrief, category string, artworks *[]ArtworkBrief) bool { + for _, item := range items { + if isValidResult(item, category) { + *artworks = append(*artworks, item) + if len(*artworks) >= resultsLimit { + return true + } + } + } + return false +} + +// fetchResultsForSuffix retrieves all results for a specific popularity threshold +func fetchResultsForSuffix(ctx context.Context, auditor *audit.Auditor, r *http.Request, settings SearchPageSettings, suffix string, artworks *[]ArtworkBrief) error { + page := 1 + + for len(*artworks) < resultsLimit { + // Copy settings to avoid modifying the original + currentSettings := settings + + // Adjust settings.Name to include the suffix + currentSettings.Name = fmt.Sprintf("%s%susers入り", settings.Name, suffix) + + // Set page number + currentSettings.Page = strconv.Itoa(page) + + // Generate URL + url := GetSearchArtworksURL(currentSettings.ReturnMap()) + + auditor.SugaredLogger.Debugw("Fetching search results", "url", url) + + body, err := API_GET_UnwrapJson(ctx, auditor, url, "", r.Header) + if err != nil { + auditor.SugaredLogger.Errorw("Error fetching results", "error", err, "suffix", suffix, "page", page) + return err + } + + proxiedBody, err := session.ProxyImageUrl(r, body) + if err != nil { + auditor.SugaredLogger.Errorw("Error proxying image URLs", "error", err) + return err + } + + // Unmarshal the proxied JSON body into SearchResponse + var resp SearchResponse + err = json.Unmarshal([]byte(proxiedBody), &resp) + if err != nil { + auditor.SugaredLogger.Errorw("Failed to unmarshal JSON", "error", err, "body", proxiedBody) + return err + } + + if len(resp.IllustManga.Data) == 0 { + auditor.SugaredLogger.Debugw("No more results found", "suffix", suffix, "page", page) + break + } + + reachedLimit := processSearchResults(resp.IllustManga.Data, currentSettings.Category, artworks) + if reachedLimit { + return nil + } + + page++ + auditor.SugaredLogger.Debugw("Moving to next page", "suffix", suffix, "page", page) + } + return nil +} + +// searchPopular performs a search across all popularity thresholds +func searchPopular(ctx context.Context, auditor *audit.Auditor, r *http.Request, settings SearchPageSettings, headers http.Header) (SearchArtworks, error) { + auditor.SugaredLogger.Infow("Starting popular search", "query", settings.Name, "mode", settings.Category) + var artworks []ArtworkBrief + + for _, suffix := range suffixes { + auditor.SugaredLogger.Debugw("Searching with suffix", "suffix", suffix) + err := fetchResultsForSuffix(ctx, auditor, r, settings, suffix, &artworks) + if err != nil { + auditor.SugaredLogger.Errorw("Error fetching results for suffix", "suffix", suffix, "error", err) + continue + } + + if len(artworks) >= resultsLimit { + break + } + } + + total := len(artworks) + auditor.SugaredLogger.Infow("Completed popular search", "query", settings.Name, "mode", settings.Category, "resultsCount", total) + + // Sort the results by ID in descending order since higher IDs are newer + sort.Slice(artworks, func(i, j int) bool { + return artworks[i].ID > artworks[j].ID + }) + + return SearchArtworks{ + Artworks: artworks, + Total: total, + }, nil +} diff --git a/core/tag.go b/core/tag.go index 2e94392..4b74423 100644 --- a/core/tag.go +++ b/core/tag.go @@ -7,6 +7,8 @@ import ( "strings" "codeberg.org/vnpower/pixivfe/v2/audit" + "codeberg.org/vnpower/pixivfe/v2/config" + "codeberg.org/vnpower/pixivfe/v2/i18n" "codeberg.org/vnpower/pixivfe/v2/server/session" "github.com/goccy/go-json" ) @@ -138,7 +140,38 @@ func GetTagData(auditor *audit.Auditor, r *http.Request, name string) (TagDetail return tag, nil } +// GetSearch delegates the search operation to either getPopularSearch or getStandardSearch based on settings.Order. func GetSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSettings) (*SearchResult, error) { + if strings.ToLower(settings.Order) == "popular" { + return getPopularSearch(auditor, r, settings) + } + return getStandardSearch(auditor, r, settings) +} + +// getPopularSearch handles the popular search logic. +func getPopularSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSettings) (*SearchResult, error) { + // Check if popular search is enabled + if !config.GlobalConfig.PopularSearchEnabled { + return nil, i18n.Errorf("Popular search is disabled by server configuration.") + } + + // Perform popular search + searchArtworks, err := searchPopular(r.Context(), auditor, r, settings, r.Header) + if err != nil { + return nil, err + } + + // Create SearchResult + result := &SearchResult{ + Artworks: searchArtworks, + // TODO: populate Popular (the regular one) and RelatedTags + } + + return result, nil +} + +// getStandardSearch handles the standard search logic. +func getStandardSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSettings) (*SearchResult, error) { URL := GetSearchArtworksURL(settings.ReturnMap()) resp, err := API_GET_UnwrapJson(r.Context(), auditor, URL, "", r.Header) @@ -150,6 +183,7 @@ func GetSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSetti if err != nil { return nil, err } + // Replace "illust", "manga", and "illustManga" with "works" in the JSON response // // VnPower: IDK how to do better than this lol diff --git a/doc/hosting/environment-variables.md b/doc/hosting/environment-variables.md index f0c40d9..62cd22d 100644 --- a/doc/hosting/environment-variables.md +++ b/doc/hosting/environment-variables.md @@ -215,6 +215,21 @@ Specifies a shorter Time To Live (TTL) for specific API endpoints that change fr These paths are considered more dynamic, and the shorter TTL ensures more up-to-date content. +## Feature configuration + +### `PIXIVFE_POPULAR_SEARCH_ENABLED` + +**Required**: No + +**Default:** `false` + +Controls whether searching by popularity for a given tag is enabled. + +!!! warning + This feature requires several API calls for each search, which may lead to rate limiting by the Pixiv API. + + As such, enabling caching (`PIXIVFE_CACHE_ENABLED=true`) is recommended to minimize repeated requests. + ## Network proxy configuration Used to set the [proxy server](https://en.wikipedia.org/wiki/Proxy_server) that PixivFE will use for all requests. Not to be confused with the image proxy, which is used to comply with the `Referer` check required by `i.pximg.net`. diff --git a/i18n/locale/en/code.json b/i18n/locale/en/code.json index 4707156..fc993ce 100644 --- a/i18n/locale/en/code.json +++ b/i18n/locale/en/code.json @@ -30,6 +30,7 @@ "core/requests.go:fEaOk30bc9I": "failed to read response body: %w", "core/requests.go:lLy9SHFUtQQ": "failed to copy response body: %w", "core/requests.go:o8eDPgojH4w": "Incompatible response body", + "core/tag.go:LI4dIQHgYP4": "Popular search is disabled by server configuration.", "core/user.go:T3RwaHcnsYQ": "Invalid work category: %#v. Only \"%s\", \"%s\", \"%s\", \"%s\", \"%s\" and \"%s\" are available", "server/router/router.go:HORkE0Obn1U": "Failed to redirect to %s", "server/router/router.go:Z9UG-DYmQCk": "Route not found", diff --git a/i18n/locale/en/template.json b/i18n/locale/en/template.json index 9df7827..a735177 100644 --- a/i18n/locale/en/template.json +++ b/i18n/locale/en/template.json @@ -531,11 +531,14 @@ "assets/views/tag.jet.html:5gLsaoIn1hw": "Newest", "assets/views/tag.jet.html:8R6wargzfWI": "Square", "assets/views/tag.jet.html:8lMwxcLFwfQ": "R-18", + "assets/views/tag.jet.html:9KmARXlO47Y": "#アイマス", "assets/views/tag.jet.html:DJ9HQmApqX4": "Maximum image width", "assets/views/tag.jet.html:FSdckWFtaaQ": "works", "assets/views/tag.jet.html:FpuEclq1XG4": "Maximum image height", "assets/views/tag.jet.html:L1ef_2BFV8U": "Reset filters", + "assets/views/tag.jet.html:LbYiwrOSQuo": "bookmark count tags to show the most-liked works, but is limited to one page and specific milestones. If no results appear, the tag likely lacks bookmark data. Try broader tags like", "assets/views/tag.jet.html:LlTlrHqgCXI": "Partial (tags)", + "assets/views/tag.jet.html:MA-a3uM1m7s": "#アイドルマスターシャイニーカラーズ", "assets/views/tag.jet.html:O8pAU2K_Lco": "Title/Caption", "assets/views/tag.jet.html:Obw6P7wVCaA": "Tool", "assets/views/tag.jet.html:P6fO_UTqr7E": "Manga", @@ -544,8 +547,10 @@ "assets/views/tag.jet.html:R-DIbx-3Pww": "Advanced settings", "assets/views/tag.jet.html:RcfE_VBY72s": "Set", "assets/views/tag.jet.html:Sccoo133FQI": "All", + "assets/views/tag.jet.html:XS2z_BaQ4Tw": "users入り", "assets/views/tag.jet.html:ZqXoa4qWM54": "Recent", "assets/views/tag.jet.html:_HgJq4ezzIg": "Posted after (format: yyyy-mm-dd)", + "assets/views/tag.jet.html:cL47N2weMW8": "for better results.", "assets/views/tag.jet.html:chTuQxGY_sk": "Popular works", "assets/views/tag.jet.html:gEYVuWDipug": "Illustrations", "assets/views/tag.jet.html:gwWDcxEUA58": "Ratio", @@ -555,8 +560,10 @@ "assets/views/tag.jet.html:qPuGKC6JtSY": "Exact (tags)", "assets/views/tag.jet.html:qepcz_2ar6E": "Artworks", "assets/views/tag.jet.html:tLDBgUWw9-g": "Safe", + "assets/views/tag.jet.html:v0F5PrG9DyA": "instead of", "assets/views/tag.jet.html:vQknITzlB0I": "Search options", "assets/views/tag.jet.html:vUR6oAGwbwE": "Order", + "assets/views/tag.jet.html:wsrqIAla6TE": "The \"Popular\" sorting uses", "assets/views/tag.jet.html:x-I1EHIg1-o": "All time", "assets/views/tag.jet.html:xd9cjh9m0Go": "({{ .TranslatedName }})", "assets/views/tag.jet.html:yHft19aQloM": "Minimum image width", diff --git a/server/routes/tag.go b/server/routes/tag.go index 65b2dde..341e9dc 100644 --- a/server/routes/tag.go +++ b/server/routes/tag.go @@ -6,6 +6,7 @@ import ( "strconv" "codeberg.org/vnpower/pixivfe/v2/audit" + "codeberg.org/vnpower/pixivfe/v2/config" "codeberg.org/vnpower/pixivfe/v2/core" "codeberg.org/vnpower/pixivfe/v2/server/template" "codeberg.org/vnpower/pixivfe/v2/server/utils" @@ -60,17 +61,18 @@ func TagPage(auditor *audit.Auditor, w http.ResponseWriter, r *http.Request) err PreloadImage(w, tag.Metadata.ImageMaster) return RenderHTML(w, r, Data_tag{ - Title: "Results for " + name, - Tag: tag, - Data: *result, - QueriesC: urlc, - TrueTag: param, - Page: pageInt, - ActiveCategory: queries.Category, - ActiveOrder: queries.Order, - ActiveMode: queries.Mode, - ActiveRatio: queries.Ratio, - ActiveSearchMode: GetQueryParam(r, "smode", ""), + Title: "Results for " + name, + Tag: tag, + Data: *result, + QueriesC: urlc, + TrueTag: param, + Page: pageInt, + ActiveCategory: queries.Category, + ActiveOrder: queries.Order, + ActiveMode: queries.Mode, + ActiveRatio: queries.Ratio, + ActiveSearchMode: GetQueryParam(r, "smode", ""), + PopularSearchEnabled: config.GlobalConfig.PopularSearchEnabled, }) } diff --git a/server/routes/types.go b/server/routes/types.go index ce11677..347d8e0 100644 --- a/server/routes/types.go +++ b/server/routes/types.go @@ -140,17 +140,18 @@ type Data_settings struct { DefaultProxyServer string } type Data_tag struct { - Title string - Tag core.TagDetail - Data core.SearchResult - QueriesC template.PartialURL - TrueTag string - Page int - ActiveCategory string - ActiveOrder string - ActiveMode string - ActiveRatio string - ActiveSearchMode string + Title string + Tag core.TagDetail + Data core.SearchResult + QueriesC template.PartialURL + TrueTag string + Page int + ActiveCategory string + ActiveOrder string + ActiveMode string + ActiveRatio string + ActiveSearchMode string + PopularSearchEnabled bool } type Data_user struct { Title string