VnPower/PixivFE
1
0
Fork 0
mirror of https://codeberg.org/VnPower/PixivFE synced 2024-12-06 19:16:23 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

tag search by popular, experimental

Browse Source 
update frontend for popular search

add PIXIVFE_POPULAR_SEARCH_ENABLED option
This commit is contained in:
perennial
2024-10-25 19:32:09 +11:00
parent a224361f4f
commit 52fdb0fd51
12 changed files with 320 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+4 -1
View File
@@ -29,7 +29,10 @@ PIXIVFE_HOST='127.0.0.1'
# PIXIVFE_CACHE_ENABLED=
# PIXIVFE_CACHE_SIZE=
# PIXIVFE_CACHE_TTL=
# PIXIVFE_CACHE_SHORT_TTL
# PIXIVFE_CACHE_SHORT_TTL=
### Feature configuration
# PIXIVFE_POPULAR_SEARCH_ENABLED=
### Development options
# PIXIVFE_DEV=
assets/views/tag.jet.html
+24 -16
View File
@@ -166,10 +166,30 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body p-0">
<!-- The first row represents Ratio and Order -->
<div class="row">
<!-- Ratio: Second in small screens, but third in large screens -->
<div class="col-lg-7 mb-5">
<div class="col-12 mb-4">
<h3><i class="bi bi-sort-down me-2"></i>Order</h3>
{{- if .PopularSearchEnabled == true }}
{{- _Order := isset(.Queries.order) ? .Queries.order : "date_d" }}
{{- URL := unfinishedQuery(.QueriesC, "order") }}
{{- path := slice("date_d", "date", "popular") }}
{{- name := slice("Newest", "Oldest", "Popular") }}
{{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveOrder) }}
<div class="text-body-secondary rounded-end-5 border border-5 border-top-0 border-end-0 border-bottom-0 border-custom-color bg-neutral-800 ps-2 py-2 mt-3 mb-0">
The "Popular" sorting uses <code>users入り</code> bookmark count tags to show the most-liked works, but is limited to one page and specific milestones. If no results appear, the tag likely lacks bookmark data. Try broader tags like <code>#アイマス</code> instead of <code>#アイドルマスターシャイニーカラーズ</code> for better results.
</div>
{{- else }}
{{- _Order := isset(.Queries.order) ? .Queries.order : "date_d" }}
{{- URL := unfinishedQuery(.QueriesC, "order") }}
{{- path := slice("date_d", "date") }}
{{- name := slice("Newest", "Oldest") }}
{{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveOrder) }}
{{- end }}
</div>
</div>
<div class="row">
<div class="col-12 mb-4">
<h3><i class="bi bi-aspect-ratio-fill me-2"></i>Ratio</h3>
{{- _Ratio := isset(.Queries.ratio) ? .Queries.ratio : "" }}
{{- URL := unfinishedQuery(.QueriesC, "ratio") }}
@@ -177,22 +197,10 @@
{{- name := slice("All", "Portrait", "Square", "Landscape") }}
{{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveRatio) }}
</div>
<!-- Order: Fourth in both large and small screens -->
<div class="col-lg-5 mb-5">
<h3><i class="bi bi-sort-down me-2"></i>Order</h3>
{{- _Order := isset(.Queries.order) ? .Queries.order : "date_d" }}
{{- URL := unfinishedQuery(.QueriesC, "order") }}
{{- path := slice("date_d", "date") }}
{{- name := slice("Newest", "Oldest") }}
{{- yield UnderlineNav(baseURL=URL, paths=path, names=name, activeState=.ActiveOrder) }}
</div>
</div>
<!-- The second row represents Search Mode which is always last -->
<div class="row">
<!-- Search Mode: Always last on both large and small screens -->
<div class="col-lg-12 mb-5">
<div class="col-12 mb-4">
<h3><i class="bi bi-search me-2"></i>Search mode</h3>
{{- SearchMode := isset(.Queries.smode) ? .Queries.smode : "" }}
{{- URL := unfinishedQuery(.QueriesC, "smode") }}
config/config.go
+12
View File
@@ -45,6 +45,7 @@ const (
defaultCacheSize = 100
defaultCacheTTL = 60 * time.Minute
defaultCacheShortTTL = 10 * time.Second
defaultPopularSearchEnabled = false
defaultResponseSaveLocation = "/tmp/pixivfe/responses"
defaultLogLevel = "info"
defaultLogFormat = "console"
@@ -91,6 +92,9 @@ type ServerConfig struct {
CacheTTL time.Duration `env:"PIXIVFE_CACHE_TTL,overwrite"`
CacheShortTTL time.Duration `env:"PIXIVFE_CACHE_SHORT_TTL,overwrite"`
// Feature configuration
PopularSearchEnabled bool `env:"PIXIVFE_POPULAR_SEARCH_ENABLED,overwrite"`
// Development options
InDevelopment bool `env:"PIXIVFE_DEV"`
ResponseSaveLocation string `env:"PIXIVFE_RESPONSE_SAVE_LOCATION,overwrite"`
@@ -161,6 +165,8 @@ func (s *ServerConfig) LoadConfig() error {
s.CacheTTL = defaultCacheTTL
s.CacheShortTTL = defaultCacheShortTTL
s.PopularSearchEnabled = defaultPopularSearchEnabled
s.ResponseSaveLocation = defaultResponseSaveLocation
s.LogLevel = defaultLogLevel
@@ -240,6 +246,12 @@ func (s *ServerConfig) LoadConfig() error {
log.Println("API response cache is disabled")
}
if s.PopularSearchEnabled {
log.Println("Tag search by popularity is enabled")
} else {
log.Println("Tag search by popularity is disabled")
}
// Only print ResponseSaveLocation if InDevelopment is set
if s.InDevelopment {
log.Printf("Response save location: %s\n", s.ResponseSaveLocation)
core/artwork.go
+12 -11
View File
@@ -107,17 +107,18 @@ type UserBrief struct {
}
type ArtworkBrief struct {
ID string `json:"id"`
Title string `json:"title"`
ArtistID string `json:"userId"`
ArtistName string `json:"userName"`
ArtistAvatar string `json:"profileImageUrl"`
Thumbnail string `json:"url"`
Pages int `json:"pageCount"`
XRestrict int `json:"xRestrict"`
AiType int `json:"aiType"`
Bookmarked any `json:"bookmarkData"`
IllustType int `json:"illustType"`
ID string `json:"id"`
Title string `json:"title"`
ArtistID string `json:"userId"`
ArtistName string `json:"userName"`
ArtistAvatar string `json:"profileImageUrl"`
Thumbnail string `json:"url"`
Pages int `json:"pageCount"`
XRestrict int `json:"xRestrict"`
AiType int `json:"aiType"`
Bookmarked any `json:"bookmarkData"`
IllustType int `json:"illustType"`
Tags []string `json:"tags"` // used by core/popular_search
}
type Illust struct {
core/endpoints.go
+48 -2
View File
@@ -118,9 +118,55 @@ func GetTagDetailURL(unescapedTag string) string {
func GetSearchArtworksURL(s map[string]string) string {
// Long.
base := "https://www.pixiv.net/ajax/search/%s/%s?order=%s&mode=%s&ratio=%s&s_mode=%s&wlt=%s&wgt=%s&hlt=%s&hgt=%s&tool=%s&scd=%s&ecd=%s&p=%s"
base := "https://www.pixiv.net/ajax/search/%s/%s"
return fmt.Sprintf(base, s["Category"], s["Name"], s["Order"], s["Mode"], s["Ratio"], s["Smode"], s["Wlt"], s["Wgt"], s["Hlt"], s["Hgt"], s["Tool"], s["Scd"], s["Ecd"], s["Page"])
// URL-encode the category and name
category := url.PathEscape(s["Category"])
name := url.PathEscape(s["Name"])
// Base URL
baseURL := fmt.Sprintf(base, category, name)
// Build the query parameters
params := url.Values{}
if s["Order"] != "" {
params.Add("order", s["Order"])
}
if s["Mode"] != "" {
params.Add("mode", s["Mode"])
}
if s["Ratio"] != "" {
params.Add("ratio", s["Ratio"])
}
if s["Smode"] != "" {
params.Add("s_mode", s["Smode"])
}
if s["Wlt"] != "" {
params.Add("wlt", s["Wlt"])
}
if s["Wgt"] != "" {
params.Add("wgt", s["Wgt"])
}
if s["Hlt"] != "" {
params.Add("hlt", s["Hlt"])
}
if s["Hgt"] != "" {
params.Add("hgt", s["Hgt"])
}
if s["Tool"] != "" {
params.Add("tool", s["Tool"])
}
if s["Scd"] != "" {
params.Add("scd", s["Scd"])
}
if s["Ecd"] != "" {
params.Add("ecd", s["Ecd"])
}
if s["Page"] != "" {
params.Add("p", s["Page"])
}
return baseURL + "?" + params.Encode()
}
func GetLandingURL(mode string) string {
core/popular_search.go
+138
View File
@@ -0,0 +1,138 @@
package core
import (
"context"
"encoding/json"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
"codeberg.org/vnpower/pixivfe/v2/audit"
"codeberg.org/vnpower/pixivfe/v2/server/session"
)
// Constants for result limits
const (
resultsLimit = 100
)
// Popularity thresholds
var suffixes = []string{"100000", "50000", "30000", "10000", "5000", "1000", "500", "100", "50"}
// SearchResponse represents the JSON structure returned by the Pixiv API for the search endpoint
type SearchResponse struct {
IllustManga struct {
Data []ArtworkBrief `json:"data"`
} `json:"illustManga"`
}
// isValidResult checks if an artwork meets the search criteria
func isValidResult(item ArtworkBrief, category string) bool {
hasFakeTag := strings.Contains(strings.Join(item.Tags, " "), "虚偽users入りタグ")
isCorrectMode := (category == "artworks" && (item.IllustType == 0 || item.IllustType == 1)) ||
(category == "illustrations" && item.IllustType == 0) ||
(category == "manga" && item.IllustType == 1)
return !hasFakeTag && isCorrectMode
}
// processSearchResults filters and adds valid results to the artworks slice
func processSearchResults(items []ArtworkBrief, category string, artworks *[]ArtworkBrief) bool {
for _, item := range items {
if isValidResult(item, category) {
*artworks = append(*artworks, item)
if len(*artworks) >= resultsLimit {
return true
}
}
}
return false
}
// fetchResultsForSuffix retrieves all results for a specific popularity threshold
func fetchResultsForSuffix(ctx context.Context, auditor *audit.Auditor, r *http.Request, settings SearchPageSettings, suffix string, artworks *[]ArtworkBrief) error {
page := 1
for len(*artworks) < resultsLimit {
// Copy settings to avoid modifying the original
currentSettings := settings
// Adjust settings.Name to include the suffix
currentSettings.Name = fmt.Sprintf("%s%susers入り", settings.Name, suffix)
// Set page number
currentSettings.Page = strconv.Itoa(page)
// Generate URL
url := GetSearchArtworksURL(currentSettings.ReturnMap())
auditor.SugaredLogger.Debugw("Fetching search results", "url", url)
body, err := API_GET_UnwrapJson(ctx, auditor, url, "", r.Header)
if err != nil {
auditor.SugaredLogger.Errorw("Error fetching results", "error", err, "suffix", suffix, "page", page)
return err
}
proxiedBody, err := session.ProxyImageUrl(r, body)
if err != nil {
auditor.SugaredLogger.Errorw("Error proxying image URLs", "error", err)
return err
}
// Unmarshal the proxied JSON body into SearchResponse
var resp SearchResponse
err = json.Unmarshal([]byte(proxiedBody), &resp)
if err != nil {
auditor.SugaredLogger.Errorw("Failed to unmarshal JSON", "error", err, "body", proxiedBody)
return err
}
if len(resp.IllustManga.Data) == 0 {
auditor.SugaredLogger.Debugw("No more results found", "suffix", suffix, "page", page)
break
}
reachedLimit := processSearchResults(resp.IllustManga.Data, currentSettings.Category, artworks)
if reachedLimit {
return nil
}
page++
auditor.SugaredLogger.Debugw("Moving to next page", "suffix", suffix, "page", page)
}
return nil
}
// searchPopular performs a search across all popularity thresholds
func searchPopular(ctx context.Context, auditor *audit.Auditor, r *http.Request, settings SearchPageSettings, headers http.Header) (SearchArtworks, error) {
auditor.SugaredLogger.Infow("Starting popular search", "query", settings.Name, "mode", settings.Category)
var artworks []ArtworkBrief
for _, suffix := range suffixes {
auditor.SugaredLogger.Debugw("Searching with suffix", "suffix", suffix)
err := fetchResultsForSuffix(ctx, auditor, r, settings, suffix, &artworks)
if err != nil {
auditor.SugaredLogger.Errorw("Error fetching results for suffix", "suffix", suffix, "error", err)
continue
}
if len(artworks) >= resultsLimit {
break
}
}
total := len(artworks)
auditor.SugaredLogger.Infow("Completed popular search", "query", settings.Name, "mode", settings.Category, "resultsCount", total)
// Sort the results by ID in descending order since higher IDs are newer
sort.Slice(artworks, func(i, j int) bool {
return artworks[i].ID > artworks[j].ID
})
return SearchArtworks{
Artworks: artworks,
Total: total,
}, nil
}
core/tag.go
+34
View File
@@ -7,6 +7,8 @@ import (
"strings"
"codeberg.org/vnpower/pixivfe/v2/audit"
"codeberg.org/vnpower/pixivfe/v2/config"
"codeberg.org/vnpower/pixivfe/v2/i18n"
"codeberg.org/vnpower/pixivfe/v2/server/session"
"github.com/goccy/go-json"
)
@@ -138,7 +140,38 @@ func GetTagData(auditor *audit.Auditor, r *http.Request, name string) (TagDetail
return tag, nil
}
// GetSearch delegates the search operation to either getPopularSearch or getStandardSearch based on settings.Order.
func GetSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSettings) (*SearchResult, error) {
if strings.ToLower(settings.Order) == "popular" {
return getPopularSearch(auditor, r, settings)
}
return getStandardSearch(auditor, r, settings)
}
// getPopularSearch handles the popular search logic.
func getPopularSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSettings) (*SearchResult, error) {
// Check if popular search is enabled
if !config.GlobalConfig.PopularSearchEnabled {
return nil, i18n.Errorf("Popular search is disabled by server configuration.")
}
// Perform popular search
searchArtworks, err := searchPopular(r.Context(), auditor, r, settings, r.Header)
if err != nil {
return nil, err
}
// Create SearchResult
result := &SearchResult{
Artworks: searchArtworks,
// TODO: populate Popular (the regular one) and RelatedTags
}
return result, nil
}
// getStandardSearch handles the standard search logic.
func getStandardSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSettings) (*SearchResult, error) {
URL := GetSearchArtworksURL(settings.ReturnMap())
resp, err := API_GET_UnwrapJson(r.Context(), auditor, URL, "", r.Header)
@@ -150,6 +183,7 @@ func GetSearch(auditor *audit.Auditor, r *http.Request, settings SearchPageSetti
if err != nil {
return nil, err
}
// Replace "illust", "manga", and "illustManga" with "works" in the JSON response
//
// VnPower: IDK how to do better than this lol
doc/hosting/environment-variables.md
+15
View File
@@ -215,6 +215,21 @@ Specifies a shorter Time To Live (TTL) for specific API endpoints that change fr
These paths are considered more dynamic, and the shorter TTL ensures more up-to-date content.
## Feature configuration
### `PIXIVFE_POPULAR_SEARCH_ENABLED`
**Required**: No
**Default:** `false`
Controls whether searching by popularity for a given tag is enabled.
!!! warning
This feature requires several API calls for each search, which may lead to rate limiting by the Pixiv API.
As such, enabling caching (`PIXIVFE_CACHE_ENABLED=true`) is recommended to minimize repeated requests.
## Network proxy configuration
Used to set the [proxy server](https://en.wikipedia.org/wiki/Proxy_server) that PixivFE will use for all requests. Not to be confused with the image proxy, which is used to comply with the `Referer` check required by `i.pximg.net`.
i18n/locale/en/code.json
+1
View File
@@ -30,6 +30,7 @@
"core/requests.go:fEaOk30bc9I": "failed to read response body: %w",
"core/requests.go:lLy9SHFUtQQ": "failed to copy response body: %w",
"core/requests.go:o8eDPgojH4w": "Incompatible response body",
"core/tag.go:LI4dIQHgYP4": "Popular search is disabled by server configuration.",
"core/user.go:T3RwaHcnsYQ": "Invalid work category: %#v. Only \"%s\", \"%s\", \"%s\", \"%s\", \"%s\" and \"%s\" are available",
"server/router/router.go:HORkE0Obn1U": "Failed to redirect to %s",
"server/router/router.go:Z9UG-DYmQCk": "Route not found",
i18n/locale/en/template.json
+7
View File
@@ -531,11 +531,14 @@
"assets/views/tag.jet.html:5gLsaoIn1hw": "Newest",
"assets/views/tag.jet.html:8R6wargzfWI": "Square",
"assets/views/tag.jet.html:8lMwxcLFwfQ": "R-18",
"assets/views/tag.jet.html:9KmARXlO47Y": "#アイマス",
"assets/views/tag.jet.html:DJ9HQmApqX4": "Maximum image width",
"assets/views/tag.jet.html:FSdckWFtaaQ": "works",
"assets/views/tag.jet.html:FpuEclq1XG4": "Maximum image height",
"assets/views/tag.jet.html:L1ef_2BFV8U": "Reset filters",
"assets/views/tag.jet.html:LbYiwrOSQuo": "bookmark count tags to show the most-liked works, but is limited to one page and specific milestones. If no results appear, the tag likely lacks bookmark data. Try broader tags like",
"assets/views/tag.jet.html:LlTlrHqgCXI": "Partial (tags)",
"assets/views/tag.jet.html:MA-a3uM1m7s": "#アイドルマスターシャイニーカラーズ",
"assets/views/tag.jet.html:O8pAU2K_Lco": "Title/Caption",
"assets/views/tag.jet.html:Obw6P7wVCaA": "Tool",
"assets/views/tag.jet.html:P6fO_UTqr7E": "Manga",
@@ -544,8 +547,10 @@
"assets/views/tag.jet.html:R-DIbx-3Pww": "Advanced settings",
"assets/views/tag.jet.html:RcfE_VBY72s": "Set",
"assets/views/tag.jet.html:Sccoo133FQI": "All",
"assets/views/tag.jet.html:XS2z_BaQ4Tw": "users入り",
"assets/views/tag.jet.html:ZqXoa4qWM54": "Recent",
"assets/views/tag.jet.html:_HgJq4ezzIg": "Posted after (format: yyyy-mm-dd)",
"assets/views/tag.jet.html:cL47N2weMW8": "for better results.",
"assets/views/tag.jet.html:chTuQxGY_sk": "Popular works",
"assets/views/tag.jet.html:gEYVuWDipug": "Illustrations",
"assets/views/tag.jet.html:gwWDcxEUA58": "Ratio",
@@ -555,8 +560,10 @@
"assets/views/tag.jet.html:qPuGKC6JtSY": "Exact (tags)",
"assets/views/tag.jet.html:qepcz_2ar6E": "Artworks",
"assets/views/tag.jet.html:tLDBgUWw9-g": "Safe",
"assets/views/tag.jet.html:v0F5PrG9DyA": "instead of",
"assets/views/tag.jet.html:vQknITzlB0I": "Search options",
"assets/views/tag.jet.html:vUR6oAGwbwE": "Order",
"assets/views/tag.jet.html:wsrqIAla6TE": "The \"Popular\" sorting uses",
"assets/views/tag.jet.html:x-I1EHIg1-o": "All time",
"assets/views/tag.jet.html:xd9cjh9m0Go": "({{ .TranslatedName }})",
"assets/views/tag.jet.html:yHft19aQloM": "Minimum image width",
server/routes/tag.go
+13 -11
View File
@@ -6,6 +6,7 @@ import (
"strconv"
"codeberg.org/vnpower/pixivfe/v2/audit"
"codeberg.org/vnpower/pixivfe/v2/config"
"codeberg.org/vnpower/pixivfe/v2/core"
"codeberg.org/vnpower/pixivfe/v2/server/template"
"codeberg.org/vnpower/pixivfe/v2/server/utils"
@@ -60,17 +61,18 @@ func TagPage(auditor *audit.Auditor, w http.ResponseWriter, r *http.Request) err
PreloadImage(w, tag.Metadata.ImageMaster)
return RenderHTML(w, r, Data_tag{
Title: "Results for " + name,
Tag: tag,
Data: *result,
QueriesC: urlc,
TrueTag: param,
Page: pageInt,
ActiveCategory: queries.Category,
ActiveOrder: queries.Order,
ActiveMode: queries.Mode,
ActiveRatio: queries.Ratio,
ActiveSearchMode: GetQueryParam(r, "smode", ""),
Title: "Results for " + name,
Tag: tag,
Data: *result,
QueriesC: urlc,
TrueTag: param,
Page: pageInt,
ActiveCategory: queries.Category,
ActiveOrder: queries.Order,
ActiveMode: queries.Mode,
ActiveRatio: queries.Ratio,
ActiveSearchMode: GetQueryParam(r, "smode", ""),
PopularSearchEnabled: config.GlobalConfig.PopularSearchEnabled,
})
}
server/routes/types.go
+12 -11
View File
@@ -140,17 +140,18 @@ type Data_settings struct {
DefaultProxyServer string
}
type Data_tag struct {
Title string
Tag core.TagDetail
Data core.SearchResult
QueriesC template.PartialURL
TrueTag string
Page int
ActiveCategory string
ActiveOrder string
ActiveMode string
ActiveRatio string
ActiveSearchMode string
Title string
Tag core.TagDetail
Data core.SearchResult
QueriesC template.PartialURL
TrueTag string
Page int
ActiveCategory string
ActiveOrder string
ActiveMode string
ActiveRatio string
ActiveSearchMode string
PopularSearchEnabled bool
}
type Data_user struct {
Title string