VnPower/PixivFE
1
0
Fork 0
mirror of https://codeberg.org/VnPower/PixivFE synced 2024-12-06 19:16:23 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implement random User-Agent selection

Browse Source 
This commit introduces a mechanism for randomly selecting User-Agents
from a predefined list to be used for requests to the Pixiv API.

References to setting a custom User-Agent via the PIXIVFE_USERAGENT
environment variable have been removed as it is no longer used.
This commit is contained in:
perennial
2024-09-23 21:22:34 +10:00
parent d25b3a3ec9
commit d1534682e2
5 changed files with 59 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+1 -2
View File
@@ -10,8 +10,7 @@ PIXIVFE_PORT='8282'
PIXIVFE_HOST='127.0.0.1'
# PIXIVFE_REQUESTLIMIT=
# PIXIVFE_IMAGEPROXY=
# PIXIVFE_USERAGENT='Mozilla/5.0 (Windows NT 10.0; rv:123.0) Gecko/20100101 Firefox/123.0'
# PIXIVFE_ACCEPTLANGUAGE='en-US,en;q=0.5'
# PIXIVFE_ACCEPTLANGUAGE=
# PIXIVFE_PROXY_CHECK_INTERVAL=
# PIXIVFE_TOKEN_LOAD_BALANCING=
# PIXIVFE_REPO_URL=
config/config.go
-2
View File
@@ -52,7 +52,6 @@ type ServerConfig struct {
APIBaseTimeout time.Duration `env:"PIXIVFE_API_BASE_TIMEOUT,overwrite"`
APIMaxBackoffTime time.Duration `env:"PIXIVFE_API_MAX_BACKOFF_TIME,overwrite"`
UserAgent string `env:"PIXIVFE_USERAGENT,overwrite"`
AcceptLanguage string `env:"PIXIVFE_ACCEPTLANGUAGE,overwrite"`
RequestLimit int `env:"PIXIVFE_REQUESTLIMIT"` // if 0, request limit is disabled
@@ -118,7 +117,6 @@ func (s *ServerConfig) LoadConfig() error {
// set default values with env:"...,overwrite"
s.RepoURL = "https://codeberg.org/VnPower/PixivFE"
s.UserAgent = "Mozilla/5.0 (Windows NT 10.0; rv:123.0) Gecko/20100101 Firefox/123.0"
s.AcceptLanguage = "en-US,en;q=0.5"
s.ProxyServer_staging = BuiltinProxyUrl
s.ProxyCheckInterval = 8 * time.Hour
config/user_agent_list.go
+53
View File
@@ -0,0 +1,53 @@
package config
import "math/rand"
// TODO: Consider allowing a comma-separated string of User-Agents to be
// defined in the (now removed) PIXIVFE_USERAGENT environment variable, a la PIXIVFE_TOKEN.
//
// On the other hand, this implementation currently is *good enough*
// and doesn't risk duplicating token_manager logic.
//
// Abstracting token_manager logic to be used more generally (via interfaces?)
// is a tradeoff; is randomly picking from user-defined User-Agents *really*
// that important?
// BuiltinUserAgentList is a list of random real User-Agents.
// Copied manually from https://github.com/spider-rs/ua_generator
var BuiltinUserAgentList = []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Android 12; Tablet; rv:110.0) Gecko/110.0 Firefox/110.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Android 12; Tablet; rv:110.0) Gecko/110.0 Firefox/110.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
// Chrome User-Agents
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
}
// GetRandomUserAgent returns a random user agent from the BuiltinUserAgentList
func GetRandomUserAgent() string {
return BuiltinUserAgentList[rand.Intn(len(BuiltinUserAgentList))]
}
core/requests.go
+2 -2
View File
@@ -119,7 +119,7 @@ func API_GET(ctx context.Context, url string, userToken string) (SimpleHTTPRespo
return nil, err
}
req = req.WithContext(ctx)
req.Header.Add("User-Agent", config.GlobalConfig.UserAgent)
req.Header.Add("User-Agent", config.GetRandomUserAgent())
req.Header.Add("Accept-Language", config.GlobalConfig.AcceptLanguage)
req.AddCookie(&http.Cookie{
Name: "PHPSESSID",
@@ -165,7 +165,7 @@ func API_POST(ctx context.Context, url, payload, userToken, csrf string, isJSON
return nil, err
}
req = req.WithContext(ctx)
req.Header.Add("User-Agent", "Mozilla/5.0")
req.Header.Add("User-Agent", config.GetRandomUserAgent())
req.Header.Add("Accept", "application/json")
req.Header.Add("x-csrf-token", csrf)
req.AddCookie(&http.Cookie{
doc/hosting/environment-variables.md
+3 -9
View File
@@ -68,7 +68,9 @@ It's recommended to enable rate limiting in the reverse proxy in front of PixivF
## `PIXIVFE_IMAGEPROXY`
**Required**: No, defaults to using the built-in proxy
**Required**: No
**Default:** Uses the built-in proxy.
!!! note
The protocol **must** be included in the URL, e.g., `https://piximg.example.com`, where `https://` is the protocol used.
@@ -77,14 +79,6 @@ The URL of the image proxy server. Pixiv requires `Referer: https://www.pixiv.ne
See [hosting an image proxy server](image-proxy-server.md) or the [list of public image proxies](../public-image-proxies.md).
## `PIXIVFE_USERAGENT`
**Required**: No
**Default:** `Mozilla/5.0 (Windows NT 10.0; rv:123.0) Gecko/20100101 Firefox/123.0`
The value of the `User-Agent` header used for requests to Pixiv's API.
## `PIXIVFE_ACCEPTLANGUAGE`
**Required**: No