From d1534682e2951d594133fc07cb109bfa922f0a2f Mon Sep 17 00:00:00 2001 From: perennial Date: Mon, 23 Sep 2024 21:22:34 +1000 Subject: [PATCH] Implement random User-Agent selection This commit introduces a mechanism for randomly selecting User-Agents from a predefined list to be used for requests to the Pixiv API. References to setting a custom User-Agent via the PIXIVFE_USERAGENT environment variable have been removed as it is no longer used. --- .env.example | 3 +- config/config.go | 2 -- config/user_agent_list.go | 53 ++++++++++++++++++++++++++++ core/requests.go | 4 +-- doc/hosting/environment-variables.md | 12 ++----- 5 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 config/user_agent_list.go diff --git a/.env.example b/.env.example index 9cac2c2..086e670 100644 --- a/.env.example +++ b/.env.example @@ -10,8 +10,7 @@ PIXIVFE_PORT='8282' PIXIVFE_HOST='127.0.0.1' # PIXIVFE_REQUESTLIMIT= # PIXIVFE_IMAGEPROXY= -# PIXIVFE_USERAGENT='Mozilla/5.0 (Windows NT 10.0; rv:123.0) Gecko/20100101 Firefox/123.0' -# PIXIVFE_ACCEPTLANGUAGE='en-US,en;q=0.5' +# PIXIVFE_ACCEPTLANGUAGE= # PIXIVFE_PROXY_CHECK_INTERVAL= # PIXIVFE_TOKEN_LOAD_BALANCING= # PIXIVFE_REPO_URL= diff --git a/config/config.go b/config/config.go index 22cb0a2..2dd3a8c 100644 --- a/config/config.go +++ b/config/config.go @@ -52,7 +52,6 @@ type ServerConfig struct { APIBaseTimeout time.Duration `env:"PIXIVFE_API_BASE_TIMEOUT,overwrite"` APIMaxBackoffTime time.Duration `env:"PIXIVFE_API_MAX_BACKOFF_TIME,overwrite"` - UserAgent string `env:"PIXIVFE_USERAGENT,overwrite"` AcceptLanguage string `env:"PIXIVFE_ACCEPTLANGUAGE,overwrite"` RequestLimit int `env:"PIXIVFE_REQUESTLIMIT"` // if 0, request limit is disabled @@ -118,7 +117,6 @@ func (s *ServerConfig) LoadConfig() error { // set default values with env:"...,overwrite" s.RepoURL = "https://codeberg.org/VnPower/PixivFE" - s.UserAgent = "Mozilla/5.0 (Windows NT 10.0; rv:123.0) Gecko/20100101 Firefox/123.0" s.AcceptLanguage = "en-US,en;q=0.5" s.ProxyServer_staging = BuiltinProxyUrl s.ProxyCheckInterval = 8 * time.Hour diff --git a/config/user_agent_list.go b/config/user_agent_list.go new file mode 100644 index 0000000..2d41fce --- /dev/null +++ b/config/user_agent_list.go @@ -0,0 +1,53 @@ +package config + +import "math/rand" + +// TODO: Consider allowing a comma-separated string of User-Agents to be +// defined in the (now removed) PIXIVFE_USERAGENT environment variable, a la PIXIVFE_TOKEN. +// +// On the other hand, this implementation currently is *good enough* +// and doesn't risk duplicating token_manager logic. +// +// Abstracting token_manager logic to be used more generally (via interfaces?) +// is a tradeoff; is randomly picking from user-defined User-Agents *really* +// that important? + +// BuiltinUserAgentList is a list of random real User-Agents. +// Copied manually from https://github.com/spider-rs/ua_generator +var BuiltinUserAgentList = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Android 12; Tablet; rv:110.0) Gecko/110.0 Firefox/110.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Android 12; Tablet; rv:110.0) Gecko/110.0 Firefox/110.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + // Chrome User-Agents + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", +} + +// GetRandomUserAgent returns a random user agent from the BuiltinUserAgentList +func GetRandomUserAgent() string { + return BuiltinUserAgentList[rand.Intn(len(BuiltinUserAgentList))] +} diff --git a/core/requests.go b/core/requests.go index 6b23944..5f3329c 100644 --- a/core/requests.go +++ b/core/requests.go @@ -119,7 +119,7 @@ func API_GET(ctx context.Context, url string, userToken string) (SimpleHTTPRespo return nil, err } req = req.WithContext(ctx) - req.Header.Add("User-Agent", config.GlobalConfig.UserAgent) + req.Header.Add("User-Agent", config.GetRandomUserAgent()) req.Header.Add("Accept-Language", config.GlobalConfig.AcceptLanguage) req.AddCookie(&http.Cookie{ Name: "PHPSESSID", @@ -165,7 +165,7 @@ func API_POST(ctx context.Context, url, payload, userToken, csrf string, isJSON return nil, err } req = req.WithContext(ctx) - req.Header.Add("User-Agent", "Mozilla/5.0") + req.Header.Add("User-Agent", config.GetRandomUserAgent()) req.Header.Add("Accept", "application/json") req.Header.Add("x-csrf-token", csrf) req.AddCookie(&http.Cookie{ diff --git a/doc/hosting/environment-variables.md b/doc/hosting/environment-variables.md index a41a731..b1ca3ed 100644 --- a/doc/hosting/environment-variables.md +++ b/doc/hosting/environment-variables.md @@ -68,7 +68,9 @@ It's recommended to enable rate limiting in the reverse proxy in front of PixivF ## `PIXIVFE_IMAGEPROXY` -**Required**: No, defaults to using the built-in proxy +**Required**: No + +**Default:** Uses the built-in proxy. !!! note The protocol **must** be included in the URL, e.g., `https://piximg.example.com`, where `https://` is the protocol used. @@ -77,14 +79,6 @@ The URL of the image proxy server. Pixiv requires `Referer: https://www.pixiv.ne See [hosting an image proxy server](image-proxy-server.md) or the [list of public image proxies](../public-image-proxies.md). -## `PIXIVFE_USERAGENT` - -**Required**: No - -**Default:** `Mozilla/5.0 (Windows NT 10.0; rv:123.0) Gecko/20100101 Firefox/123.0` - -The value of the `User-Agent` header used for requests to Pixiv's API. - ## `PIXIVFE_ACCEPTLANGUAGE` **Required**: No