From c3bb4771329fa00f2905d86f3f2fbcba84d303ac Mon Sep 17 00:00:00 2001 From: perennial Date: Tue, 24 Sep 2024 19:12:58 +1000 Subject: [PATCH] Rename token management environment variables This commit clarifies the purpose of certain environment variables by explicitly associating them with token management. The variables PIXIVFE_MAX_RETRIES, PIXIVFE_BASE_TIMEOUT, and PIXIVFE_MAX_BACKOFF_TIME are now prefixed with "TOKEN_" to better distinguish them from API-level settings. The ServerConfig struct, configuration loading, logging, and documentation have been updated accordingly. --- .env.example | 6 ++-- config/config.go | 22 ++++++------- doc/dev/features/exponential_backoff.md | 42 ++++--------------------- doc/hosting/environment-variables.md | 6 ++-- 4 files changed, 23 insertions(+), 53 deletions(-) diff --git a/.env.example b/.env.example index 7352ff0..b3db250 100644 --- a/.env.example +++ b/.env.example @@ -27,9 +27,9 @@ PIXIVFE_HOST='127.0.0.1' # PIXIVFE_API_MAX_BACKOFF_TIME= #### Token management level backoff -# PIXIVFE_MAX_RETRIES= -# PIXIVFE_BASE_TIMEOUT= -# PIXIVFE_MAX_BACKOFF_TIME= +# PIXIVFE_TOKEN_MAX_RETRIES= +# PIXIVFE_TOKEN_BASE_TIMEOUT= +# PIXIVFE_TOKEN_MAX_BACKOFF_TIME= ### Development options # PIXIVFE_DEV= diff --git a/config/config.go b/config/config.go index e566cf4..b18c0d9 100644 --- a/config/config.go +++ b/config/config.go @@ -40,12 +40,12 @@ type ServerConfig struct { RepoURL string `env:"PIXIVFE_REPO_URL,overwrite"` // used in /about page - Token []string `env:"PIXIVFE_TOKEN,required"` // may be multiple tokens. delimiter is ',' - TokenManager *token_manager.TokenManager - TokenLoadBalancing string `env:"PIXIVFE_TOKEN_LOAD_BALANCING,overwrite"` - MaxRetries int `env:"PIXIVFE_MAX_RETRIES,overwrite"` - BaseTimeout time.Duration `env:"PIXIVFE_BASE_TIMEOUT,overwrite"` - MaxBackoffTime time.Duration `env:"PIXIVFE_MAX_BACKOFF_TIME,overwrite"` + Token []string `env:"PIXIVFE_TOKEN,required"` // may be multiple tokens. delimiter is ',' + TokenManager *token_manager.TokenManager + TokenLoadBalancing string `env:"PIXIVFE_TOKEN_LOAD_BALANCING,overwrite"` + TokenMaxRetries int `env:"PIXIVFE_TOKEN_MAX_RETRIES,overwrite"` + TokenBaseTimeout time.Duration `env:"PIXIVFE_TOKEN_BASE_TIMEOUT,overwrite"` + TokenMaxBackoffTime time.Duration `env:"PIXIVFE_TOKEN_MAX_BACKOFF_TIME,overwrite"` // API request level backoff settings APIMaxRetries int `env:"PIXIVFE_API_MAX_RETRIES,overwrite"` @@ -123,9 +123,9 @@ func (s *ServerConfig) LoadConfig() error { s.ProxyCheckEnabled = true s.ProxyCheckInterval = 8 * time.Hour s.TokenLoadBalancing = "round-robin" - s.MaxRetries = 5 - s.BaseTimeout = 1000 * time.Millisecond - s.MaxBackoffTime = 32000 * time.Millisecond + s.TokenMaxRetries = 5 + s.TokenBaseTimeout = 1000 * time.Millisecond + s.TokenMaxBackoffTime = 32000 * time.Millisecond s.APIMaxRetries = 3 s.APIBaseTimeout = 500 * time.Millisecond @@ -181,9 +181,9 @@ func (s *ServerConfig) LoadConfig() error { } // Initialize TokenManager - s.TokenManager = token_manager.NewTokenManager(s.Token, s.MaxRetries, s.BaseTimeout, s.MaxBackoffTime, s.TokenLoadBalancing) + s.TokenManager = token_manager.NewTokenManager(s.Token, s.TokenMaxRetries, s.TokenBaseTimeout, s.TokenMaxBackoffTime, s.TokenLoadBalancing) log.Printf("Token manager initialized with %d tokens\n", len(s.Token)) - log.Printf("Token manager settings: Max retries: %d, Base timeout: %v, Max backoff time: %v\n", s.MaxRetries, s.BaseTimeout, s.MaxBackoffTime) + log.Printf("Token manager settings: Max retries: %d, Base timeout: %v, Max backoff time: %v\n", s.TokenMaxRetries, s.TokenBaseTimeout, s.TokenMaxBackoffTime) log.Printf("Token load balancing method: %s\n", s.TokenLoadBalancing) log.Printf("API request backoff settings: Max retries: %d, Base timeout: %v, Max backoff time: %v\n", s.APIMaxRetries, s.APIBaseTimeout, s.APIMaxBackoffTime) diff --git a/doc/dev/features/exponential_backoff.md b/doc/dev/features/exponential_backoff.md index dd2e476..ae5cdfa 100644 --- a/doc/dev/features/exponential_backoff.md +++ b/doc/dev/features/exponential_backoff.md @@ -9,6 +9,12 @@ Exponential backoff is a technique used to gradually increase the wait time betw 1. API request level 2. Token management level +### Configuration + +The `ServerConfig` struct in [`config/config.go`](https://codeberg.org/VnPower/PixivFE/src/branch/v2/config/config.go) includes fields for both API request level and token management level backoff settings. + +The `LoadConfig` method sets default values for these settings if they are not provided through environment variables. + ## API request level backoff **Location: `core/requests.go`** @@ -53,39 +59,3 @@ The `TokenManager` implements exponential backoff for individual tokens: - The token's `TimeoutUntil` is set to the current time plus this calculated duration. This approach allows tokens that repeatedly fail increasingly longer "cool-down" periods before being used again, helping to manage rate limiting of individual tokens by the Pixiv API. - -## Implementation details - -### Configuration (`config/config.go`) - -The `ServerConfig` struct in `config/config.go` includes fields for both API request level and token management level backoff settings: - -```go -type ServerConfig struct { - // ... other fields ... - MaxRetries int `env:"PIXIVFE_MAX_RETRIES,overwrite"` - BaseTimeout time.Duration `env:"PIXIVFE_BASE_TIMEOUT,overwrite"` - MaxBackoffTime time.Duration `env:"PIXIVFE_MAX_BACKOFF_TIME,overwrite"` - - APIMaxRetries int `env:"PIXIVFE_API_MAX_RETRIES,overwrite"` - APIBaseTimeout time.Duration `env:"PIXIVFE_API_BASE_TIMEOUT,overwrite"` - APIMaxBackoffTime time.Duration `env:"PIXIVFE_API_MAX_BACKOFF_TIME,overwrite"` - // ... other fields ... -} -``` - -The `LoadConfig` method sets default values for these settings if they are not provided through environment variables: - -```go -func (s *ServerConfig) LoadConfig() error { - // ... other initializations ... - s.MaxRetries = 5 - s.BaseTimeout = 1 * time.Second - s.MaxBackoffTime = 32 * time.Second - - s.APIMaxRetries = 3 - s.APIBaseTimeout = 500 * time.Millisecond - s.APIMaxBackoffTime = 8 * time.Second - // ... rest of the method ... -} -``` diff --git a/doc/hosting/environment-variables.md b/doc/hosting/environment-variables.md index 5b26a8a..0ea05b3 100644 --- a/doc/hosting/environment-variables.md +++ b/doc/hosting/environment-variables.md @@ -169,7 +169,7 @@ Maximum backoff time for API requests. These settings control how PixivFE manages token timeouts when a token encounters repeated failures. The backoff time for a token starts at the base timeout and doubles with each failure, up to the maximum backoff time. -#### `PIXIVFE_MAX_RETRIES` +#### `PIXIVFE_TOKEN_MAX_RETRIES` **Required**: No @@ -177,7 +177,7 @@ These settings control how PixivFE manages token timeouts when a token encounter Maximum number of retries for token management. -#### `PIXIVFE_BASE_TIMEOUT` +#### `PIXIVFE_TOKEN_BASE_TIMEOUT` **Required**: No @@ -185,7 +185,7 @@ Maximum number of retries for token management. Base timeout duration for token management. -#### `PIXIVFE_MAX_BACKOFF_TIME` +#### `PIXIVFE_TOKEN_MAX_BACKOFF_TIME` **Required**: No