From 87aba9c0d29f62b86175fbc9985f500831cf3347 Mon Sep 17 00:00:00 2001 From: perennial Date: Thu, 17 Oct 2024 17:11:18 +1100 Subject: [PATCH] requests.go: consolidate functions --- core/requests.go | 149 +++++++++++++++++++---------------------------- 1 file changed, 61 insertions(+), 88 deletions(-) diff --git a/core/requests.go b/core/requests.go index e485071..ad11d74 100644 --- a/core/requests.go +++ b/core/requests.go @@ -52,7 +52,6 @@ var ( "/ajax/discovery/artworks", "/ajax/discovery/novels", "/ajax/illust/new", - "/ranking.php", } // excludedCachePaths lists API endpoints that should *never* be cached, regardless of any other factors. @@ -117,113 +116,86 @@ func generateCacheKey(url, userToken string) string { return fmt.Sprintf("%x", hash) } -// isShortTTLPath determines whether a given URL path should use a shorter TTL -// based on predefined paths that have more dynamic content. -func isShortTTLPath(rawURL string) bool { +// shouldUseCache determines the caching policy for a given request. +// +// It returns three values: +// - shouldFetch: whether to attempt fetching from the cache. +// - shouldStore: whether to store the response in the cache after retrieval. +// - cachedResponse: the cached response if available and valid. +func shouldUseCache(rawURL, userToken string, incomingHeaders http.Header) (shouldFetch bool, shouldStore bool, cachedResponse *SimpleHTTPResponse) { + if !config.GlobalConfig.CacheEnabled { + return false, false, nil + } + parsedURL, err := url.Parse(rawURL) if err != nil { - return false - } - - urlPath := path.Clean(parsedURL.Path) - - for _, shortPath := range shortTTLPaths { - if urlPath == shortPath { - return true - } - } - - return false -} - -// isExcludedFromCache determines whether a specific API endpoint should *never* be cached, regardless of any other factors. -func isExcludedFromCache(rawURL string) bool { - parsedURL, err := url.Parse(rawURL) - if err != nil { - return false + return false, false, nil } urlPath := path.Clean(parsedURL.Path) + // Check if the path is excluded from caching for _, exclPath := range excludedCachePaths { if strings.HasPrefix(urlPath, exclPath) { - return true + return false, false, nil } } - return false -} -// shouldInvalidateCache checks if the incoming request headers indicate that the cache should be invalidated. -func shouldInvalidateCache(incomingHeaders http.Header) bool { + cacheKey := generateCacheKey(rawURL, userToken) + + // Check if cache invalidation is requested redirectedHeader := incomingHeaders.Get("X-Handled-Redirected") - return strings.Contains(strings.ToLower(redirectedHeader), "true") -} + if strings.Contains(strings.ToLower(redirectedHeader), "true") { + cache.Remove(cacheKey) + return false, false, nil + } -// fetchFromCache retrieves a cached response if it exists and is not expired. -func fetchFromCache(cacheKey string) (*CachedItem, bool) { + // Attempt to fetch from cache if cachedItem, found := cache.Get(cacheKey); found { item := cachedItem.(CachedItem) if time.Now().Before(item.ExpiresAt) { - return &item, true + return false, false, item.Response } - } - return nil, false -} - -// shouldCacheResponse determines whether a particular response should be cached based on incoming headers. -func shouldCacheResponse(incomingHeaders http.Header) bool { - cacheControl := incomingHeaders.Get("Cache-Control") - return cacheControl == "" || !strings.Contains(strings.ToLower(cacheControl), "no-cache") -} - -// determineTTL selects the appropriate TTL based on the URL path. -func determineTTL(url string) time.Duration { - if isShortTTLPath(url) { - return config.GlobalConfig.CacheShortTTL - } - return config.GlobalConfig.CacheTTL -} - -// getCachedResponse attempts to retrieve a cached response if caching is enabled. -// -// It also handles cache invalidation based on redirection headers. -func getCachedResponse(url, userToken string, incomingHeaders http.Header) (*SimpleHTTPResponse, bool) { - if !config.GlobalConfig.CacheEnabled { - return nil, false - } - - if isExcludedFromCache(url) { - return nil, false - } - - cacheKey := generateCacheKey(url, userToken) - - if shouldInvalidateCache(incomingHeaders) { + // Cache expired cache.Remove(cacheKey) - } else if cachedItem, found := fetchFromCache(cacheKey); found { - return cachedItem.Response, true } - return nil, false + // Determine if response should be cached based on headers + cacheControl := incomingHeaders.Get("Cache-Control") + shouldStore = cacheControl == "" || !strings.Contains(strings.ToLower(cacheControl), "no-cache") + + return true, shouldStore, nil } -// cacheSuccessfulResponse caches the response if caching is enabled and allowed by headers. -func cacheSuccessfulResponse(url, userToken string, incomingHeaders http.Header, resp *SimpleHTTPResponse) { - if !config.GlobalConfig.CacheEnabled { - return +// handleCaching manages the caching logic during and after a request. +// It either returns a cached response or allows the request to proceed and optionally caches the new response. +func handleCaching(rawURL, userToken string, incomingHeaders http.Header, resp *SimpleHTTPResponse) (*SimpleHTTPResponse, bool) { + shouldFetch, shouldStore, cachedResponse := shouldUseCache(rawURL, userToken, incomingHeaders) + if !shouldFetch && cachedResponse != nil { + return cachedResponse, true } - if isExcludedFromCache(url) { - return - } + if shouldStore && resp != nil { + ttl := config.GlobalConfig.CacheTTL + parsedURL, err := url.Parse(rawURL) + if err != nil { + return resp, false + } + urlPath := path.Clean(parsedURL.Path) + for _, shortPath := range shortTTLPaths { + if urlPath == shortPath { + ttl = config.GlobalConfig.CacheShortTTL + break + } + } - if shouldCacheResponse(incomingHeaders) { - ttl := determineTTL(url) - cache.Add(generateCacheKey(url, userToken), CachedItem{ + cache.Add(generateCacheKey(rawURL, userToken), CachedItem{ Response: resp, ExpiresAt: time.Now().Add(ttl), }) } + + return resp, false } // makeRequest is a helper function that encapsulates the common logic for making HTTP requests. @@ -327,7 +299,13 @@ func executeGetWithRetries( if resp.StatusCode == http.StatusOK { tokenManager.MarkTokenStatus(token, token_manager.Good) - cacheSuccessfulResponse(url, userToken, incomingHeaders, resp) + + // Handle caching logic after we receieve an HTTP 200 response + cachedResp, isCached := handleCaching(url, userToken, incomingHeaders, resp) + if isCached { + return cachedResp, nil + } + return resp, nil } @@ -346,9 +324,6 @@ func executeGetWithRetries( } // retryRequest handles the logic for executing HTTP requests with automatic retries and token management. -// -// It distinguishes between GET and POST requests, caching only GET requests when appropriate, -// and ensures that tokens are managed correctly for authentication. func retryRequest( ctx context.Context, reqFunc func(context.Context, string) (*retryablehttp.Request, error), @@ -361,11 +336,9 @@ func retryRequest( return executeSinglePostRequest(ctx, reqFunc, userToken, url) } - if isExcludedFromCache(url) { - return executeGetWithRetries(ctx, reqFunc, userToken, url, incomingHeaders) - } - - if cachedResponse, shouldReturn := getCachedResponse(url, userToken, incomingHeaders); shouldReturn { + // Handle caching logic before we send a request + shouldFetch, _, cachedResponse := shouldUseCache(url, userToken, incomingHeaders) + if !shouldFetch && cachedResponse != nil { return cachedResponse, nil }