invalidate API resp cache upon user action

This commit is contained in:
perennial
2024-10-16 14:24:49 +11:00
parent 31dc0d660d
commit cbbe302ea7
18 changed files with 160 additions and 99 deletions
+5 -5
View File
@@ -172,7 +172,7 @@ func GetUserBasicInformation(r *http.Request, id string) (UserBrief, error) {
URL := GetUserInformationURL(id)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return user, err
}
@@ -192,7 +192,7 @@ func GetArtworkImages(r *http.Request, id string, illustType int) ([]Image, erro
URL := GetArtworkImagesURL(id)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
@@ -237,7 +237,7 @@ func GetArtworkComments(r *http.Request, id string) ([]Comment, error) {
URL := GetArtworkCommentsURL(id)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
@@ -259,7 +259,7 @@ func GetRelatedArtworks(r *http.Request, id string) ([]ArtworkBrief, error) {
// TODO: keep the hard-coded limit?
URL := GetArtworkRelatedURL(id, 96)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
@@ -305,7 +305,7 @@ func GetArtworkByID(r *http.Request, id string, full bool) (*Illust, error) {
// Fetch core artwork metadata as the starting point for assembling all related information.
urlArtInfo := GetArtworkInformationURL(id)
response, err := API_GET_UnwrapJson(r.Context(), urlArtInfo, token)
response, err := API_GET_UnwrapJson(r.Context(), urlArtInfo, token, r.Header)
if err != nil {
cerr <- err
return
+2 -2
View File
@@ -17,7 +17,7 @@ func GetDiscoveryArtwork(r *http.Request, mode string) ([]ArtworkBrief, error) {
var artworks []ArtworkBrief
resp, err := API_GET_UnwrapJson(r.Context(), URL, token)
resp, err := API_GET_UnwrapJson(r.Context(), URL, token, r.Header)
if err != nil {
return nil, err
}
@@ -42,7 +42,7 @@ func GetDiscoveryNovels(r *http.Request, mode string) ([]NovelBrief, error) {
var novels []NovelBrief
resp, err := API_GET_UnwrapJson(r.Context(), URL, token)
resp, err := API_GET_UnwrapJson(r.Context(), URL, token, r.Header)
if err != nil {
return nil, err
}
+1 -1
View File
@@ -58,7 +58,7 @@ func GetLanding(r *http.Request, mode string, isLoggedIn bool) (*LandingArtworks
var landing LandingArtworks
resp, err := API_GET_UnwrapJson(r.Context(), URL, "")
resp, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return &landing, err
}
+1 -1
View File
@@ -49,7 +49,7 @@ func GetMangaSeriesContentByID(r *http.Request, id string, page int) (MangaSerie
URL := GetMangaSeriesContentURL(id, page)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return series_content, err
}
+1 -1
View File
@@ -16,7 +16,7 @@ func GetNewestArtworks(r *http.Request, worktype string, r18 string) ([]ArtworkB
// LastId string
}
resp, err := API_GET_UnwrapJson(r.Context(), URL, token)
resp, err := API_GET_UnwrapJson(r.Context(), URL, token, r.Header)
if err != nil {
return nil, err
}
+4 -4
View File
@@ -139,7 +139,7 @@ func GetNovelByID(r *http.Request, id string) (Novel, error) {
URL := GetNovelURL(id)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return novel, err
}
@@ -176,7 +176,7 @@ func GetNovelByID(r *http.Request, id string) (Novel, error) {
illustid := re_d.FindString(s)
URL := GetInsertIllustURL(novel.ID, illustid)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return "Cannot insert illust" + illustid
}
@@ -227,7 +227,7 @@ func GetNovelRelated(r *http.Request, id string) ([]NovelBrief, error) {
// hard-coded value, may change
URL := GetNovelRelatedURL(id, 50)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return novels.List, err
}
@@ -248,7 +248,7 @@ func GetNovelComments(r *http.Request, id string) ([]Comment, error) {
URL := GetNovelCommentsURL(id)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
+3 -3
View File
@@ -98,7 +98,7 @@ func GetNovelSeriesByID(r *http.Request, id string) (NovelSeries, error) {
URL := GetNovelSeriesURL(id)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return series, err
}
@@ -125,7 +125,7 @@ func GetNovelSeriesContentByID(r *http.Request, id string, page int, perPage int
URL := GetNovelSeriesContentURL(id, page, perPage)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return novels.Page.SeriesContents, err
}
@@ -158,7 +158,7 @@ func GetNovelSeriesContentTitlesByID(r *http.Request, id int) ([]NovelSeriesCont
URL := GetNovelSeriesContentTitlesURL(id)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return contentTitles, err
}
+1 -1
View File
@@ -20,7 +20,7 @@ func GetNewestFromFollowing(r *http.Request, mode, page string) ([]ArtworkBrief,
Artworks []ArtworkBrief `json:"illust"`
}
resp, err := API_GET_UnwrapJson(r.Context(), URL, token)
resp, err := API_GET_UnwrapJson(r.Context(), URL, token, r.Header)
if err != nil {
return nil, err
}
+1 -1
View File
@@ -64,7 +64,7 @@ func GetRanking(r *http.Request, mode, content, date, page string) (Ranking, err
for attempt := 0; attempt < maxRetries; attempt++ {
i18n.Sprintf("Attempt %d of %d", attempt+1, maxRetries)
resp, err := API_GET(r.Context(), URL, "")
resp, err := API_GET(r.Context(), URL, "", r.Header)
if err != nil {
i18n.Errorf("API_GET error: %v", err)
return ranking, err
+1 -1
View File
@@ -65,7 +65,7 @@ func GetRankingCalendar(r *http.Request, mode string, year, month int) ([]DayCal
URL := GetRankingCalendarURL(mode, year, month)
// Make an API request to Pixiv
resp, err := API_GET(r.Context(), URL, token)
resp, err := API_GET(r.Context(), URL, token, r.Header)
if err != nil {
return nil, err
}
+94 -64
View File
@@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"path"
"strings"
"sync"
"time"
@@ -39,8 +40,8 @@ type CachedItem struct {
}
var (
cache *lru.Cache
cacheLock sync.RWMutex
cache *lru.Cache
cacheLock sync.RWMutex
// NOTE: these are the URI paths for the API request being made, not the downstream user request
shortTTLPaths = []string{
"/ajax/discovery/artworks",
@@ -67,6 +68,40 @@ func init() {
retryClient.Logger = nil // Disables the default logger in go-retryablehttp
}
// generateCacheKey creates a unique cache key based on URL and user token
func generateCacheKey(url, userToken string) string {
return fmt.Sprintf("%s:%s", url, userToken)
}
// isShortTTLPath checks if the given URL should use the shorter TTL
func isShortTTLPath(rawURL string) bool {
// TODO: clean up debug statements
fmt.Printf("Checking if URL is a short TTL path: %s\n", rawURL)
parsedURL, err := url.Parse(rawURL)
if err != nil {
fmt.Printf("Error parsing URL: %v\n", err)
// If we can't parse the URL, we'll assume it's not a short TTL path
return false
}
// Extract the path and remove trailing slash if present
urlPath := path.Clean(parsedURL.Path)
fmt.Printf("Cleaned URL path: %s\n", urlPath)
for _, shortPath := range shortTTLPaths {
fmt.Printf("Comparing with short TTL path: %s\n", shortPath)
if urlPath == shortPath {
fmt.Printf("Match found! URL is a short TTL path\n")
return true
}
}
fmt.Println("No match found. URL is not a short TTL path")
return false
}
// Helper function to handle common request logic
func makeRequest(ctx context.Context, reqFunc func(context.Context, string) (*retryablehttp.Request, error), token *token_manager.Token, url string) (*SimpleHTTPResponse, error) {
req, err := reqFunc(ctx, token.Value)
@@ -106,48 +141,16 @@ func makeRequest(ctx context.Context, reqFunc func(context.Context, string) (*re
}, nil
}
// generateCacheKey creates a unique cache key based on URL and user token
func generateCacheKey(url, userToken string) string {
return fmt.Sprintf("%s:%s", url, userToken)
}
// isShortTTLPath checks if the given URL should use the shorter TTL
func isShortTTLPath(rawURL string) bool {
// TODO: clean up debug statements
fmt.Printf("Checking if URL is a short TTL path: %s\n", rawURL)
parsedURL, err := url.Parse(rawURL)
if err != nil {
fmt.Printf("Error parsing URL: %v\n", err)
// If we can't parse the URL, we'll assume it's not a short TTL path
return false
}
// Extract the path and remove trailing slash if present
urlPath := path.Clean(parsedURL.Path)
fmt.Printf("Cleaned URL path: %s\n", urlPath)
for _, shortPath := range shortTTLPaths {
fmt.Printf("Comparing with short TTL path: %s\n", shortPath)
if urlPath == shortPath {
fmt.Printf("Match found! URL is a short TTL path\n")
return true
}
}
fmt.Println("No match found. URL is not a short TTL path")
return false
}
// retryRequest performs a request with automatic retries and token management
func retryRequest(
ctx context.Context,
reqFunc func(context.Context, string) (*retryablehttp.Request, error),
userToken string,
isPost bool,
url string, // used for logging in audit.LogAPIRoundTrip only
url string,
incomingHeaders http.Header,
) (*SimpleHTTPResponse, error) {
fmt.Printf("retryRequest: Received headers: %v\n", incomingHeaders)
var lastErr error
tokenManager := config.GlobalConfig.TokenManager
@@ -159,17 +162,36 @@ func retryRequest(
// For GET requests, check cache first
cacheKey := generateCacheKey(url, userToken)
cacheLock.RLock()
if cachedItem, found := cache.Get(cacheKey); found {
item := cachedItem.(CachedItem)
if time.Now().Before(item.ExpiresAt) {
cacheLock.RUnlock()
return item.Response, nil
}
}
cacheLock.RUnlock()
// If not in cache or expired, use the retry logic
// Check for X-Handled-Redirected in the incoming request
if redirectedHeader := incomingHeaders.Get("X-Handled-Redirected"); redirectedHeader != "" {
fmt.Printf("retryRequest: X-Handled-Redirected header found: %s\n", redirectedHeader)
if strings.Contains(strings.ToLower(redirectedHeader), "no-cache") {
fmt.Println("retryRequest: X-Handled-Redirected found, removing existing cache entry if present")
// Remove existing cache entry if present
cacheLock.Lock()
cache.Remove(cacheKey)
cacheLock.Unlock()
}
} else {
fmt.Println("retryRequest: No X-Handled-Redirected header found")
// Check cache if no-cache is not specified
cacheLock.RLock()
if cachedItem, found := cache.Get(cacheKey); found {
item := cachedItem.(CachedItem)
if time.Now().Before(item.ExpiresAt) {
fmt.Println("retryRequest: Valid cached item found, returning from cache")
cacheLock.RUnlock()
return item.Response, nil
}
fmt.Println("retryRequest: Cached item found but expired")
} else {
fmt.Println("retryRequest: No cached item found")
}
cacheLock.RUnlock()
}
// If not in cache, expired, or no-cache requested, use the retry logic
for i := 0; i < config.GlobalConfig.APIMaxRetries; i++ {
var token *token_manager.Token
if userToken != "" {
@@ -198,19 +220,21 @@ Please refer the following documentation for additional information:
if resp.StatusCode == http.StatusOK {
tokenManager.MarkTokenStatus(token, token_manager.Good)
// Cache the successful response
cacheLock.Lock()
var ttl time.Duration
if isShortTTLPath(url) {
ttl = config.GlobalConfig.CacheShortTTL
} else {
ttl = config.GlobalConfig.CacheTTL
// Cache the successful response only if no-cache was not requested
if incomingHeaders.Get("Cache-Control") == "" || !strings.Contains(strings.ToLower(incomingHeaders.Get("Cache-Control")), "no-cache") {
cacheLock.Lock()
var ttl time.Duration
if isShortTTLPath(url) {
ttl = config.GlobalConfig.CacheShortTTL
} else {
ttl = config.GlobalConfig.CacheTTL
}
cache.Add(cacheKey, CachedItem{
Response: resp,
ExpiresAt: time.Now().Add(ttl),
})
cacheLock.Unlock()
}
cache.Add(cacheKey, CachedItem{
Response: resp,
ExpiresAt: time.Now().Add(ttl),
})
cacheLock.Unlock()
return resp, nil
}
@@ -229,7 +253,12 @@ Please refer the following documentation for additional information:
}
// API_GET performs a GET request to the Pixiv API with automatic retries
func API_GET(ctx context.Context, url string, userToken string) (*SimpleHTTPResponse, error) {
func API_GET(
ctx context.Context,
url string,
userToken string,
incomingHeaders http.Header,
) (*SimpleHTTPResponse, error) {
return retryRequest(ctx, func(ctx context.Context, token string) (*retryablehttp.Request, error) {
req, err := retryablehttp.NewRequest("GET", url, nil)
if err != nil {
@@ -243,12 +272,12 @@ func API_GET(ctx context.Context, url string, userToken string) (*SimpleHTTPResp
Value: token,
})
return req, nil
}, userToken, false, url)
}, userToken, false, url, incomingHeaders)
}
// API_GET_UnwrapJson performs a GET request and unwraps the JSON response
func API_GET_UnwrapJson(ctx context.Context, url, userToken string) (string, error) {
resp, err := API_GET(ctx, url, userToken)
func API_GET_UnwrapJson(ctx context.Context, url, userToken string, incomingHeaders http.Header) (string, error) {
resp, err := API_GET(ctx, url, userToken, incomingHeaders)
if err != nil {
return "", err
}
@@ -293,6 +322,7 @@ func API_POST(
payload interface{},
userToken, csrf string,
contentType string,
incomingHeaders http.Header,
) (*SimpleHTTPResponse, error) {
if userToken == "" {
return nil, i18n.Error("userToken is required for POST requests")
@@ -333,7 +363,7 @@ func API_POST(
req.Header.Add("Content-Type", contentType)
return req, nil
}, userToken, true, url)
}, userToken, true, url, incomingHeaders)
if err != nil {
return nil, err
}
+2 -2
View File
@@ -76,7 +76,7 @@ func GetTagData(r *http.Request, name string) (TagDetail, error) {
URL := GetTagDetailURL(name)
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return tag, err
}
@@ -94,7 +94,7 @@ func GetTagData(r *http.Request, name string) (TagDetail, error) {
func GetSearch(r *http.Request, settings SearchPageSettings) (*SearchResult, error) {
URL := GetSearchArtworksURL(settings.ReturnMap())
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
+6 -6
View File
@@ -116,7 +116,7 @@ func GetFrequentTags(r *http.Request, ids string, category UserArtCategory) ([]F
URL = GetFrequentNovelTagsURL(ids)
}
response, err := API_GET_UnwrapJson(r.Context(), URL, "")
response, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
@@ -136,7 +136,7 @@ func GetUserArtworkList(r *http.Request, id, ids string) ([]ArtworkBrief, error)
URL := GetUserFullArtworkURL(id, ids)
resp, err := API_GET_UnwrapJson(r.Context(), URL, "")
resp, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
@@ -176,7 +176,7 @@ func GetUserNovels(r *http.Request, id, ids string) ([]NovelBrief, error) {
URL := GetUserFullNovelURL(id, ids)
resp, err := API_GET_UnwrapJson(r.Context(), URL, "")
resp, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, err
}
@@ -207,7 +207,7 @@ func GetUserNovels(r *http.Request, id, ids string) ([]NovelBrief, error) {
func GetUserArtworksIDAndSeries(r *http.Request, id string, category UserArtCategory, page int) (string, int, int, int, int, json.RawMessage, error) {
URL := GetUserArtworksURL(id)
resp, err := API_GET_UnwrapJson(r.Context(), URL, "")
resp, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return "", -1, -1, -1, -1, nil, err
}
@@ -350,7 +350,7 @@ func GetUserProfile(r *http.Request, id string, category UserArtCategory, page i
URL := GetUserInformationURL(id)
resp, err := API_GET_UnwrapJson(r.Context(), URL, token)
resp, err := API_GET_UnwrapJson(r.Context(), URL, token, r.Header)
if err != nil {
return user, err
}
@@ -453,7 +453,7 @@ func GetUserBookmarks(r *http.Request, id, mode string, page int) ([]ArtworkBrie
URL := GetUserBookmarksURL(id, mode, page)
resp, err := API_GET_UnwrapJson(r.Context(), URL, "")
resp, err := API_GET_UnwrapJson(r.Context(), URL, "", r.Header)
if err != nil {
return nil, -1, err
}
+1
View File
@@ -77,6 +77,7 @@ func main() {
router.Use(middleware.SetLocaleFromCookie) // needed for i18n.*()
router.Use(middleware.LogRequest) // all pages need this
router.Use(middleware.SetPrivacyHeaders) // all pages need this
router.Use(middleware.SetRedirectHeader) // needed to prevent a cached page from being served upon a user action
router.Use(middleware.HandleError) // if the inner handler fails, this shows the error page instead
router.Use(middleware.InitializeRateLimiter())
+22
View File
@@ -0,0 +1,22 @@
package middleware
import (
"net/http"
)
// SetRedirectHeader is a middleware that sets an X-Handled-Redirected
// HTTP header when a request contains the "redirected=1" query parameter.
func SetRedirectHeader(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Check if the request contains the "redirected" query parameter
redirected := r.URL.Query().Get("redirected")
if redirected == "1" {
// If redirected, set headers on the request
r.Header.Set("X-Handled-Redirected", "true")
}
// Call the next handler
h.ServeHTTP(w, r)
})
}
+5 -5
View File
@@ -40,7 +40,7 @@ func AddBookmarkRoute(w http.ResponseWriter, r *http.Request) error {
}`, id)
contentType := "application/json; charset=utf-8"
_, err := core.API_POST(r.Context(), URL, payload, token, csrf, contentType)
_, err := core.API_POST(r.Context(), URL, payload, token, csrf, contentType, r.Header)
if err != nil {
return err
}
@@ -70,7 +70,7 @@ func DeleteBookmarkRoute(w http.ResponseWriter, r *http.Request) error {
payload := fmt.Sprintf("bookmark_id=%s", id)
contentType := "application/x-www-form-urlencoded; charset=utf-8"
_, err := core.API_POST(r.Context(), URL, payload, token, csrf, contentType)
_, err := core.API_POST(r.Context(), URL, payload, token, csrf, contentType, r.Header)
if err != nil {
return err
}
@@ -100,7 +100,7 @@ func LikeRoute(w http.ResponseWriter, r *http.Request) error {
payload := fmt.Sprintf(`{"illust_id": "%s"}`, id)
contentType := "application/json; charset=utf-8"
_, err := core.API_POST(r.Context(), URL, payload, token, csrf, contentType)
_, err := core.API_POST(r.Context(), URL, payload, token, csrf, contentType, r.Header)
if err != nil {
return err
}
@@ -159,7 +159,7 @@ func FollowUserRoute(w http.ResponseWriter, r *http.Request) error {
"restrict": restrict,
"user_id": id,
}
_, err := core.API_POST(r.Context(), URL, fields, token, csrf, "")
_, err := core.API_POST(r.Context(), URL, fields, token, csrf, "", r.Header)
if err != nil {
return err
}
@@ -197,7 +197,7 @@ func UnfollowUserRoute(w http.ResponseWriter, r *http.Request) error {
"mode": "delete_bookmark_user",
"user_id": id,
}
_, err := core.API_POST(r.Context(), URL, fields, token, csrf, "")
_, err := core.API_POST(r.Context(), URL, fields, token, csrf, "", r.Header)
if err != nil {
return err
}
+2 -2
View File
@@ -23,14 +23,14 @@ func setToken(w http.ResponseWriter, r *http.Request) (string, error) {
if token != "" {
URL := core.GetNewestFromFollowingURL("all", "1")
_, err := core.API_GET_UnwrapJson(r.Context(), URL, token)
_, err := core.API_GET_UnwrapJson(r.Context(), URL, token, r.Header)
if err != nil {
return "", i18n.Error("Cannot authorize with supplied token.")
}
// Make a test request to verify the token.
// THE TEST URL IS NSFW!
resp, err := core.API_GET(r.Context(), "https://www.pixiv.net/en/artworks/115365120", token)
resp, err := core.API_GET(r.Context(), "https://www.pixiv.net/en/artworks/115365120", token, r.Header)
if err != nil {
return "", err
}
+8
View File
@@ -39,6 +39,14 @@ func RedirectTo(w http.ResponseWriter, r *http.Request, path string, query_param
func RedirectToWhenceYouCame(w http.ResponseWriter, r *http.Request) {
referrer := r.Referer()
if strings.HasPrefix(referrer, Origin(r)) {
// Append a query parameter indicating a redirect (if not already present).
referrerURL, err := url.Parse(referrer)
if err == nil && referrerURL.Query().Get("redirected") == "" {
q := referrerURL.Query()
q.Set("redirected", "1")
referrerURL.RawQuery = q.Encode()
referrer = referrerURL.String()
}
http.Redirect(w, r, referrer, http.StatusSeeOther)
} else {
w.WriteHeader(200)