From 85b75b74eaaced60e5b039a2c68c0ba02e695a2e Mon Sep 17 00:00:00 2001 From: perennial Date: Fri, 25 Oct 2024 16:40:53 +1100 Subject: [PATCH] refactor core/artwork to use sync/errgroup --- core/artwork.go | 287 +++++++++++++++++++----------------------------- go.mod | 1 + go.sum | 2 + 3 files changed, 115 insertions(+), 175 deletions(-) diff --git a/core/artwork.go b/core/artwork.go index 21122ef..1999f09 100644 --- a/core/artwork.go +++ b/core/artwork.go @@ -1,17 +1,18 @@ package core import ( - "errors" "fmt" "net/http" "sort" "strings" - "sync" "time" + "golang.org/x/sync/errgroup" + "github.com/goccy/go-json" "codeberg.org/vnpower/pixivfe/v2/audit" + "codeberg.org/vnpower/pixivfe/v2/i18n" "codeberg.org/vnpower/pixivfe/v2/server/session" ) @@ -263,205 +264,141 @@ func GetRelatedArtworks(auditor *audit.Auditor, r *http.Request, id string) ([]A } // GetArtworkByID retrieves information about a specific artwork, with the option to include additional related data. -// External API calls are made to gather metadata, images, related works, user info, and comments. func GetArtworkByID(auditor *audit.Auditor, r *http.Request, id string, full bool) (*Illust, error) { token := session.GetUserToken(r) - // Core artwork metadata fetched from Pixiv. + // Core artwork metadata fetched synchronously. var illust struct { Illust - UserIllusts map[int]any `json:"userIllusts"` // Allows fetching user's other artworks. - RawTags json.RawMessage `json:"tags"` // Retain untranslated tags for later processing. + UserIllusts map[int]any `json:"userIllusts"` // User's other artworks. + RawTags json.RawMessage `json:"tags"` // Untranslated tags. } - // Auxiliary data stores that are fetched asynchronously. - var illustAuxilary struct { - Images []Image // To store the artwork's images in various sizes. - RelatedWorks []ArtworkBrief // Place to gather related artworks, if requested. - CommentsList []Comment // Where fetched comments will be stored, if requested. - } - - // We use a WaitGroup to synchronize multiple concurrent API requests to different endpoints. - var wg sync.WaitGroup - cerr := make(chan error, 7) // Channel to gather errors from multiple API calls. - // Fetch core artwork metadata. - wg.Add(1) - go func() { - defer wg.Done() + urlArtInfo := GetArtworkInformationURL(id) + response, err := API_GET_UnwrapJson(r.Context(), auditor, urlArtInfo, token, r.Header) + if err != nil { + return nil, err + } - // Fetch core artwork metadata as the starting point for assembling all related information. - urlArtInfo := GetArtworkInformationURL(id) - response, err := API_GET_UnwrapJson(r.Context(), auditor, urlArtInfo, token, r.Header) - if err != nil { - cerr <- err - return - } + // Decode basic artwork data. + err = json.Unmarshal([]byte(response), &illust) + if err != nil { + return nil, err + } - // Decode basic artwork data. This forms the foundation for all subsequent requests. - err = json.Unmarshal([]byte(response), &illust) - if err != nil { - cerr <- err - return - } - - // If the artwork is bookmarked, store its BookmarkID for user-specific interactions. - if illust.BookmarkData != nil { - t := illust.BookmarkData.(map[string]any) + // If the artwork is bookmarked, store its BookmarkID. + if illust.BookmarkData != nil { + t, ok := illust.BookmarkData.(map[string]any) + if ok && t["id"] != nil { illust.BookmarkID = t["id"].(string) } + } - // Fetch user details like avatar and name (necessary because the user object isn’t always fully populated in the artwork data). - wg.Add(1) - go func() { - defer wg.Done() + // Use an errgroup to manage goroutines and errors. + var g errgroup.Group - // This request ensures that we have the user's avatar and other basic details, which might not always be present in the artwork info. - userInfo, err := GetUserBasicInformation(auditor, r, illust.UserID) - if err != nil { - cerr <- err - return - } - illust.User = userInfo - }() - - // Fetch available image sizes (e.g. small, medium, large, original). - // Art types vary (e.g. manga, illustrations), so this request happens after we obtain artwork metadata. - wg.Add(1) - go func() { - defer wg.Done() - - // Fetch image variants using the artwork ID and its IllustType. - images, err := GetArtworkImages(auditor, r, id, illust.IllustType) - if err != nil { - cerr <- err - return - } - illustAuxilary.Images = images - }() - - // If full details are requested, fetch related artworks. - if full { - wg.Add(1) - go func() { - defer wg.Done() - - // Fetch related artworks to help users explore similar content. - related, err := GetRelatedArtworks(auditor, r, id) - if err != nil { - cerr <- err - return - } - illustAuxilary.RelatedWorks = related - }() + // Fetch user basic information. + g.Go(func() error { + userInfo, err := GetUserBasicInformation(auditor, r, illust.UserID) + if err != nil { + return err } + illust.User = userInfo + return nil + }) - // The tags are stored untranslated; fetch translated tags in parallel with other metadata. - wg.Add(1) - go func() { - defer wg.Done() - - // Unmarshal the raw tag data and translate them so users can visualize the tags in their preferred language. - var tags struct { - Tags []struct { - Tag string `json:"tag"` - Translation map[string]string `json:"translation"` - } `json:"tags"` - } - - err := json.Unmarshal(illust.RawTags, &tags) - if err != nil { - cerr <- err - return - } - - // Store translated tags and preserve the original language form for international users. - var tagsList []ArtworkTag - for _, tag := range tags.Tags { - var newTag ArtworkTag - newTag.Name = tag.Tag - newTag.TranslatedName = tag.Translation["en"] - - tagsList = append(tagsList, newTag) - } - illust.Tags = tagsList - }() - - // If full details are requested, fetch the user's recent works. - if full { - wg.Add(1) - go func() { - defer wg.Done() - - // Fetch the list of the user's recent works by ID. Collect a limited number of artwork metadata (e.g., recent 20). - ids := make([]int, 0) - for k := range illust.UserIllusts { - ids = append(ids, k) - } - sort.Sort(sort.Reverse(sort.IntSlice(ids))) - - // Prepare the artwork ID string to request metadata for recent works. - idsString := "" - count := min(len(ids), 20) - for i := 0; i < count; i++ { - idsString += fmt.Sprintf("&ids[]=%d", ids[i]) - } - - // Fetch the user's recent artworks for display. - recent, err := fetchArtworkIDs(auditor, r, illust.UserID, idsString) - if err != nil { - cerr <- err - return - } - sort.Slice(recent[:], func(i, j int) bool { - left := recent[i].ID - right := recent[j].ID - return numberGreaterThan(left, right) - }) - illust.RecentWorks = recent - }() + // Fetch artwork images. + g.Go(func() error { + images, err := GetArtworkImages(auditor, r, id, illust.IllustType) + if err != nil { + return err } + illust.Images = images + return nil + }) - // We only fetch comments if full details are requested and comments are enabled. - // - // This condition is evaluated *after* fetching basic artwork information since we first need the comment-off flag. - if full && illust.CommentDisabled != 1 { - wg.Add(1) - go func() { - defer wg.Done() + // Fetch translated tags. + g.Go(func() error { + var tags struct { + Tags []struct { + Tag string `json:"tag"` + Translation map[string]string `json:"translation"` + } `json:"tags"` + } + err := json.Unmarshal(illust.RawTags, &tags) + if err != nil { + return err + } + // Store translated tags. + var tagsList []ArtworkTag + for _, tag := range tags.Tags { + newTag := ArtworkTag{ + Name: tag.Tag, + TranslatedName: tag.Translation["en"], + } + tagsList = append(tagsList, newTag) + } + illust.Tags = tagsList + return nil + }) + // Conditionally fetch related artworks if 'full' is true. + if full { + // Fetch related artworks. + g.Go(func() error { + related, err := GetRelatedArtworks(auditor, r, id) + if err != nil { + return err + } + illust.RelatedWorks = related + return nil + }) + + // Fetch recent works. + g.Go(func() error { + ids := make([]int, 0, len(illust.UserIllusts)) + for k := range illust.UserIllusts { + ids = append(ids, k) + } + sort.Sort(sort.Reverse(sort.IntSlice(ids))) + idsString := "" + count := min(len(ids), 20) + for i := 0; i < count; i++ { + idsString += fmt.Sprintf("&ids[]=%d", ids[i]) + } + recent, err := fetchArtworkIDs(auditor, r, illust.UserID, idsString) + if err != nil { + return err + } + sort.Slice(recent[:], func(i, j int) bool { + return numberGreaterThan(recent[i].ID, recent[j].ID) + }) + illust.RecentWorks = recent + return nil + }) + + // Fetch comments if comments are enabled. + if illust.CommentDisabled != 1 { + g.Go(func() error { comments, err := artworkCommentService.GetComments(auditor, r, id) if err != nil { - cerr <- err - return + return err } - illustAuxilary.CommentsList = comments - }() + illust.CommentsList = comments + return nil + }) } - }() - - // Wait until all the asynchronous requests complete their operations. - wg.Wait() - close(cerr) - - // Integrate the auxiliary data (images, related works, comments) back into the core `illust` structure. - illust.Images = illustAuxilary.Images - illust.RelatedWorks = illustAuxilary.RelatedWorks - illust.CommentsList = illustAuxilary.CommentsList - - // Handle all dispatched errors. If one or more requests failed, aggregate them into one summary error. - all_errors := []error{} - for suberr := range cerr { - all_errors = append(all_errors, suberr) - } - err_summary := errors.Join(all_errors...) - if err_summary != nil { - return nil, err_summary } - // If detected, flag this artwork as a ugoira for further handling by the Jet template engine. + // Wait for all goroutines to finish. + if err := g.Wait(); err != nil { + return nil, err + } + + // Check if the artwork is a ugoira. illust.IsUgoira = strings.Contains(illust.Images[0].Original, "ugoira") - // Return the fully assembled `illust` object, complete with metadata, images, related works, tags, and comments (if requested in full mode). + // Return the fully assembled 'illust' object. return &illust.Illust, nil } diff --git a/go.mod b/go.mod index bd1f370..58d2b58 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/zeebo/xxh3 v1.0.2 go.uber.org/zap v1.27.0 golang.org/x/net v0.30.0 + golang.org/x/sync v0.8.0 ) require ( diff --git a/go.sum b/go.sum index 28c9d09..0ef37af 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=