diff --git a/core/rankingCalendar.go b/core/rankingCalendar.go index c953b44..3138b91 100644 --- a/core/rankingCalendar.go +++ b/core/rankingCalendar.go @@ -2,14 +2,15 @@ package core import ( "fmt" - "html/template" "net/http" "time" + "golang.org/x/net/html" + "github.com/andybalholm/cascadia" + "codeberg.org/vnpower/pixivfe/v2/session" "codeberg.org/vnpower/pixivfe/v2/utils" - "golang.org/x/net/html" ) func get_weekday(n time.Weekday) int { @@ -32,16 +33,18 @@ func get_weekday(n time.Weekday) int { return 0 } +var selector_img = cascadia.MustCompile("img") + // note(@iacore): // so the funny thing about Pixiv is that they will return this month's data for a request of a future date // is it a bug or a feature? -func GetRankingCalendar(r *http.Request, mode string, year, month int) (template.HTML, error) { +func GetRankingCalendar(r *http.Request, mode string, year, month int) (HTML, error) { token := session.GetPixivToken(r) URL := GetRankingCalendarURL(mode, year, month) req, err := http.NewRequestWithContext(r.Context(), "GET", URL, nil) if err != nil { - return template.HTML(""), err + return HTML(""), err } req.Header.Add("User-Agent", "Mozilla/5.0") req.Header.Add("Cookie", "PHPSESSID="+token) @@ -61,26 +64,17 @@ func GetRankingCalendar(r *http.Request, mode string, year, month int) (template if err != nil { return "", err } - + // Find and print all links on the web page var links []string - var link func(*html.Node) - link = func(n *html.Node) { - if n.Type == html.ElementNode && n.Data == "img" { - for _, a := range n.Attr { - if a.Key == "data-src" { - // adds a new link entry when the attribute matches - links = append(links, session.ProxyImageUrlNoEscape(r, a.Val)) - } + for _, node := range cascadia.QueryAll(doc, selector_img) { + for _, attr := range node.Attr { + if attr.Key == "data-src" { + // adds a new link entry when the attribute matches + links = append(links, session.ProxyImageUrlNoEscape(r, attr.Val)) } } - - // traverses the HTML of the webpage from the first child node - for r := n.FirstChild; r != nil; r = r.NextSibling { - link(r) - } } - link(doc) // now := r.Context().Time() // yearNow := now.Year() @@ -100,5 +94,5 @@ func GetRankingCalendar(r *http.Request, mode string, year, month int) (template renderString += fmt.Sprintf(`
%d
`, i+1) } } - return template.HTML(renderString), nil + return HTML(renderString), nil } diff --git a/go.mod b/go.mod index 3aee8d2..f4d614f 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,10 @@ go 1.22.5 require ( codeberg.org/vnpower/pixivision v0.1.1 github.com/CloudyKit/jet/v6 v6.2.0 + github.com/andybalholm/cascadia v1.3.2 github.com/go-faker/faker/v4 v4.4.2 github.com/goccy/go-json v0.10.3 + github.com/gorilla/mux v1.8.1 github.com/playwright-community/playwright-go v0.4501.1 github.com/tidwall/gjson v1.17.3 golang.org/x/net v0.28.0 @@ -15,15 +17,12 @@ require ( require ( github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect github.com/PuerkitoBio/goquery v1.9.2 // indirect - github.com/andybalholm/cascadia v1.3.2 // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-stack/stack v1.8.1 // indirect - github.com/gorilla/mux v1.8.1 - github.com/stretchr/testify v1.9.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/text v0.17.0 // indirect ) diff --git a/go.sum b/go.sum index 7c3c1a2..7241c56 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -47,8 +47,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/semgrep.yml b/semgrep.yml index 8b3644e..9564a07 100644 --- a/semgrep.yml +++ b/semgrep.yml @@ -153,3 +153,14 @@ rules: patterns: - pattern: | StatusFound +- id: rule-12 + message: "response body not closed" + comment: "Needed to reuse connections" + languages: [go] + severity: ERROR + patterns: + - pattern: | + ($RESP : *http.Response), $ERR := ... + - pattern-not-inside: | + ... + defer $RESP.Body.Close()