VnPower/PixivFE
1
0
Fork 0
mirror of https://codeberg.org/VnPower/PixivFE synced 2024-12-06 19:16:23 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add flamegraph page

Browse Source
This commit is contained in:
iacore
2024-08-29 14:11:51 +00:00
parent 45e93a9003
commit 13cd5929c1
29 changed files with 368 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
assets/js/flamegraph.js
+23
View File
@@ -0,0 +1,23 @@
const vlSpec = {
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"description": "flamegraph",
"data": {
"url": "/diagnostics/spans.json"
},
"width": "container",
"height": "container",
"config": {
"legend": {
"orient": "bottom",
}
},
"encoding": {
"y": { "field": "LogLine", "type": "nominal", "axis": null },
"x": { "field": "StartTime", "type": "temporal" },
"x2": { "field": "EndTime", "type": "temporal" },
"color": { "field": "RequestId", "type": "nominal" },
},
"mark": { "type": "bar", "tooltip": {"content": "data"} },
};
vegaEmbed('#vis', vlSpec);
assets/js/vega-embed@6.26.0.js
+7
View File
File diff suppressed because one or more lines are too long
assets/js/vega-lite@5.21.0.js
+2
View File
File diff suppressed because one or more lines are too long
assets/js/vega@5.30.0.js
+2
View File
File diff suppressed because one or more lines are too long
assets/views/diagnostics.jet.html
+11
View File
@@ -0,0 +1,11 @@
{{- extends "layout/default" }}
{{- block body() }}
<div class="container">
<p>Tip: Refresh page to see updated data. <a href="/diagnostics/reset">Reset Data</a></p>
<div id="vis" style="display: block; height: 100vh;"></div>
</div>
<script type="text/javascript" src="/js/vega@5.30.0.js"></script>
<script type="text/javascript" src="/js/vega-lite@5.21.0.js"></script>
<script type="text/javascript" src="/js/vega-embed@6.26.0.js"></script>
<script type="text/javascript" src="/js/flamegraph.js"></script>
{{- end }}
audit/spans.go
+90
View File
@@ -0,0 +1,90 @@
package audit
import (
"fmt"
"log"
"net/http"
"os"
"time"
)
type Span interface {
GetStartTime() time.Time
GetEndTime() time.Time
GetRequestId() string
LogLine() string
}
func Duration(span Span) time.Duration {
return span.GetEndTime().Sub(span.GetStartTime())
}
// logger with no timestamp prefix, because we control the timestamps
var Logger = log.New(os.Stderr, "", 0)
var RecordedSpans = []Span{}
// should be configurable. set to 0 to disable recording
var MaxRecordedCount = 128
func LogAndRecord(span Span) {
Logger.Printf("%v +%-5.3f %s", span.GetStartTime().Format("2006-01-02 15:04:05.000"), float64(Duration(span))/float64(time.Second), span.LogLine())
if MaxRecordedCount != 0 {
if len(RecordedSpans)+1 == MaxRecordedCount {
RecordedSpans = RecordedSpans[1:]
}
RecordedSpans = append(RecordedSpans, span)
}
}
type ServedRequestSpan struct {
StartTime time.Time
EndTime time.Time
RequestId string
Method string
Path string `json:"Url"`
Status int
Referer string
RemoteAddr string
Error error
}
func (span ServedRequestSpan) GetStartTime() time.Time {
return span.StartTime
}
func (span ServedRequestSpan) GetEndTime() time.Time {
return span.EndTime
}
func (span ServedRequestSpan) GetRequestId() string {
return span.RequestId
}
func (span ServedRequestSpan) LogLine() string {
return fmt.Sprintf("%v %v %v %v", span.Method, span.Path, span.Status, span.Error)
}
type APIRequestSpan struct {
StartTime time.Time
EndTime time.Time
RequestId string
Response *http.Response `json:"-"`
Error error
Method string
Url string
Token string
Body string `json:"-"`
ResponseFilename string
}
func (span APIRequestSpan) GetStartTime() time.Time {
return span.StartTime
}
func (span APIRequestSpan) GetEndTime() time.Time {
return span.EndTime
}
func (span APIRequestSpan) GetRequestId() string {
return span.RequestId
}
func (span APIRequestSpan) LogLine() string {
return fmt.Sprintf("-> %v %v %v %v", span.Method, span.Url, span.Error, span.ResponseFilename)
}
audit/tracing.go
+4 -5
View File
@@ -1,7 +1,6 @@
package audit
import (
"context"
"log"
"os"
"path"
@@ -25,15 +24,15 @@ func Init(saveResponse bool) error {
return nil
}
func LogServerRoundTrip(context context.Context, perf ServedRequestSpan) {
func LogServerRoundTrip(perf ServedRequestSpan) {
if perf.Error != nil {
log.Printf("Internal Server Error: %s", perf.Error)
}
Log(perf)
LogAndRecord(perf)
}
func LogAPIRoundTrip(context context.Context, perf APIRequestSpan) {
func LogAPIRoundTrip(perf APIRequestSpan) {
if perf.Response != nil {
if perf.Body != "" && optionSaveResponse {
var err error
@@ -47,7 +46,7 @@ func LogAPIRoundTrip(context context.Context, perf APIRequestSpan) {
}
}
Log(perf)
LogAndRecord(perf)
}
func writeResponseBodyToFile(body string) (string, error) {
audit/types.go
-70
View File
@@ -1,70 +0,0 @@
package audit
import (
"fmt"
"log"
"net/http"
"os"
"time"
)
type Span interface {
GetStartTime() time.Time
GetEndTime() time.Time
LogLine() string
}
func Duration(span Span) time.Duration {
return span.GetEndTime().Sub(span.GetStartTime())
}
// logger with no timestamp prefix, because we control the timestamps
var Logger = log.New(os.Stderr, "", 0)
func Log(span Span) {
Logger.Printf("%v +%-5.3f %s", span.GetStartTime().Format("2006-01-02 15:04:05.000"), float64(Duration(span))/float64(time.Second), span.LogLine())
}
type ServedRequestSpan struct {
StartTime time.Time
EndTime time.Time
RemoteAddr string
Method string
Path string
Status int
Error error
}
func (perf ServedRequestSpan) GetStartTime() time.Time {
return perf.StartTime
}
func (perf ServedRequestSpan) GetEndTime() time.Time {
return perf.EndTime
}
func (perf ServedRequestSpan) LogLine() string {
return fmt.Sprintf("%v %v %v %v", perf.Method, perf.Path, perf.Status, perf.Error)
}
type APIRequestSpan struct {
StartTime time.Time
EndTime time.Time
Response *http.Response
Error error
Method string
Url string
Token string
Body string
ResponseFilename string
}
func (perf APIRequestSpan) GetStartTime() time.Time {
return perf.StartTime
}
func (perf APIRequestSpan) GetEndTime() time.Time {
return perf.EndTime
}
func (perf APIRequestSpan) LogLine() string {
return fmt.Sprintf("-> %v %v %v %v", perf.Method, perf.Url, perf.Error, perf.ResponseFilename)
}
core/requests.go
+7 -2
View File
@@ -11,6 +11,7 @@ import (
"codeberg.org/vnpower/pixivfe/v2/audit"
config "codeberg.org/vnpower/pixivfe/v2/config"
"codeberg.org/vnpower/pixivfe/v2/request_context"
"codeberg.org/vnpower/pixivfe/v2/utils"
"github.com/tidwall/gjson"
)
@@ -25,7 +26,9 @@ func API_GET(context context.Context, url string, token string) (SimpleHTTPRespo
start_time := time.Now()
res, resp, err := _API_GET(context, url, token)
end_time := time.Now()
audit.LogAPIRoundTrip(context, audit.APIRequestSpan{Response: resp, Error: err, Method: "GET", Url: url, Token: token, Body: res.Body, StartTime: start_time, EndTime: end_time})
audit.LogAPIRoundTrip(audit.APIRequestSpan{
RequestId: request_context.GetFromContext(context).RequestId,
Response: resp, Error: err, Method: "GET", Url: url, Token: token, Body: res.Body, StartTime: start_time, EndTime: end_time})
if err != nil {
return SimpleHTTPResponse{}, fmt.Errorf("While GET %s: %w", url, err)
}
@@ -102,7 +105,9 @@ func API_POST(context context.Context, url, payload, token, csrf string, isJSON
start_time := time.Now()
resp, err := _API_POST(context, url, payload, token, csrf, isJSON)
end_time := time.Now()
audit.LogAPIRoundTrip(context, audit.APIRequestSpan{Response: resp, Error: err, Method: "POST", Url: url, Token: token, Body: "", StartTime: start_time, EndTime: end_time})
audit.LogAPIRoundTrip(audit.APIRequestSpan{
RequestId: request_context.GetFromContext(context).RequestId,
Response: resp, Error: err, Method: "POST", Url: url, Token: token, Body: "", StartTime: start_time, EndTime: end_time})
if err != nil {
return fmt.Errorf("While POST %s: %w", url, err)
}
doc/dev/features/tracing-flamegraph.md
+16
View File
@@ -0,0 +1,16 @@
# About tracing in PixivFE
Every request to pixiv websites should go through core/requests.go.
Every request to pixiv websites is traced.
Every server request is traced.
## How to see flamegraph
Run PixivFE in dev mode, visit some pages, then visit URL /diagnostics.
## Useful for fixing Vega-Lite
https://vega.github.io/vega-lite/examples/interactive_legend.html
https://vega.github.io/editor/#/examples/vega-lite/interactive_legend
https://vega.github.io/vega-lite/docs/tooltip.html
doc/dev/features/tracing.md
-10
View File
@@ -1,10 +0,0 @@
# About tracing in PixivFE
Every request to pixiv websites should go through core/requests.go.
Every request to pixiv websites is traced.
Every server request is traced.
## How to see flamegraph
(todo) Run PixivFE in dev mode, then visit URL /debug/flamegraph.
go.mod
+1
View File
@@ -11,6 +11,7 @@ require (
github.com/gorilla/mux v1.8.1
github.com/oklog/ulid/v2 v2.1.0
github.com/playwright-community/playwright-go v0.4501.1
github.com/soluble-ai/go-jnode v0.1.11
github.com/tidwall/gjson v1.17.3
golang.org/x/net v0.28.0
golang.org/x/time v0.6.0
go.sum
+3
View File
@@ -33,6 +33,8 @@ github.com/playwright-community/playwright-go v0.4501.1 h1:kz8SIfR6nEI8blk77nTVD
github.com/playwright-community/playwright-go v0.4501.1/go.mod h1:bpArn5TqNzmP0jroCgw4poSOG9gSeQg490iLqWAaa7w=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/soluble-ai/go-jnode v0.1.11 h1:To9h6zWI6qr5Vphz0XO4JtCRlkhfilH2EDNwtAZkZT4=
github.com/soluble-ai/go-jnode v0.1.11/go.mod h1:mKFUM7xxXMBVOygZeE8od2Qls1f3E3NHY1ZhMlzwx/U=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
@@ -92,6 +94,7 @@ golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
handlers/csp.go
+4 -1
View File
@@ -3,6 +3,7 @@ package handlers
import (
"fmt"
"net/http"
"strings"
"codeberg.org/vnpower/pixivfe/v2/session"
)
@@ -15,9 +16,11 @@ func SetPrivacyHeaders(h http.Handler) http.Handler {
// use this if need iframe: `X-Frame-Options: SAMEORIGIN`
header.Add("X-Content-Type-Options", "nosniff")
header.Add("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
header.Add("Content-Security-Policy", fmt.Sprintf("base-uri 'self'; default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' %s; media-src 'self' %s; connect-src 'self'; form-action 'self'; frame-ancestors 'none';", session.GetImageProxyOrigin(r), session.GetImageProxyOrigin(r)))
// use this if need iframe: `frame-ancestors 'self'`
header.Add("Permissions-Policy", "accelerometer=(), ambient-light-sensor=(), battery=(), camera=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), web-share=(), xr-spatial-tracking=()")
if !strings.HasPrefix(r.URL.Path, "/diagnostics") {
header.Add("Content-Security-Policy", fmt.Sprintf("base-uri 'self'; default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' %s; media-src 'self' %s; connect-src 'self'; form-action 'self'; frame-ancestors 'none';", session.GetImageProxyOrigin(r), session.GetImageProxyOrigin(r)))
}
h.ServeHTTP(w, r)
})
handlers/error_handler.go
+4 -10
View File
@@ -2,13 +2,13 @@ package handlers
import (
"bytes"
"log"
"maps"
"net/http"
"net/http/httptest"
"slices"
"codeberg.org/vnpower/pixivfe/v2/routes"
"codeberg.org/vnpower/pixivfe/v2/request_context"
)
func CatchError(handler func(w http.ResponseWriter, r *http.Request) error) http.HandlerFunc {
@@ -26,7 +26,7 @@ func CatchError(handler func(w http.ResponseWriter, r *http.Request) error) http
if err != nil {
clear(header_backup)
maps.Copy(w.Header(), header_backup)
GetUserContext(r).Error = err
request_context.Get(r).CaughtError = err
} else {
w.WriteHeader(recorder.Code)
_, _ = recorder.Body.WriteTo(w)
@@ -38,16 +38,10 @@ func HandleError(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h.ServeHTTP(w, r)
err := GetUserContext(r).Error
err := request_context.Get(r).CaughtError
if err != nil {
code := GetUserContext(r).ErrorStatusCode
w.WriteHeader(code)
// Send custom error page
err = routes.ErrorPage(w, r, err)
if err != nil {
log.Panicf("[fix this ASAP] Error rendering error route: %s", err)
}
routes.ErrorPage(w, r, err, http.StatusInternalServerError)
}
})
}
handlers/limiter.go
+2 -13
View File
@@ -19,8 +19,7 @@ func CanRequestSkipLimiter(r *http.Request) bool {
return strings.HasPrefix(path, "/img/") ||
strings.HasPrefix(path, "/css/") ||
strings.HasPrefix(path, "/js/") ||
strings.HasPrefix(path, "/proxy/s.pximg.net/") ||
strings.HasPrefix(path, "/favicon.ico")
strings.HasPrefix(path, "/proxy/s.pximg.net/")
}
// Todo: Should we put middlewares in a separate file?
@@ -79,17 +78,7 @@ func RateLimitRequest(h http.Handler) http.Handler {
}
if !limiter.Allow(ip) {
CatchError(func(w http.ResponseWriter, r *http.Request) error {
err := errors.New("Too many requests")
GetUserContext(r).Error = err
GetUserContext(r).ErrorStatusCode = http.StatusTooManyRequests
err = routes.ErrorPage(w, r, err)
if err != nil {
println("Error rendering error route: %s", err)
}
return err
})(w, r)
routes.ErrorPage(w, r, errors.New("Too many requests"), http.StatusTooManyRequests)
} else {
h.ServeHTTP(w, r)
}
handlers/logger.go
+7 -3
View File
@@ -7,6 +7,7 @@ import (
"codeberg.org/vnpower/pixivfe/v2/audit"
"codeberg.org/vnpower/pixivfe/v2/config"
"codeberg.org/vnpower/pixivfe/v2/request_context"
)
type ResponseWriterInterceptStatus struct {
@@ -25,6 +26,7 @@ func CanRequestSkipLogger(r *http.Request) bool {
return strings.HasPrefix(path, "/img/") ||
strings.HasPrefix(path, "/css/") ||
strings.HasPrefix(path, "/js/") ||
strings.HasPrefix(path, "/diagnostics") ||
(config.GlobalConfig.InDevelopment &&
(strings.HasPrefix(path, "/proxy/s.pximg.net/") || strings.HasPrefix(path, "/proxy/i.pximg.net/")))
}
@@ -46,14 +48,16 @@ func LogRequest(h http.Handler) http.Handler {
end_time := time.Now()
audit.LogServerRoundTrip(r.Context(), audit.ServedRequestSpan{
audit.LogServerRoundTrip(audit.ServedRequestSpan{
StartTime: start_time,
EndTime: end_time,
RemoteAddr: r.RemoteAddr,
RequestId: request_context.Get(r).RequestId,
Method: r.Method,
Path: r.URL.Path,
Status: w.statusCode,
Error: GetUserContext(r).Error,
Referer: r.Referer(),
RemoteAddr: r.RemoteAddr,
Error: request_context.Get(r).CaughtError,
})
}
})
handlers/router.go
+22 -14
View File
@@ -9,7 +9,7 @@ import (
"github.com/gorilla/mux"
)
func handlePrefix(router *mux.Router, pathPrefix string, handler http.Handler) *mux.Route {
func handleStripPrefix(router *mux.Router, pathPrefix string, handler http.Handler) *mux.Route {
return router.PathPrefix(pathPrefix).Handler(http.StripPrefix(pathPrefix, handler))
}
@@ -32,30 +32,39 @@ func DefineRoutes() *mux.Router {
})
router.HandleFunc("/robots.txt", serveFile("./assets/robots.txt"))
handlePrefix(router, "/img/", http.FileServer(http.Dir("./assets/img")))
handlePrefix(router, "/css/", http.FileServer(http.Dir("./assets/css")))
handlePrefix(router, "/js/", http.FileServer(http.Dir("./assets/js")))
handleStripPrefix(router, "/img/", http.FileServer(http.Dir("./assets/img")))
handleStripPrefix(router, "/css/", http.FileServer(http.Dir("./assets/css")))
handleStripPrefix(router, "/js/", http.FileServer(http.Dir("./assets/js")))
// Proxy routes. cache headers set by upstream servers.
handlePrefix(router, "/proxy/i.pximg.net/", CatchError(routes.IPximgProxy)).Methods("GET")
handlePrefix(router, "/proxy/s.pximg.net/", CatchError(routes.SPximgProxy)).Methods("GET")
handlePrefix(router, "/proxy/ugoira.com/", CatchError(routes.UgoiraProxy)).Methods("GET")
handleStripPrefix(router, "/proxy/i.pximg.net/", CatchError(routes.IPximgProxy)).Methods("GET")
handleStripPrefix(router, "/proxy/s.pximg.net/", CatchError(routes.SPximgProxy)).Methods("GET")
handleStripPrefix(router, "/proxy/ugoira.com/", CatchError(routes.UgoiraProxy)).Methods("GET")
router.HandleFunc("/", CatchError(routes.IndexPage)).Methods("GET")
router.HandleFunc("/about", CatchError(routes.AboutPage)).Methods("GET")
router.HandleFunc("/newest", CatchError(routes.NewestPage)).Methods("GET")
router.HandleFunc("/discovery", CatchError(routes.DiscoveryPage)).Methods("GET")
router.HandleFunc("/discovery/novel", CatchError(routes.NovelDiscoveryPage)).Methods("GET")
router.HandleFunc("/ranking", CatchError(routes.RankingPage)).Methods("GET")
router.HandleFunc("/rankingCalendar", CatchError(routes.RankingCalendarPage)).Methods("GET")
router.HandleFunc("/rankingCalendar", CatchError(routes.RankingCalendarPicker)).Methods("POST")
router.HandleFunc("/users/{id}.atom.xml", CatchError(routes.UserAtomFeed)).Methods("GET")
router.HandleFunc("/users/{id}/{category}.atom.xml", CatchError(routes.UserAtomFeed)).Methods("GET")
router.HandleFunc("/users/{id}", CatchError(routes.UserPage)).Methods("GET")
router.HandleFunc("/users/{id}/{category}", CatchError(routes.UserPage)).Methods("GET")
router.HandleFunc("/artworks/{id}", CatchError(routes.ArtworkPage)).Methods("GET")
router.HandleFunc("/artworks-multi/{ids}", CatchError(routes.ArtworkMultiPage)).Methods("GET")
// Legacy illust URL
router.HandleFunc("/member_illust.php", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/artworks/"+routes.GetQueryParam(r, "illust_id"), http.StatusPermanentRedirect)
}).Methods("GET")
router.HandleFunc("/novel/{id}", CatchError(routes.NovelPage)).Methods("GET")
router.HandleFunc("/pixivision", CatchError(routes.PixivisionHomePage)).Methods("GET")
router.HandleFunc("/pixivision/a/{id}", CatchError(routes.PixivisionArticlePage)).Methods("GET")
@@ -76,15 +85,14 @@ func DefineRoutes() *mux.Router {
router.HandleFunc("/tags", CatchError(routes.TagPage)).Methods("GET")
router.HandleFunc("/tags", CatchError(routes.AdvancedTagPost)).Methods("POST")
// Legacy illust URL
router.HandleFunc("/member_illust.php", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/artworks/"+routes.GetQueryParam(r, "illust_id"), http.StatusPermanentRedirect)
}).Methods("GET")
router.HandleFunc("/diagnostics", CatchError(routes.Diagnostics)).Methods("GET")
router.HandleFunc("/diagnostics/spans.json", CatchError(routes.DiagnosticsData)).Methods("GET")
router.HandleFunc("/diagnostics/reset", routes.ResetDiagnosticsData)
// fallback route (if nothing else matches)
router.NewRoute().HandlerFunc(CatchError(func(w http.ResponseWriter, r *http.Request) error {
return errors.New("Route not found")
}))
router.NewRoute().HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
routes.ErrorPage(w, r, errors.New("Route not found"), http.StatusNotFound)
})
return router
}
handlers/set_context.go
+2 -8
View File
@@ -3,17 +3,11 @@ package handlers
import (
"net/http"
"codeberg.org/vnpower/pixivfe/v2/handlers/user_context"
"codeberg.org/vnpower/pixivfe/v2/request_context"
)
type UserContext = user_context.UserContext
func GetUserContext(r *http.Request) *UserContext {
return user_context.GetUserContext(r.Context())
}
func ProvideUserContext(h http.Handler) http.Handler {
return http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) {
h.ServeHTTP(w, r.WithContext(user_context.WithContext(r.Context())))
h.ServeHTTP(w, r.WithContext(request_context.ProvideWith(r.Context())))
})
}
handlers/user_context/user_context.go
-36
View File
@@ -1,36 +0,0 @@
// pain: why go no cyclic import
package user_context
import (
"context"
"net/http"
"github.com/oklog/ulid/v2"
)
type UserContextKeyType struct{}
var UserContextKey = UserContextKeyType{}
type UserContext struct {
RequestId string
Error error
ErrorStatusCode int
}
func MakeUserContext() UserContext {
return UserContext{
RequestId: ulid.Make().String(),
ErrorStatusCode: http.StatusInternalServerError,
}
}
func WithContext(ctx context.Context) context.Context {
uc := MakeUserContext()
return context.WithValue(ctx, UserContextKey, &uc)
}
func GetUserContext(context context.Context) *UserContext {
return context.Value(UserContextKey).(*UserContext)
}
main.go
+5 -4
View File
@@ -29,11 +29,12 @@ func main() {
handlers.InitializeRateLimiter()
router := handlers.DefineRoutes()
router.Use(handlers.ProvideUserContext) // most outer / first executed middleware
router.Use(handlers.LogRequest)
router.Use(handlers.SetPrivacyHeaders)
// the first middleware is the most outer / first executed one
router.Use(handlers.ProvideUserContext) // needed for everything else
router.Use(handlers.LogRequest) // all pages need this
router.Use(handlers.SetPrivacyHeaders) // all pages need this
router.Use(handlers.HandleError) // if the inner handler fails, this shows the error page instead
router.Use(handlers.RateLimitRequest)
router.Use(handlers.HandleError)
// watch and compile sass when in development mode
if config.GlobalConfig.InDevelopment {
request_context/request_context.go
+43
View File
@@ -0,0 +1,43 @@
// this package is separate because Go disallows cyclic import. pain
package request_context
import (
"context"
"net/http"
"github.com/oklog/ulid/v2"
)
type userContextKeyType struct{}
var userContextKey = userContextKeyType{}
type RequestContext struct {
// for tracing
RequestId string
// for error handling. normal error page don't need to set this.
CaughtError error
// for Render[T]
RenderStatusCode int
}
func Make() RequestContext {
return RequestContext{
RequestId: ulid.Make().String(),
RenderStatusCode: 200,
}
}
func ProvideWith(ctx context.Context) context.Context {
uc := Make()
return context.WithValue(ctx, userContextKey, &uc)
}
func GetFromContext(context context.Context) *RequestContext {
return context.Value(userContextKey).(*RequestContext)
}
func Get(r *http.Request) *RequestContext {
return GetFromContext(r.Context())
}
routes/diagnostics.go
+39
View File
@@ -0,0 +1,39 @@
package routes
import (
"net/http"
"github.com/goccy/go-json"
"github.com/soluble-ai/go-jnode"
"codeberg.org/vnpower/pixivfe/v2/audit"
"codeberg.org/vnpower/pixivfe/v2/utils"
)
func Diagnostics(w http.ResponseWriter, r *http.Request) error {
return Render(w, r, Data_diagnostics{})
}
func ResetDiagnosticsData(w http.ResponseWriter, r *http.Request) {
audit.RecordedSpans = audit.RecordedSpans[:0]
utils.RedirectToWhenceYouCame(w, r)
}
func DiagnosticsData(w http.ResponseWriter, _ *http.Request) error {
data := jnode.NewArrayNode()
for _, span := range audit.RecordedSpans {
bytes, err := json.Marshal(span)
if err != nil {
return err
}
obj, err := jnode.FromJSON(bytes)
if err != nil {
return err
}
obj.Put("LogLine", span.LogLine())
data.Append(obj)
}
w.Header().Set("content-type", "application/json")
w.WriteHeader(200)
return json.NewEncoder(w).Encode(data)
}
routes/error.go
+8 -2
View File
@@ -1,11 +1,17 @@
package routes
import (
"log"
"net/http"
"codeberg.org/vnpower/pixivfe/v2/template"
"codeberg.org/vnpower/pixivfe/v2/request_context"
)
func ErrorPage(w http.ResponseWriter, r *http.Request, err error) error {
return template.Render(w, r, Data_error{Title: "Error", Error: err})
func ErrorPage(w http.ResponseWriter, r *http.Request, err error, statusCode int) {
request_context.Get(r).RenderStatusCode = statusCode
err = template.Render(w, r, Data_error{Title: "Error", Error: err})
if err != nil {
log.Printf("Error rendering error route: %s", err)
}
}
routes/personal.go
+2 -1
View File
@@ -7,14 +7,15 @@ import (
"codeberg.org/vnpower/pixivfe/v2/core"
"codeberg.org/vnpower/pixivfe/v2/session"
"codeberg.org/vnpower/pixivfe/v2/request_context"
)
func PromptUserToLoginPage(w http.ResponseWriter, r *http.Request) error {
request_context.Get(r).RenderStatusCode = http.StatusUnauthorized
err := Render(w, r, Data_unauthorized{})
if err != nil {
return err
}
w.WriteHeader(http.StatusUnauthorized)
return nil
}
routes/types.go
+34 -31
View File
@@ -14,13 +14,11 @@ func Render[T any](w http.ResponseWriter, r *http.Request, data T) error {
// Tutorial: adding new types in this file
// Whenever you add new types, update `TestTemplates` in render_test.go to include the type in the test
// Do not use pointer in Data_* struct. faker will insert nil.
// Do not name template file a.b.jet.html or it won't be able to be used here, since Data_a.b is not a valid identifier.
//
// Warnings:
// - Do not use pointer in Data_* struct. faker will insert nil.
// - Do not name template file a.b.jet.html or it won't be able to be used here, since Data_a.b is not a valid identifier.
type Data_error struct {
Title string
Error error
}
type Data_about struct {
Time string
Version string
@@ -39,15 +37,22 @@ type Data_artworkMulti struct {
Artworks []core.Illust
Title string
}
type Data_userAtom struct {
URL string
Title string
User core.User
Category core.UserArtCategory
Updated string
PageLimit int
Page int
// MetaImage string
type Data_diagnostics struct {}
type Data_discovery struct {
Artworks []core.ArtworkBrief
Title string
Queries template.PartialURL
}
type Data_error struct {
Title string
Error error
}
type Data_following struct {
Title string
Mode string
Artworks []core.ArtworkBrief
CurPage string
Page int
}
type Data_index struct {
Title string
@@ -55,16 +60,6 @@ type Data_index struct {
Data core.LandingArtworks
NoTokenData core.Ranking
}
type Data_unauthorized struct{}
type Data_discovery struct {
Artworks []core.ArtworkBrief
Title string
Queries template.PartialURL
}
type Data_novelDiscovery struct {
Novels []core.NovelBrief
Title string
}
type Data_newest struct {
Items []core.ArtworkBrief
Title string
@@ -78,12 +73,9 @@ type Data_novel struct {
ViewMode string
Language string
}
type Data_following struct {
Title string
Mode string
Artworks []core.ArtworkBrief
CurPage string
Page int
type Data_novelDiscovery struct {
Novels []core.NovelBrief
Title string
}
type Data_pixivision_index struct {
Data []pixivision.Article
@@ -120,6 +112,7 @@ type Data_tag struct {
TrueTag string
Page int
}
type Data_unauthorized struct{}
type Data_user struct {
Title string
User core.User
@@ -128,3 +121,13 @@ type Data_user struct {
Page int
MetaImage string
}
type Data_userAtom struct {
URL string
Title string
User core.User
Category core.UserArtCategory
Updated string
PageLimit int
Page int
// MetaImage string
}
template/render.go
+2 -1
View File
@@ -8,6 +8,7 @@ import (
"strings"
"codeberg.org/vnpower/pixivfe/v2/session"
"codeberg.org/vnpower/pixivfe/v2/request_context"
"codeberg.org/vnpower/pixivfe/v2/utils"
"github.com/CloudyKit/jet/v6"
@@ -37,7 +38,7 @@ func Render[T any](w http.ResponseWriter, r *http.Request, data T) error {
w.Header().Set("content-type", "text/html; charset=utf-8")
// todo: think about caching a bit more. see doc/dev/features/caching.md
// w.Header().Set("expires", time.Now().Add(config.ExpiresIn).Format(time.RFC1123))
w.WriteHeader(200)
w.WriteHeader(request_context.Get(r).RenderStatusCode)
return RenderInner(w, GetTemplatingVariables(r), data)
}
template/render_test.go
+28 -27
View File
@@ -15,29 +15,26 @@ import (
)
func TestTemplates(t *testing.T) {
{
data := fakeData[Data_error]()
data.Error = io.EOF
manualTest(t, data)
}
autoTest[Data_about](t)
autoTest[Data_artwork](t)
autoTest[Data_artworkMulti](t)
autoTest[Data_discovery](t)
autoTest[Data_following](t)
autoTest[Data_index](t)
autoTest[Data_newest](t)
autoTest[Data_novel](t)
autoTest[Data_novelDiscovery](t)
autoTest[Data_pixivision_article](t)
autoTest[Data_pixivision_index](t)
autoTest[Data_rank](t)
autoTest[Data_rankingCalendar](t)
autoTest[Data_settings](t)
autoTest[Data_tag](t)
autoTest[Data_unauthorized](t)
autoTest[Data_user](t)
autoTest[Data_userAtom](t)
test[Data_about](t)
test[Data_artwork](t)
test[Data_artworkMulti](t)
test[Data_diagnostics](t)
test[Data_discovery](t)
test[Data_error](t, Data_error{Title: fakeData[string](), Error: io.EOF})
test[Data_following](t)
test[Data_index](t)
test[Data_newest](t)
test[Data_novel](t)
test[Data_novelDiscovery](t)
test[Data_pixivision_article](t)
test[Data_pixivision_index](t)
test[Data_rank](t)
test[Data_rankingCalendar](t)
test[Data_settings](t)
test[Data_tag](t)
test[Data_unauthorized](t)
test[Data_user](t)
test[Data_userAtom](t)
}
func TestMain(m *testing.M) {
@@ -56,12 +53,16 @@ func fakeData[T any]() T {
return data
}
// autoTest template with fake data
func autoTest[T any](t *testing.T) {
manualTest(t, fakeData[T]())
// test template with fake data
func test[T any](t *testing.T, data ...T) {
if len(data) == 0 {
testWith(t, fakeData[T]())
} else {
testWith(t, data[0])
}
}
func manualTest[T any](t *testing.T, data T) {
func testWith[T any](t *testing.T, data T) {
route_name, found := strings.CutPrefix(reflect.TypeFor[T]().Name(), "Data_")
if !found {
log.Panicf("struct name does not start with 'Data_': %s", route_name)
utils/kmutex/kmutex.go
-52
View File
@@ -1,52 +0,0 @@
package kmutex
import "sync"
// Map, Refencence-counted by ID (any).
type Kmutex struct {
Map sync.Map
}
// Create new Kmutex
func New() *Kmutex {
return &Kmutex{}
}
// decrement ID ref count
// Returns: ref count after
func (km *Kmutex) Unlock(key any) uint64 {
for {
actual, ok := km.Map.Load(key)
if !ok {
panic("impossible! memory corruption?")
}
if actual.(uint64) == 1 {
deleted := km.Map.CompareAndDelete(key, actual.(uint64))
if deleted {
return 0
}
} else {
after := actual.(uint64) - 1
swapped := km.Map.CompareAndSwap(key, actual.(uint64), after)
if swapped {
return after
}
}
}
}
// increment ID ref count
// Returns: ref count after
func (km *Kmutex) Lock(key any) uint64 {
for {
actual, loaded := km.Map.LoadOrStore(key, uint64(1))
if !loaded {
return 1
}
after := actual.(uint64) + 1
swapped := km.Map.CompareAndSwap(key, actual.(uint64), after)
if swapped {
return after
}
}
}