diff --git a/README.md b/README.md index d7f2516..7af957f 100644 --- a/README.md +++ b/README.md @@ -63,9 +63,11 @@ gothub.example.com { reverse_proxy localhost:(the port for GotHub) } ``` +## Scraping +Variables: 1) GOTHUB_USER_AGENT - Sets the User Agent to use for making the requests. Defaults to Chrome 110 on Windows 10. ## DMCA/Legal notice All content shown on a GotHub instance is from GitHub. Any issues with content shown on a GotHub instance needs to be reported to GitHub, not the instance host's internet or domain provider. -GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc. \ No newline at end of file +GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc. diff --git a/go.mod b/go.mod index d6aaab7..d0c3642 100644 --- a/go.mod +++ b/go.mod @@ -14,18 +14,32 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.3.0 // indirect + github.com/antchfx/xmlquery v1.3.15 // indirect + github.com/antchfx/xpath v1.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/enescakir/emoji v1.0.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/gocolly/colly v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jawher/mow.cli v1.2.0 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.15.14 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/rivo/uniseg v0.4.3 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -33,4 +47,7 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 844adec..de5ddd8 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= @@ -66,6 +68,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= +github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= +github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= +github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= +github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -114,6 +124,8 @@ github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55k github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= +github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -139,6 +151,10 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.40.1/go.mod h1:Gko04sLksnHbzLSRBFWPFdzM9Ws9pRxvvIaohJK1dsk= github.com/gofiber/fiber/v2 v2.41.0 h1:YhNoUS/OTjEz+/WLYuQ01xI7RXgKEFnGBKMagAu5f0M= @@ -151,6 +167,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -177,6 +194,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK0Ja9a3OUa2Fo+EaN0cbLu0eKpBwPFzc8= @@ -261,6 +279,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jawher/mow.cli v1.2.0 h1:e6ViPPy+82A/NFF/cfbq3Lr6q4JHKT9tyHwTCcUQgQw= +github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -268,6 +288,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -350,6 +372,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -366,6 +390,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -373,6 +398,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -395,6 +422,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= @@ -419,6 +447,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -457,6 +486,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -499,8 +529,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= @@ -532,6 +564,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -598,12 +631,15 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -613,6 +649,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -671,6 +709,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -713,6 +752,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -818,6 +858,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pages/repo.go b/pages/repo.go index 8e7792c..987b6f3 100644 --- a/pages/repo.go +++ b/pages/repo.go @@ -69,7 +69,7 @@ func HandleRepo(c *fiber.Ctx) error { mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) // Trust Nobody - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) + readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) repoArray = append(repoArray, Repo{ Fullname: repo.Get("full_name").String(), diff --git a/pages/user.go b/pages/user.go index 4971fb8..7e3cd1b 100644 --- a/pages/user.go +++ b/pages/user.go @@ -1,317 +1,170 @@ package pages import ( - "fmt" - "log" - + "codeberg.org/gothub/gothub/utils" + "context" "github.com/carlmjohnson/requests" + "github.com/enescakir/emoji" + "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" - "github.com/microcosm-cc/bluemonday" - - "codeberg.org/gothub/gothub/utils" - - "context" - "regexp" + "log" + "net/http" + "os" + "strconv" "strings" ) type User struct { - Login string - Name string - Bio string - AvatarUrl string - Location string - Following int64 - Followers int64 - Link string - Company string - Type string - EwTwitter string - Readme string -} - -type Ratelimit struct { - Remaining int64 - Limit int64 + Login string + Name string + Bio string + Status string + StatusEmoji string + AvatarUrl string + Location string + Email string + Timezone string + Following int64 + Followers int64 + Link string + Social []string + Organizations []string + OrgMembers []string + Company string + Type string + Contributions string + Readme string } // HandleUser handles the user page. func HandleUser(c *fiber.Ctx) error { - user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) - if user.Get("message").String() == "Not Found" { + // Declare Array used for displaying data + var userArray []User + + resp, err := http.Get("https://github.com/" + c.Params("user")) + if err != nil { + log.Println(err) + } + if resp.StatusCode == 404 { + // I need a better way to do this return c.Status(404).Render("error", fiber.Map{ - "error": "User " + c.Params("user") + " not found", + "error": "User " + c.Params("user") + " not found", }) } - if strings.Contains(user.Get("message").String(), "rate limit") { // dont wanna get the status code so i'll just do this instead 👍 - ratelimitJSON := utils.GetRequest("https://api.github.com/rate_limit") - fmt.Println(ratelimitJSON) - var ratelimitArray []Ratelimit - ratelimitArray = append(ratelimitArray, Ratelimit{ - Remaining: ratelimitJSON.Get("resources.core.remaining").Int(), - Limit: ratelimitJSON.Get("resources.core.limit").Int(), - }) + // User README + var readmee string - fmt.Println(ratelimitArray) - - return c.Render("ratelimit", fiber.Map{ - "Title": "GitHub API /users endpoint rate limit exceeded", - "ratelimit": ratelimitArray, - }) - } else { - var userArray []User - - var readmee string - - err := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). + err0 := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err0 != nil { + err2 := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). ToString(&readmee). Fetch(context.Background()) - if err != nil { - err2 := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). - ToString(&readmee). - Fetch(context.Background()) - if err2 != nil { - readmee = "" - log.Println(err) - } + if err2 != nil { + readmee = "" + log.Println(err0) } + } + mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) + // Sanitize the user README incase there is any weird shit in it + readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) - mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) + // Scraping + Scrape := User{ + Readme: string(readmeOutput), + } - // Trust Nobody - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + sc1 := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) - var link string + sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { + Scrape.Type = e.Attr("itemtype") + }) - if user.Get("blog").String() == "" { - link = "" - } else { - link = user.Get("blog").String() - if strings.HasPrefix(link, "https://") { - link = strings.TrimPrefix(link, "https://") - } else if strings.HasPrefix(link, "http://") { - link = strings.TrimPrefix(link, "http://") - } else { - log.Println("Has no prefix") - } - } - - userArray = append(userArray, User{ - Login: user.Get("login").String(), - Name: user.Get("name").String(), - Bio: user.Get("bio").String(), - AvatarUrl: "/avatar/" + user.Get("id").String(), - Location: user.Get("location").String(), - Following: user.Get("following").Int(), - Followers: user.Get("followers").Int(), - Link: link, - Company: user.Get("company").String(), - Type: user.Get("type").String(), - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), + sc1.Visit("https://github.com/" + c.Params("user") + "/") + sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) + if Scrape.Type == "http://schema.org/Person" { + sc.OnHTML("div.js-profile-editable-replace", func(e *colly.HTMLElement) { + // Main info + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") + Scrape.Bio = e.ChildText("div[data-bio-text] div") + Scrape.AvatarUrl = e.ChildAttr("img[alt*='Avatar']", "src") + // Metadata + Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") + Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.Link = e.ChildText("a[itemprop*='url']") + e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { + Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary")) + }) + // Followers/Following + Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i] span"), 10, 64) + Scrape.Following, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i] span"), 10, 64) + // Organizations + e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { + Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) + }) + // User Status + Scrape.Status = e.ChildText("div.user-status-circle-badge div.user-status-message-wrapper div") + Scrape.StatusEmoji = e.ChildAttr("div.user-status-circle-badge div.user-status-emoji-container g-emoji" ,"alias") }) - - fmt.Println(userArray) - - return c.Render("user", fiber.Map{ - "Title": "User " + c.Params("user"), - "user": userArray, + // Contributions + sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { + Scrape.Contributions = e.ChildText("h2") + }) + } else { + sc.OnHTML("div.container-xl div.flex-md-items-center", func(e *colly.HTMLElement) { + // Main info + Scrape.Login = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "alt") // Hacky + Scrape.Name = e.ChildText("h1.h2") + Scrape.Bio = e.ChildText("div.color-fg-muted div") + Scrape.AvatarUrl = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "src") + // Metadata + Scrape.Location = e.ChildText("span[itemprop*='location']") + Scrape.Email = e.ChildText("a[itemprop*='email']") + Scrape.Link = e.ChildText("a[itemprop*='url']") + e.ForEach("a.Link--primary", func(i int, el *colly.HTMLElement) { + Scrape.Social = append(Scrape.Social, el.Attr("href")) + }) + // Followers + Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64) + }) + // Org Members + sc.OnHTML("div.clearfix", func(e *colly.HTMLElement) { + e.ForEach("a[data-hovercard-type*='user'] img", func(i int, el *colly.HTMLElement) { + Scrape.OrgMembers = append(Scrape.OrgMembers, strings.TrimPrefix(el.Attr("alt"), "@")) + }) }) } -} - -// copied from bluemonday's GitHub repostiory, with some adaptations -func UGCPolicy() *bluemonday.Policy { - - p := bluemonday.NewPolicy() - - /////////////////////// - // Global attributes // - /////////////////////// - - // "class" is not permitted as we are not allowing users to style their own - // content - - p.AllowStandardAttributes() - - ////////////////////////////// - // Global URL format policy // - ////////////////////////////// - - p.AllowStandardURLs() - - //////////////////////////////// - // Declarations and structure // - //////////////////////////////// - - // "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are - // expecting user generated content to be a fragment of HTML and not a full - // document. - - ////////////////////////// - // Sectioning root tags // - ////////////////////////// - - // "article" and "aside" are permitted and takes no attributes - p.AllowElements("article", "aside") - - // "body" is not permitted as we are expecting user generated content to be a fragment - // of HTML and not a full document. - - // "details" is permitted, including the "open" attribute which can either - // be blank or the value "open". - p.AllowAttrs( - "open", - ).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details") - - // "fieldset" is not permitted as we are not allowing forms to be created. - - // "figure" is permitted and takes no attributes - p.AllowElements("figure") - - // "nav" is not permitted as it is assumed that the site (and not the user) - // has defined navigation elements - - // "section" is permitted and takes no attributes - p.AllowElements("section") - - // "summary" is permitted and takes no attributes - p.AllowElements("summary") - - ////////////////////////// - // Headings and footers // - ////////////////////////// - - // "footer" is not permitted as we expect user content to be a fragment and - // not structural to this extent - - // "h1" through "h6" are permitted and take no attributes - p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6") - - // "header" is not permitted as we expect user content to be a fragment and - // not structural to this extent - - // "hgroup" is permitted and takes no attributes - p.AllowElements("hgroup") - - ///////////////////////////////////// - // Content grouping and separating // - ///////////////////////////////////// - - // "blockquote" is permitted, including the "cite" attribute which must be - // a standard URL. - p.AllowAttrs("cite").OnElements("blockquote") - - // "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes - p.AllowElements("br", "div", "hr", "p", "span", "wbr") - - /////////// - // Links // - /////////// - - // "a" is permitted - p.AllowAttrs("href").OnElements("a") - - // "area" is permitted along with the attributes that map image maps work - p.AllowAttrs("name").Matching( - regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`), - ).OnElements("map") - p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area") - p.AllowAttrs("coords").Matching( - regexp.MustCompile(`^([0-9]+,)+[0-9]+$`), - ).OnElements("area") - p.AllowAttrs("href").OnElements("area") - p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area") - p.AllowAttrs("shape").Matching( - regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`), - ).OnElements("area") - p.AllowAttrs("usemap").Matching( - regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`), - ).OnElements("img") - - // "link" is not permitted - - ///////////////////// - // Phrase elements // - ///////////////////// - - // The following are all inline phrasing elements - p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em", - "figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var") - - // "q" is permitted and "cite" is a URL and handled by URL policies - p.AllowAttrs("cite").OnElements("q") - - // "time" is permitted - p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time") - - //////////////////// - // Style elements // - //////////////////// - - // block and inline elements that impart no semantic meaning but style the - // document - p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u") - - // "style" is not permitted as we are not yet sanitising CSS and it is an - // XSS attack vector - - ////////////////////// - // HTML5 Formatting // - ////////////////////// - - // "bdi" "bdo" are permitted - p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo") - - // "rp" "rt" "ruby" are permitted - p.AllowElements("rp", "rt", "ruby") - - /////////////////////////// - // HTML5 Change tracking // - /////////////////////////// - - // "del" "ins" are permitted - p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins") - p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins") - - /////////// - // Lists // - /////////// - - p.AllowLists() - - //////////// - // Tables // - //////////// - - p.AllowTables() - - /////////// - // Forms // - /////////// - - // By and large, forms are not permitted. However there are some form - // elements that can be used to present data, and we do permit those - // - // "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist" - // "textarea" "optgroup" "option" are all not permitted - - // "meter" is permitted - p.AllowAttrs( - "value", - "min", - "max", - "low", - "high", - "optimum", - ).Matching(bluemonday.Number).OnElements("meter") - - // "progress" is permitted - p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress") - - return p + sc.Visit("https://github.com/" + c.Params("user") + "/") + // Fixing the output a bit + Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/") + Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl // Avatar needs to be in /avatar so its proxied + if Scrape.StatusEmoji != "" { + Scrape.StatusEmoji = emoji.Parse(":" + Scrape.StatusEmoji + ":") // Convert the emoji code to an actual emoji + } + Scrape.Login = strings.TrimPrefix(Scrape.Login, "@") // Only for orgs + // Remove HTTP(s) from user website url if it exists + if strings.HasPrefix(Scrape.Link, "https://") { + Scrape.Link = strings.TrimPrefix(Scrape.Link, "https://") + } else if strings.HasPrefix(Scrape.Link, "http://") { + Scrape.Link = strings.TrimPrefix(Scrape.Link, "http://") + } + + // Add scrape-based info to userArray + userArray = append(userArray, Scrape) + + return c.Render("user", fiber.Map{ + "Title": "User " + c.Params("user"), + "user": userArray, + }) } diff --git a/utils/getrequest.go b/utils/getrequest.go index 6edf2c0..1c486fa 100644 --- a/utils/getrequest.go +++ b/utils/getrequest.go @@ -3,7 +3,7 @@ package utils import ( "io" "net/http" - + "os" "github.com/tidwall/gjson" ) @@ -13,8 +13,12 @@ func GetRequest(url string) gjson.Result { panic(err) } + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } r.Header.Set("Accept", "application/vnd.github.v3+json") - r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36") + r.Header.Set("User-Agent", UserAgent) client := &http.Client{} res, err := client.Do(r) diff --git a/utils/ugcpolicy.go b/utils/ugcpolicy.go new file mode 100644 index 0000000..ffcb4a8 --- /dev/null +++ b/utils/ugcpolicy.go @@ -0,0 +1,198 @@ +package utils + +import ( + "github.com/microcosm-cc/bluemonday" + "regexp" +) + +// copied from bluemonday's GitHub repostiory, with some adaptations +func UGCPolicy() *bluemonday.Policy { + + p := bluemonday.NewPolicy() + + /////////////////////// + // Global attributes // + /////////////////////// + + // "class" is not permitted as we are not allowing users to style their own + // content + + p.AllowStandardAttributes() + + ////////////////////////////// + // Global URL format policy // + ////////////////////////////// + + p.AllowStandardURLs() + + //////////////////////////////// + // Declarations and structure // + //////////////////////////////// + + // "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are + // expecting user generated content to be a fragment of HTML and not a full + // document. + + ////////////////////////// + // Sectioning root tags // + ////////////////////////// + + // "article" and "aside" are permitted and takes no attributes + p.AllowElements("article", "aside") + + // "body" is not permitted as we are expecting user generated content to be a fragment + // of HTML and not a full document. + + // "details" is permitted, including the "open" attribute which can either + // be blank or the value "open". + p.AllowAttrs( + "open", + ).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details") + + // "fieldset" is not permitted as we are not allowing forms to be created. + + // "figure" is permitted and takes no attributes + p.AllowElements("figure") + + // "nav" is not permitted as it is assumed that the site (and not the user) + // has defined navigation elements + + // "section" is permitted and takes no attributes + p.AllowElements("section") + + // "summary" is permitted and takes no attributes + p.AllowElements("summary") + + ////////////////////////// + // Headings and footers // + ////////////////////////// + + // "footer" is not permitted as we expect user content to be a fragment and + // not structural to this extent + + // "h1" through "h6" are permitted and take no attributes + p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6") + + // "header" is not permitted as we expect user content to be a fragment and + // not structural to this extent + + // "hgroup" is permitted and takes no attributes + p.AllowElements("hgroup") + + ///////////////////////////////////// + // Content grouping and separating // + ///////////////////////////////////// + + // "blockquote" is permitted, including the "cite" attribute which must be + // a standard URL. + p.AllowAttrs("cite").OnElements("blockquote") + + // "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes + p.AllowElements("br", "div", "hr", "p", "span", "wbr") + + /////////// + // Links // + /////////// + + // "a" is permitted + p.AllowAttrs("href").OnElements("a") + + // "area" is permitted along with the attributes that map image maps work + p.AllowAttrs("name").Matching( + regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`), + ).OnElements("map") + p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area") + p.AllowAttrs("coords").Matching( + regexp.MustCompile(`^([0-9]+,)+[0-9]+$`), + ).OnElements("area") + p.AllowAttrs("href").OnElements("area") + p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area") + p.AllowAttrs("shape").Matching( + regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`), + ).OnElements("area") + p.AllowAttrs("usemap").Matching( + regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`), + ).OnElements("img") + + // "link" is not permitted + + ///////////////////// + // Phrase elements // + ///////////////////// + + // The following are all inline phrasing elements + p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em", + "figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var") + + // "q" is permitted and "cite" is a URL and handled by URL policies + p.AllowAttrs("cite").OnElements("q") + + // "time" is permitted + p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time") + + //////////////////// + // Style elements // + //////////////////// + + // block and inline elements that impart no semantic meaning but style the + // document + p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u") + + // "style" is not permitted as we are not yet sanitising CSS and it is an + // XSS attack vector + + ////////////////////// + // HTML5 Formatting // + ////////////////////// + + // "bdi" "bdo" are permitted + p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo") + + // "rp" "rt" "ruby" are permitted + p.AllowElements("rp", "rt", "ruby") + + /////////////////////////// + // HTML5 Change tracking // + /////////////////////////// + + // "del" "ins" are permitted + p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins") + p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins") + + /////////// + // Lists // + /////////// + + p.AllowLists() + + //////////// + // Tables // + //////////// + + p.AllowTables() + + /////////// + // Forms // + /////////// + + // By and large, forms are not permitted. However there are some form + // elements that can be used to present data, and we do permit those + // + // "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist" + // "textarea" "optgroup" "option" are all not permitted + + // "meter" is permitted + p.AllowAttrs( + "value", + "min", + "max", + "low", + "high", + "optimum", + ).Matching(bluemonday.Number).OnElements("meter") + + // "progress" is permitted + p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress") + + return p +} diff --git a/views/user.html b/views/user.html index 127751f..dd47b53 100644 --- a/views/user.html +++ b/views/user.html @@ -12,23 +12,51 @@

{{.Name}}

{{ end }}

{{.Login}}

- {{ if eq .Type "User" }} + {{ if eq .Type "http://schema.org/Person" }}

{{.Followers}} followers - {{.Following}} following

{{ else }}

{{.Followers}} followers

{{ end }} + {{ if .StatusEmoji }} +

{{.StatusEmoji}} {{.Status}}

+ {{ end }} {{ if .Location }}

🌍 {{.Location}}

{{ end }} + {{ if .Timezone }} +

🕑️ {{.Timezone}}

+ {{ end }} {{ if .Link }} -

🔗 {{.Link}}

+

🌐 {{.Link}}

{{ end }} + {{ if .Email }} +

✉️ {{.Email}}

+ {{ end }} + {{ if .Social }} + {{range .Social}} +

🔗 {{.}}

+ {{ end }} + {{ end }} + {{ if .Organizations }} +

Organizations: + {{range .Organizations}} + {{.}} + {{ end }} +

+ {{ end }} + {{ if .OrgMembers }} +

Members: + {{range .OrgMembers}} + @{{.}} + {{ end }} +

+ {{ end }} {{ if .Company }}

🏢 {{.Company}}

{{ end }} - {{ if .EwTwitter }} -

🐦 {{.EwTwitter}}

- {{ end }} + {{ if eq .Type "http://schema.org/Person" }} +

✏️ {{ if .Name }} {{.Name}} {{ else }} {{.Login}} {{ end }} has made {{.Contributions}}

+ {{ end }} {{ if .Bio }} @@ -48,4 +76,4 @@

User not found

That user doesn't exist.

{{ end }} - \ No newline at end of file +