GotHub/gothub
1
0
Fork 0
mirror of https://codeberg.org/gothub/gothub synced 2024-12-06 19:16:24 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request 'add basic scraping for users' (#48) from aryak/gothub:scraping into dev

Browse Source 
Reviewed-on: https://codeberg.org/gothub/gothub/pulls/48
This commit is contained in:
Odyssium
2023-02-13 15:13:24 +00:00
parent 059ca40e32 7b15df28d9
commit 412b0b0979
8 changed files with 441 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -1
View File
@@ -63,9 +63,11 @@ gothub.example.com {
reverse_proxy localhost:(the port for GotHub)
}
```
## Scraping
Variables: 1) GOTHUB_USER_AGENT - Sets the User Agent to use for making the requests. Defaults to Chrome 110 on Windows 10.
## DMCA/Legal notice
All content shown on a GotHub instance is from GitHub. Any issues with content shown on a GotHub instance needs to be reported to GitHub, not the instance host's internet or domain provider.
GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc.
GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc.
go.mod
+17
View File
@@ -14,18 +14,32 @@ require (
)
require (
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/antchfx/htmlquery v1.3.0 // indirect
github.com/antchfx/xmlquery v1.3.15 // indirect
github.com/antchfx/xpath v1.2.3 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
github.com/enescakir/emoji v1.0.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gocolly/colly v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jawher/mow.cli v1.2.0 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/klauspost/compress v1.15.14 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
@@ -33,4 +47,7 @@ require (
github.com/valyala/tcplisten v1.0.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.1 // indirect
)
go.sum
+42
View File
@@ -54,6 +54,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw=
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
@@ -66,6 +68,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=
github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=
github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw=
github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA=
github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes=
github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -114,6 +124,8 @@ github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55k
github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog=
github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -139,6 +151,10 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofiber/fiber/v2 v2.40.1/go.mod h1:Gko04sLksnHbzLSRBFWPFdzM9Ws9pRxvvIaohJK1dsk=
github.com/gofiber/fiber/v2 v2.41.0 h1:YhNoUS/OTjEz+/WLYuQ01xI7RXgKEFnGBKMagAu5f0M=
@@ -151,6 +167,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -177,6 +194,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK0Ja9a3OUa2Fo+EaN0cbLu0eKpBwPFzc8=
@@ -261,6 +279,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jawher/mow.cli v1.2.0 h1:e6ViPPy+82A/NFF/cfbq3Lr6q4JHKT9tyHwTCcUQgQw=
github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -268,6 +288,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
@@ -350,6 +372,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -366,6 +390,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -373,6 +398,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -395,6 +422,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs=
@@ -419,6 +447,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -457,6 +486,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -499,8 +529,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
@@ -532,6 +564,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -598,12 +631,15 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -613,6 +649,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -671,6 +709,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -713,6 +752,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -818,6 +858,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
pages/repo.go
+1 -1
View File
@@ -69,7 +69,7 @@ func HandleRepo(c *fiber.Ctx) error {
mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil)
// Trust Nobody
readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe)
readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe)
repoArray = append(repoArray, Repo{
Fullname: repo.Get("full_name").String(),
pages/user.go
+140 -287
View File
@@ -1,317 +1,170 @@
package pages
import (
"fmt"
"log"
"codeberg.org/gothub/gothub/utils"
"context"
"github.com/carlmjohnson/requests"
"github.com/enescakir/emoji"
"github.com/gocolly/colly"
"github.com/gofiber/fiber/v2"
"github.com/gomarkdown/markdown"
"github.com/microcosm-cc/bluemonday"
"codeberg.org/gothub/gothub/utils"
"context"
"regexp"
"log"
"net/http"
"os"
"strconv"
"strings"
)
type User struct {
Login string
Name string
Bio string
AvatarUrl string
Location string
Following int64
Followers int64
Link string
Company string
Type string
EwTwitter string
Readme string
}
type Ratelimit struct {
Remaining int64
Limit int64
Login string
Name string
Bio string
Status string
StatusEmoji string
AvatarUrl string
Location string
Email string
Timezone string
Following int64
Followers int64
Link string
Social []string
Organizations []string
OrgMembers []string
Company string
Type string
Contributions string
Readme string
}
// HandleUser handles the user page.
func HandleUser(c *fiber.Ctx) error {
user := utils.GetRequest("https://api.github.com/users/" + c.Params("user"))
if user.Get("message").String() == "Not Found" {
// Declare Array used for displaying data
var userArray []User
resp, err := http.Get("https://github.com/" + c.Params("user"))
if err != nil {
log.Println(err)
}
if resp.StatusCode == 404 {
// I need a better way to do this
return c.Status(404).Render("error", fiber.Map{
"error": "User " + c.Params("user") + " not found",
"error": "User " + c.Params("user") + " not found",
})
}
if strings.Contains(user.Get("message").String(), "rate limit") { // dont wanna get the status code so i'll just do this instead 👍
ratelimitJSON := utils.GetRequest("https://api.github.com/rate_limit")
fmt.Println(ratelimitJSON)
var ratelimitArray []Ratelimit
ratelimitArray = append(ratelimitArray, Ratelimit{
Remaining: ratelimitJSON.Get("resources.core.remaining").Int(),
Limit: ratelimitJSON.Get("resources.core.limit").Int(),
})
// User README
var readmee string
fmt.Println(ratelimitArray)
return c.Render("ratelimit", fiber.Map{
"Title": "GitHub API /users endpoint rate limit exceeded",
"ratelimit": ratelimitArray,
})
} else {
var userArray []User
var readmee string
err := requests.
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md").
err0 := requests.
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md").
ToString(&readmee).
Fetch(context.Background())
if err0 != nil {
err2 := requests.
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md").
ToString(&readmee).
Fetch(context.Background())
if err != nil {
err2 := requests.
URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md").
ToString(&readmee).
Fetch(context.Background())
if err2 != nil {
readmee = ""
log.Println(err)
}
if err2 != nil {
readmee = ""
log.Println(err0)
}
}
mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil)
// Sanitize the user README incase there is any weird shit in it
readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe)
mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil)
// Scraping
Scrape := User{
Readme: string(readmeOutput),
}
// Trust Nobody
readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe)
UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT")
if !ok {
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
sc1 := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent))
var link string
sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) {
Scrape.Type = e.Attr("itemtype")
})
if user.Get("blog").String() == "" {
link = ""
} else {
link = user.Get("blog").String()
if strings.HasPrefix(link, "https://") {
link = strings.TrimPrefix(link, "https://")
} else if strings.HasPrefix(link, "http://") {
link = strings.TrimPrefix(link, "http://")
} else {
log.Println("Has no prefix")
}
}
userArray = append(userArray, User{
Login: user.Get("login").String(),
Name: user.Get("name").String(),
Bio: user.Get("bio").String(),
AvatarUrl: "/avatar/" + user.Get("id").String(),
Location: user.Get("location").String(),
Following: user.Get("following").Int(),
Followers: user.Get("followers").Int(),
Link: link,
Company: user.Get("company").String(),
Type: user.Get("type").String(),
EwTwitter: user.Get("twitter_username").String(),
Readme: string(readmeOutput),
sc1.Visit("https://github.com/" + c.Params("user") + "/")
sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent))
if Scrape.Type == "http://schema.org/Person" {
sc.OnHTML("div.js-profile-editable-replace", func(e *colly.HTMLElement) {
// Main info
Scrape.Login = e.ChildText("span[itemprop*='additionalName']")
Scrape.Name = e.ChildText("span[itemprop*='name']")
Scrape.Bio = e.ChildText("div[data-bio-text] div")
Scrape.AvatarUrl = e.ChildAttr("img[alt*='Avatar']", "src")
// Metadata
Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span")
Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span")
Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span")
Scrape.Link = e.ChildText("a[itemprop*='url']")
e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) {
Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary"))
})
// Followers/Following
Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i] span"), 10, 64)
Scrape.Following, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i] span"), 10, 64)
// Organizations
e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) {
Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label"))
})
// User Status
Scrape.Status = e.ChildText("div.user-status-circle-badge div.user-status-message-wrapper div")
Scrape.StatusEmoji = e.ChildAttr("div.user-status-circle-badge div.user-status-emoji-container g-emoji" ,"alias")
})
fmt.Println(userArray)
return c.Render("user", fiber.Map{
"Title": "User " + c.Params("user"),
"user": userArray,
// Contributions
sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) {
Scrape.Contributions = e.ChildText("h2")
})
} else {
sc.OnHTML("div.container-xl div.flex-md-items-center", func(e *colly.HTMLElement) {
// Main info
Scrape.Login = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "alt") // Hacky
Scrape.Name = e.ChildText("h1.h2")
Scrape.Bio = e.ChildText("div.color-fg-muted div")
Scrape.AvatarUrl = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "src")
// Metadata
Scrape.Location = e.ChildText("span[itemprop*='location']")
Scrape.Email = e.ChildText("a[itemprop*='email']")
Scrape.Link = e.ChildText("a[itemprop*='url']")
e.ForEach("a.Link--primary", func(i int, el *colly.HTMLElement) {
Scrape.Social = append(Scrape.Social, el.Attr("href"))
})
// Followers
Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64)
})
// Org Members
sc.OnHTML("div.clearfix", func(e *colly.HTMLElement) {
e.ForEach("a[data-hovercard-type*='user'] img", func(i int, el *colly.HTMLElement) {
Scrape.OrgMembers = append(Scrape.OrgMembers, strings.TrimPrefix(el.Attr("alt"), "@"))
})
})
}
}
// copied from bluemonday's GitHub repostiory, with some adaptations
func UGCPolicy() *bluemonday.Policy {
p := bluemonday.NewPolicy()
///////////////////////
// Global attributes //
///////////////////////
// "class" is not permitted as we are not allowing users to style their own
// content
p.AllowStandardAttributes()
//////////////////////////////
// Global URL format policy //
//////////////////////////////
p.AllowStandardURLs()
////////////////////////////////
// Declarations and structure //
////////////////////////////////
// "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are
// expecting user generated content to be a fragment of HTML and not a full
// document.
//////////////////////////
// Sectioning root tags //
//////////////////////////
// "article" and "aside" are permitted and takes no attributes
p.AllowElements("article", "aside")
// "body" is not permitted as we are expecting user generated content to be a fragment
// of HTML and not a full document.
// "details" is permitted, including the "open" attribute which can either
// be blank or the value "open".
p.AllowAttrs(
"open",
).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details")
// "fieldset" is not permitted as we are not allowing forms to be created.
// "figure" is permitted and takes no attributes
p.AllowElements("figure")
// "nav" is not permitted as it is assumed that the site (and not the user)
// has defined navigation elements
// "section" is permitted and takes no attributes
p.AllowElements("section")
// "summary" is permitted and takes no attributes
p.AllowElements("summary")
//////////////////////////
// Headings and footers //
//////////////////////////
// "footer" is not permitted as we expect user content to be a fragment and
// not structural to this extent
// "h1" through "h6" are permitted and take no attributes
p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6")
// "header" is not permitted as we expect user content to be a fragment and
// not structural to this extent
// "hgroup" is permitted and takes no attributes
p.AllowElements("hgroup")
/////////////////////////////////////
// Content grouping and separating //
/////////////////////////////////////
// "blockquote" is permitted, including the "cite" attribute which must be
// a standard URL.
p.AllowAttrs("cite").OnElements("blockquote")
// "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes
p.AllowElements("br", "div", "hr", "p", "span", "wbr")
///////////
// Links //
///////////
// "a" is permitted
p.AllowAttrs("href").OnElements("a")
// "area" is permitted along with the attributes that map image maps work
p.AllowAttrs("name").Matching(
regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`),
).OnElements("map")
p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area")
p.AllowAttrs("coords").Matching(
regexp.MustCompile(`^([0-9]+,)+[0-9]+$`),
).OnElements("area")
p.AllowAttrs("href").OnElements("area")
p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area")
p.AllowAttrs("shape").Matching(
regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`),
).OnElements("area")
p.AllowAttrs("usemap").Matching(
regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`),
).OnElements("img")
// "link" is not permitted
/////////////////////
// Phrase elements //
/////////////////////
// The following are all inline phrasing elements
p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em",
"figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var")
// "q" is permitted and "cite" is a URL and handled by URL policies
p.AllowAttrs("cite").OnElements("q")
// "time" is permitted
p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time")
////////////////////
// Style elements //
////////////////////
// block and inline elements that impart no semantic meaning but style the
// document
p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u")
// "style" is not permitted as we are not yet sanitising CSS and it is an
// XSS attack vector
//////////////////////
// HTML5 Formatting //
//////////////////////
// "bdi" "bdo" are permitted
p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo")
// "rp" "rt" "ruby" are permitted
p.AllowElements("rp", "rt", "ruby")
///////////////////////////
// HTML5 Change tracking //
///////////////////////////
// "del" "ins" are permitted
p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins")
p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins")
///////////
// Lists //
///////////
p.AllowLists()
////////////
// Tables //
////////////
p.AllowTables()
///////////
// Forms //
///////////
// By and large, forms are not permitted. However there are some form
// elements that can be used to present data, and we do permit those
//
// "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist"
// "textarea" "optgroup" "option" are all not permitted
// "meter" is permitted
p.AllowAttrs(
"value",
"min",
"max",
"low",
"high",
"optimum",
).Matching(bluemonday.Number).OnElements("meter")
// "progress" is permitted
p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress")
return p
sc.Visit("https://github.com/" + c.Params("user") + "/")
// Fixing the output a bit
Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/")
Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl // Avatar needs to be in /avatar so its proxied
if Scrape.StatusEmoji != "" {
Scrape.StatusEmoji = emoji.Parse(":" + Scrape.StatusEmoji + ":") // Convert the emoji code to an actual emoji
}
Scrape.Login = strings.TrimPrefix(Scrape.Login, "@") // Only for orgs
// Remove HTTP(s) from user website url if it exists
if strings.HasPrefix(Scrape.Link, "https://") {
Scrape.Link = strings.TrimPrefix(Scrape.Link, "https://")
} else if strings.HasPrefix(Scrape.Link, "http://") {
Scrape.Link = strings.TrimPrefix(Scrape.Link, "http://")
}
// Add scrape-based info to userArray
userArray = append(userArray, Scrape)
return c.Render("user", fiber.Map{
"Title": "User " + c.Params("user"),
"user": userArray,
})
}
utils/getrequest.go
+6 -2
View File
@@ -3,7 +3,7 @@ package utils
import (
"io"
"net/http"
"os"
"github.com/tidwall/gjson"
)
@@ -13,8 +13,12 @@ func GetRequest(url string) gjson.Result {
panic(err)
}
UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT")
if !ok {
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
r.Header.Set("Accept", "application/vnd.github.v3+json")
r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
r.Header.Set("User-Agent", UserAgent)
client := &http.Client{}
res, err := client.Do(r)
utils/ugcpolicy.go
+198
View File
@@ -0,0 +1,198 @@
package utils
import (
"github.com/microcosm-cc/bluemonday"
"regexp"
)
// copied from bluemonday's GitHub repostiory, with some adaptations
func UGCPolicy() *bluemonday.Policy {
p := bluemonday.NewPolicy()
///////////////////////
// Global attributes //
///////////////////////
// "class" is not permitted as we are not allowing users to style their own
// content
p.AllowStandardAttributes()
//////////////////////////////
// Global URL format policy //
//////////////////////////////
p.AllowStandardURLs()
////////////////////////////////
// Declarations and structure //
////////////////////////////////
// "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are
// expecting user generated content to be a fragment of HTML and not a full
// document.
//////////////////////////
// Sectioning root tags //
//////////////////////////
// "article" and "aside" are permitted and takes no attributes
p.AllowElements("article", "aside")
// "body" is not permitted as we are expecting user generated content to be a fragment
// of HTML and not a full document.
// "details" is permitted, including the "open" attribute which can either
// be blank or the value "open".
p.AllowAttrs(
"open",
).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details")
// "fieldset" is not permitted as we are not allowing forms to be created.
// "figure" is permitted and takes no attributes
p.AllowElements("figure")
// "nav" is not permitted as it is assumed that the site (and not the user)
// has defined navigation elements
// "section" is permitted and takes no attributes
p.AllowElements("section")
// "summary" is permitted and takes no attributes
p.AllowElements("summary")
//////////////////////////
// Headings and footers //
//////////////////////////
// "footer" is not permitted as we expect user content to be a fragment and
// not structural to this extent
// "h1" through "h6" are permitted and take no attributes
p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6")
// "header" is not permitted as we expect user content to be a fragment and
// not structural to this extent
// "hgroup" is permitted and takes no attributes
p.AllowElements("hgroup")
/////////////////////////////////////
// Content grouping and separating //
/////////////////////////////////////
// "blockquote" is permitted, including the "cite" attribute which must be
// a standard URL.
p.AllowAttrs("cite").OnElements("blockquote")
// "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes
p.AllowElements("br", "div", "hr", "p", "span", "wbr")
///////////
// Links //
///////////
// "a" is permitted
p.AllowAttrs("href").OnElements("a")
// "area" is permitted along with the attributes that map image maps work
p.AllowAttrs("name").Matching(
regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`),
).OnElements("map")
p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area")
p.AllowAttrs("coords").Matching(
regexp.MustCompile(`^([0-9]+,)+[0-9]+$`),
).OnElements("area")
p.AllowAttrs("href").OnElements("area")
p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area")
p.AllowAttrs("shape").Matching(
regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`),
).OnElements("area")
p.AllowAttrs("usemap").Matching(
regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`),
).OnElements("img")
// "link" is not permitted
/////////////////////
// Phrase elements //
/////////////////////
// The following are all inline phrasing elements
p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em",
"figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var")
// "q" is permitted and "cite" is a URL and handled by URL policies
p.AllowAttrs("cite").OnElements("q")
// "time" is permitted
p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time")
////////////////////
// Style elements //
////////////////////
// block and inline elements that impart no semantic meaning but style the
// document
p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u")
// "style" is not permitted as we are not yet sanitising CSS and it is an
// XSS attack vector
//////////////////////
// HTML5 Formatting //
//////////////////////
// "bdi" "bdo" are permitted
p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo")
// "rp" "rt" "ruby" are permitted
p.AllowElements("rp", "rt", "ruby")
///////////////////////////
// HTML5 Change tracking //
///////////////////////////
// "del" "ins" are permitted
p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins")
p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins")
///////////
// Lists //
///////////
p.AllowLists()
////////////
// Tables //
////////////
p.AllowTables()
///////////
// Forms //
///////////
// By and large, forms are not permitted. However there are some form
// elements that can be used to present data, and we do permit those
//
// "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist"
// "textarea" "optgroup" "option" are all not permitted
// "meter" is permitted
p.AllowAttrs(
"value",
"min",
"max",
"low",
"high",
"optimum",
).Matching(bluemonday.Number).OnElements("meter")
// "progress" is permitted
p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress")
return p
}
views/user.html
+34 -6
View File
@@ -12,23 +12,51 @@
<h1 class="user-profile-h1">{{.Name}}</h1>
{{ end }}
<h2>{{.Login}}</h2>
{{ if eq .Type "User" }}
{{ if eq .Type "http://schema.org/Person" }}
<p><b>{{.Followers}}</b> followers - <b>{{.Following}}</b> following</p>
{{ else }}
<p><b>{{.Followers}}</b> followers</p>
{{ end }}
{{ if .StatusEmoji }}
<p>{{.StatusEmoji}} {{.Status}}</p>
{{ end }}
{{ if .Location }}
<p>🌍 {{.Location}}</p>
{{ end }}
{{ if .Timezone }}
<p>🕑️ {{.Timezone}}</p>
{{ end }}
{{ if .Link }}
<p>🔗 <a href="https://{{.Link}}" target="_blank">{{.Link}}</a></p>
<p>🌐 <a href="https://{{.Link}}" target="_blank">{{.Link}}</a></p>
{{ end }}
{{ if .Email }}
<p>✉️ {{.Email}}</p>
{{ end }}
{{ if .Social }}
{{range .Social}}
<p>🔗 <a href="{{.}}" target="_blank">{{.}}</a></p>
{{ end }}
{{ end }}
{{ if .Organizations }}
<p>Organizations:
{{range .Organizations}}
<a href="/{{.}}" target="_blank">{{.}}</a>
{{ end }}
</p>
{{ end }}
{{ if .OrgMembers }}
<p>Members:
{{range .OrgMembers}}
<a href="/{{.}}" target="_blank">@{{.}}</a>
{{ end }}
</p>
{{ end }}
{{ if .Company }}
<p>🏢 {{.Company}}</p>
{{ end }}
{{ if .EwTwitter }}
<p>🐦 <a href="https://twitter.com/{{.EwTwitter}}" target="_blank">{{.EwTwitter}}</a></p>
{{ end }}
{{ if eq .Type "http://schema.org/Person" }}
<p>✏️ {{ if .Name }} {{.Name}} {{ else }} {{.Login}} {{ end }} has made {{.Contributions}}</p>
{{ end }}
</div>
{{ if .Bio }}
@@ -48,4 +76,4 @@
<h2>User not found</h2>
<p>That user doesn't exist.</p>
{{ end }}
</main>
</main>