From 7d7865b86ae79b6eabb757317ac4f797227010d3 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 19:47:06 +0530 Subject: [PATCH 01/24] add basic scraping Signed-off-by: Arya Kiran --- go.mod | 16 +++++++++++ go.sum | 40 ++++++++++++++++++++++++++ pages/user.go | 78 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 122 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 9729f25..850c943 100644 --- a/go.mod +++ b/go.mod @@ -14,17 +14,30 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.3.0 // indirect + github.com/antchfx/xmlquery v1.3.15 // indirect + github.com/antchfx/xpath v1.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/gocolly/colly v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jawher/mow.cli v1.2.0 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.15.14 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/rivo/uniseg v0.4.3 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -32,4 +45,7 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 8ba98c8..ae4e832 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw= github.com/alecthomas/chroma/v2 v2.4.0 h1:Loe2ZjT5x3q1bcWwemqyqEi8p11/IV/ncFCeLYDpWC4= github.com/alecthomas/chroma/v2 v2.4.0/go.mod h1:6kHzqF5O6FUSJzBXW7fXELjb+e+7OXW4UpoPqMO7IBQ= @@ -64,6 +66,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= +github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= +github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= +github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= +github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -136,6 +146,10 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.40.1/go.mod h1:Gko04sLksnHbzLSRBFWPFdzM9Ws9pRxvvIaohJK1dsk= github.com/gofiber/fiber/v2 v2.41.0 h1:YhNoUS/OTjEz+/WLYuQ01xI7RXgKEFnGBKMagAu5f0M= @@ -148,6 +162,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -174,6 +189,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK0Ja9a3OUa2Fo+EaN0cbLu0eKpBwPFzc8= @@ -258,6 +274,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jawher/mow.cli v1.2.0 h1:e6ViPPy+82A/NFF/cfbq3Lr6q4JHKT9tyHwTCcUQgQw= +github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -265,6 +283,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -347,6 +367,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -363,6 +385,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -370,6 +393,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -392,6 +417,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= @@ -416,6 +442,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -454,6 +481,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -496,8 +524,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= @@ -529,6 +559,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -595,12 +626,15 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -610,6 +644,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -668,6 +704,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -710,6 +747,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -815,6 +853,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pages/user.go b/pages/user.go index 4971fb8..338f372 100644 --- a/pages/user.go +++ b/pages/user.go @@ -1,15 +1,15 @@ package pages import ( + "codeberg.org/gothub/gothub/utils" "fmt" - "log" - "github.com/carlmjohnson/requests" + "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" "github.com/microcosm-cc/bluemonday" - - "codeberg.org/gothub/gothub/utils" + "log" + "strconv" "context" "regexp" @@ -99,23 +99,77 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Has no prefix") } } + // scraping + var OrgOrUser string + var BioScrape string + var LocScrape string + var WorkplaceScrape string + var AvatarUrlScrape string + var FollowingScrape string + var FollowerScrape string + var UsernameScrape string + var FullNameScrape string + sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) + sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { + OrgOrUser = e.Attr("itemtype") + }) + sc1.Visit("https://github.com/" + c.Params("user") + "/") + sc := colly.NewCollector( + colly.AllowedDomains("github.com"), + ) + if OrgOrUser == "http://schema.org/Person" { + // Bio + sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { + BioScrape = e.Attr("data-bio-text") + }) + // Avatar + sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { + AvatarUrlScrape = e.Attr("src") + }) + // Metadata (Location/Workplace/Website/Twitter etc.) + sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { + LocScrape = e.ChildText("li[itemprop*='homeLocation'] span") + WorkplaceScrape = e.ChildText("li[itemprop*='worksFor'] span") + }) + // Followers/Following + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { + FollowerScrape = e.ChildText("span") + }) + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { + FollowingScrape = e.ChildText("span") + }) + // User/Full Name + sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { + UsernameScrape = e.ChildText("span[itemprop*='additionalName']") + FullNameScrape = e.ChildText("span[itemprop*='name']") + }) + } else { + sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { + AvatarUrlScrape = e.Attr("src") + }) + log.Println("Bio and Location routes cannot be scraped for organizations") + } + sc.Visit("https://github.com/" + c.Params("user") + "/") + FollowingScrapeInt, err := strconv.ParseInt(FollowingScrape, 10, 64) + FollowerScrapeInt, err := strconv.ParseInt(FollowerScrape, 10, 64) userArray = append(userArray, User{ - Login: user.Get("login").String(), - Name: user.Get("name").String(), - Bio: user.Get("bio").String(), - AvatarUrl: "/avatar/" + user.Get("id").String(), - Location: user.Get("location").String(), - Following: user.Get("following").Int(), - Followers: user.Get("followers").Int(), + Login: UsernameScrape, + Name: FullNameScrape, + Bio: BioScrape, + AvatarUrl: AvatarUrlScrape, + Location: LocScrape, + Following: FollowingScrapeInt, + Followers: FollowerScrapeInt, Link: link, - Company: user.Get("company").String(), + Company: WorkplaceScrape, Type: user.Get("type").String(), EwTwitter: user.Get("twitter_username").String(), Readme: string(readmeOutput), }) fmt.Println(userArray) + return c.Render("user", fiber.Map{ "Title": "User " + c.Params("user"), From 4db2d1188eecefa26b72b0ed85581d919660d13d Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 20:45:13 +0530 Subject: [PATCH 02/24] use user struct instead of declaring variables for scrape --- pages/user.go | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/pages/user.go b/pages/user.go index 338f372..9d73f8a 100644 --- a/pages/user.go +++ b/pages/user.go @@ -100,15 +100,8 @@ func HandleUser(c *fiber.Ctx) error { } } // scraping + var Scrape User var OrgOrUser string - var BioScrape string - var LocScrape string - var WorkplaceScrape string - var AvatarUrlScrape string - var FollowingScrape string - var FollowerScrape string - var UsernameScrape string - var FullNameScrape string sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { OrgOrUser = e.Attr("itemtype") @@ -120,49 +113,47 @@ func HandleUser(c *fiber.Ctx) error { if OrgOrUser == "http://schema.org/Person" { // Bio sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { - BioScrape = e.Attr("data-bio-text") + Scrape.Bio = e.Attr("data-bio-text") }) // Avatar sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { - AvatarUrlScrape = e.Attr("src") + Scrape.AvatarUrl = e.Attr("src") }) // Metadata (Location/Workplace/Website/Twitter etc.) sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { - LocScrape = e.ChildText("li[itemprop*='homeLocation'] span") - WorkplaceScrape = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") }) // Followers/Following sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { - FollowerScrape = e.ChildText("span") + Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) }) sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { - FollowingScrape = e.ChildText("span") + Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) }) // User/Full Name sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { - UsernameScrape = e.ChildText("span[itemprop*='additionalName']") - FullNameScrape = e.ChildText("span[itemprop*='name']") + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") }) } else { sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { - AvatarUrlScrape = e.Attr("src") + Scrape.AvatarUrl = e.Attr("src") }) log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - FollowingScrapeInt, err := strconv.ParseInt(FollowingScrape, 10, 64) - FollowerScrapeInt, err := strconv.ParseInt(FollowerScrape, 10, 64) userArray = append(userArray, User{ - Login: UsernameScrape, - Name: FullNameScrape, - Bio: BioScrape, - AvatarUrl: AvatarUrlScrape, - Location: LocScrape, - Following: FollowingScrapeInt, - Followers: FollowerScrapeInt, + Login: Scrape.Login, + Name: Scrape.Name, + Bio: Scrape.Bio, + AvatarUrl: Scrape.AvatarUrl, + Location: Scrape.Location, + Following: Scrape.Following, + Followers: Scrape.Followers, Link: link, - Company: WorkplaceScrape, + Company: Scrape.Company, Type: user.Get("type").String(), EwTwitter: user.Get("twitter_username").String(), Readme: string(readmeOutput), From 207fd7f52743f1e4aaa8d2fe677429595b559db0 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 21:12:18 +0530 Subject: [PATCH 03/24] cleanup the code a bit more --- pages/user.go | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/pages/user.go b/pages/user.go index 9d73f8a..d71efe9 100644 --- a/pages/user.go +++ b/pages/user.go @@ -100,16 +100,26 @@ func HandleUser(c *fiber.Ctx) error { } } // scraping - var Scrape User var OrgOrUser string + + Scrape := User{ + Link: link, + Type: user.Get("type").String(), + EwTwitter: user.Get("twitter_username").String(), + Readme: string(readmeOutput), + } + sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) + sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { OrgOrUser = e.Attr("itemtype") }) + sc1.Visit("https://github.com/" + c.Params("user") + "/") sc := colly.NewCollector( colly.AllowedDomains("github.com"), ) + if OrgOrUser == "http://schema.org/Person" { // Bio sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { @@ -144,20 +154,7 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - userArray = append(userArray, User{ - Login: Scrape.Login, - Name: Scrape.Name, - Bio: Scrape.Bio, - AvatarUrl: Scrape.AvatarUrl, - Location: Scrape.Location, - Following: Scrape.Following, - Followers: Scrape.Followers, - Link: link, - Company: Scrape.Company, - Type: user.Get("type").String(), - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), - }) + userArray = append(userArray, scrape) fmt.Println(userArray) From 9d966e1378e3db8b6fa6e6dc80a84bde35e1d839 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Tue, 7 Feb 2023 21:37:28 +0530 Subject: [PATCH 04/24] fix typo --- pages/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/user.go b/pages/user.go index d71efe9..00cf947 100644 --- a/pages/user.go +++ b/pages/user.go @@ -154,7 +154,7 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - userArray = append(userArray, scrape) + userArray = append(userArray, Scrape) fmt.Println(userArray) From 4e3bb59c41d47c0fb51a8bf1b004cf848814ab1d Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 15:56:40 +0530 Subject: [PATCH 05/24] make the user page mostly work when api is disabled --- pages/user.go | 175 +++++++++++++++++++++++------------------------- views/user.html | 4 +- 2 files changed, 87 insertions(+), 92 deletions(-) diff --git a/pages/user.go b/pages/user.go index 00cf947..e45754f 100644 --- a/pages/user.go +++ b/pages/user.go @@ -2,7 +2,6 @@ package pages import ( "codeberg.org/gothub/gothub/utils" - "fmt" "github.com/carlmjohnson/requests" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" @@ -38,6 +37,10 @@ type Ratelimit struct { // HandleUser handles the user page. func HandleUser(c *fiber.Ctx) error { + // Declare Array used for displaying data + var userArray []User + + // API user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) if user.Get("message").String() == "Not Found" { return c.Status(404).Render("error", fiber.Map{ @@ -46,7 +49,7 @@ func HandleUser(c *fiber.Ctx) error { } if strings.Contains(user.Get("message").String(), "rate limit") { // dont wanna get the status code so i'll just do this instead 👍 ratelimitJSON := utils.GetRequest("https://api.github.com/rate_limit") - fmt.Println(ratelimitJSON) + log.Println(ratelimitJSON) var ratelimitArray []Ratelimit ratelimitArray = append(ratelimitArray, Ratelimit{ @@ -54,36 +57,13 @@ func HandleUser(c *fiber.Ctx) error { Limit: ratelimitJSON.Get("resources.core.limit").Int(), }) - fmt.Println(ratelimitArray) + log.Println(ratelimitArray) return c.Render("ratelimit", fiber.Map{ "Title": "GitHub API /users endpoint rate limit exceeded", "ratelimit": ratelimitArray, }) } else { - var userArray []User - - var readmee string - - err := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). - ToString(&readmee). - Fetch(context.Background()) - if err != nil { - err2 := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). - ToString(&readmee). - Fetch(context.Background()) - if err2 != nil { - readmee = "" - log.Println(err) - } - } - - mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) - - // Trust Nobody - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) var link string @@ -99,71 +79,86 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Has no prefix") } } - // scraping - var OrgOrUser string - - Scrape := User{ - Link: link, - Type: user.Get("type").String(), - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), - } - - sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) - - sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { - OrgOrUser = e.Attr("itemtype") - }) - - sc1.Visit("https://github.com/" + c.Params("user") + "/") - sc := colly.NewCollector( - colly.AllowedDomains("github.com"), - ) - - if OrgOrUser == "http://schema.org/Person" { - // Bio - sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { - Scrape.Bio = e.Attr("data-bio-text") - }) - // Avatar - sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - }) - // Metadata (Location/Workplace/Website/Twitter etc.) - sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { - Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") - Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") - }) - // Followers/Following - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { - Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { - Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - // User/Full Name - sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { - Scrape.Login = e.ChildText("span[itemprop*='additionalName']") - Scrape.Name = e.ChildText("span[itemprop*='name']") - }) - - } else { - sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - }) - log.Println("Bio and Location routes cannot be scraped for organizations") - } - sc.Visit("https://github.com/" + c.Params("user") + "/") - userArray = append(userArray, Scrape) - - fmt.Println(userArray) - - - return c.Render("user", fiber.Map{ - "Title": "User " + c.Params("user"), - "user": userArray, - }) } + // User README + var readmee string + + err := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err != nil { + err2 := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err2 != nil { + readmee = "" + log.Println(err) + } + } + mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) + // Sanitize the user README incase there is any weird shit in it + readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) + + // scraping + Scrape := User{ + //Link: link, + EwTwitter: user.Get("twitter_username").String(), + Readme: string(readmeOutput), + } + + sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) + + sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { + Scrape.Type = e.Attr("itemtype") + }) + + sc1.Visit("https://github.com/" + c.Params("user") + "/") + sc := colly.NewCollector( + colly.AllowedDomains("github.com"), + ) + + if Scrape.Type == "http://schema.org/Person" { + // Bio + sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { + Scrape.Bio = e.Attr("data-bio-text") + }) + // Avatar + sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { + Scrape.AvatarUrl = e.Attr("src") + }) + // Metadata (Location/Workplace/Website/Twitter etc.) + sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { + Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + }) + // Followers/Following + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { + Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) + }) + sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { + Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) + }) + // User/Full Name + sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") + }) + + } else { + sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { + Scrape.AvatarUrl = e.Attr("src") + }) + log.Println("Bio and Location routes cannot be scraped for organizations") + } + sc.Visit("https://github.com/" + c.Params("user") + "/") + userArray = append(userArray, Scrape) + + return c.Render("user", fiber.Map{ + "Title": "User " + c.Params("user"), + "user": userArray, + }) } // copied from bluemonday's GitHub repostiory, with some adaptations diff --git a/views/user.html b/views/user.html index 127751f..ddcb207 100644 --- a/views/user.html +++ b/views/user.html @@ -12,7 +12,7 @@

{{.Name}}

{{ end }}

{{.Login}}

- {{ if eq .Type "User" }} + {{ if eq .Type "http://schema.org/Person" }}

{{.Followers}} followers - {{.Following}} following

{{ else }}

{{.Followers}} followers

@@ -48,4 +48,4 @@

User not found

That user doesn't exist.

{{ end }} - \ No newline at end of file + From dca78ff062683c81dc85efa6b9f84a07add49ded Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 16:55:50 +0530 Subject: [PATCH 06/24] add api-less 404 checking --- pages/user.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pages/user.go b/pages/user.go index e45754f..4dffd37 100644 --- a/pages/user.go +++ b/pages/user.go @@ -2,16 +2,16 @@ package pages import ( "codeberg.org/gothub/gothub/utils" + "context" "github.com/carlmjohnson/requests" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" "github.com/microcosm-cc/bluemonday" "log" - "strconv" - - "context" + "net/http" "regexp" + "strconv" "strings" ) @@ -39,7 +39,15 @@ type Ratelimit struct { func HandleUser(c *fiber.Ctx) error { // Declare Array used for displaying data var userArray []User - + resp, err := http.Get("https://github.com/" + c.Params("user")) + if err != nil { + log.Println(err) + } + if resp.StatusCode == 404 { + return c.Status(404).Render("error", fiber.Map{ + "error": "User " + c.Params("user") + " not found", + }) + } // API user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) if user.Get("message").String() == "Not Found" { @@ -83,18 +91,18 @@ func HandleUser(c *fiber.Ctx) error { // User README var readmee string - err := requests. + err0 := requests. URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). ToString(&readmee). Fetch(context.Background()) - if err != nil { + if err0 != nil { err2 := requests. URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). ToString(&readmee). Fetch(context.Background()) if err2 != nil { readmee = "" - log.Println(err) + log.Println(err0) } } mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) From d0843f28cacf4be82414cea654f6d603a9d4ef42 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 17:33:38 +0530 Subject: [PATCH 07/24] make link global var --- pages/user.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/user.go b/pages/user.go index 4dffd37..de9e5d9 100644 --- a/pages/user.go +++ b/pages/user.go @@ -50,6 +50,7 @@ func HandleUser(c *fiber.Ctx) error { } // API user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) + var link string if user.Get("message").String() == "Not Found" { return c.Status(404).Render("error", fiber.Map{ "error": "User " + c.Params("user") + " not found", @@ -73,7 +74,6 @@ func HandleUser(c *fiber.Ctx) error { }) } else { - var link string if user.Get("blog").String() == "" { link = "" @@ -111,7 +111,7 @@ func HandleUser(c *fiber.Ctx) error { // scraping Scrape := User{ - //Link: link, + Link: link, EwTwitter: user.Get("twitter_username").String(), Readme: string(readmeOutput), } From fc19decc13aba4bccc68e5a0fdadb4631131fc3c Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 18:37:13 +0530 Subject: [PATCH 08/24] scrape twitter --- pages/user.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pages/user.go b/pages/user.go index de9e5d9..d2aa65e 100644 --- a/pages/user.go +++ b/pages/user.go @@ -74,7 +74,6 @@ func HandleUser(c *fiber.Ctx) error { }) } else { - if user.Get("blog").String() == "" { link = "" } else { @@ -111,9 +110,8 @@ func HandleUser(c *fiber.Ctx) error { // scraping Scrape := User{ - Link: link, - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), + Link: link, + Readme: string(readmeOutput), } sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) @@ -140,6 +138,7 @@ func HandleUser(c *fiber.Ctx) error { sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.EwTwitter = e.ChildText("a[href*='https://twitter.com/' i]") }) // Followers/Following sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { @@ -153,7 +152,6 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Login = e.ChildText("span[itemprop*='additionalName']") Scrape.Name = e.ChildText("span[itemprop*='name']") }) - } else { sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") @@ -161,6 +159,7 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") + Scrape.EwTwitter = strings.Trim(Scrape.EwTwitter, "@") userArray = append(userArray, Scrape) return c.Render("user", fiber.Map{ From 24cefa1afaeddcf8f965cac27a81c702a9cc9256 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 18:57:45 +0530 Subject: [PATCH 09/24] User contributions --- pages/user.go | 29 +++++++++++++++++------------ views/user.html | 5 +++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pages/user.go b/pages/user.go index d2aa65e..cad1318 100644 --- a/pages/user.go +++ b/pages/user.go @@ -16,18 +16,19 @@ import ( ) type User struct { - Login string - Name string - Bio string - AvatarUrl string - Location string - Following int64 - Followers int64 - Link string - Company string - Type string - EwTwitter string - Readme string + Login string + Name string + Bio string + AvatarUrl string + Location string + Following int64 + Followers int64 + Link string + Company string + Type string + EwTwitter string + Contributions string + Readme string } type Ratelimit struct { @@ -152,6 +153,10 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Login = e.ChildText("span[itemprop*='additionalName']") Scrape.Name = e.ChildText("span[itemprop*='name']") }) + // Contributions + sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { + Scrape.Contributions = e.ChildText("h2") + }) } else { sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") diff --git a/views/user.html b/views/user.html index ddcb207..6f007ef 100644 --- a/views/user.html +++ b/views/user.html @@ -20,6 +20,8 @@ {{ if .Location }}

🌍 {{.Location}}

{{ end }} + {{ if .Timezone }} +

🕑️ {{.Timezone}}

{{ if .Link }}

🔗 {{.Link}}

{{ end }} @@ -29,6 +31,9 @@ {{ if .EwTwitter }}

🐦 {{.EwTwitter}}

{{ end }} + {{ if eq .Type "http://schema.org/Person" }} +

✏️ {{ if .Name }} {{.Name}} {{ else }} {{.Login}} {{ end }} has made {{.Contributions}}

+ {{ end }} {{ if .Bio }} From 7a130bf8cca3ffb2e0d219d76ee9beeb72d658ad Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 19:25:03 +0530 Subject: [PATCH 10/24] add localtime --- pages/user.go | 2 ++ views/user.html | 1 + 2 files changed, 3 insertions(+) diff --git a/pages/user.go b/pages/user.go index cad1318..d1fdc40 100644 --- a/pages/user.go +++ b/pages/user.go @@ -21,6 +21,7 @@ type User struct { Bio string AvatarUrl string Location string + Timezone string Following int64 Followers int64 Link string @@ -138,6 +139,7 @@ func HandleUser(c *fiber.Ctx) error { // Metadata (Location/Workplace/Website/Twitter etc.) sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") Scrape.EwTwitter = e.ChildText("a[href*='https://twitter.com/' i]") }) diff --git a/views/user.html b/views/user.html index 6f007ef..e8d6fd0 100644 --- a/views/user.html +++ b/views/user.html @@ -22,6 +22,7 @@ {{ end }} {{ if .Timezone }}

🕑️ {{.Timezone}}

+ {{ end }} {{ if .Link }}

🔗 {{.Link}}

{{ end }} From 1e0afded3132c35a828e311849cc43389fa995e9 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Wed, 8 Feb 2023 19:47:30 +0530 Subject: [PATCH 11/24] add social links support --- pages/user.go | 23 +++++++++++++++++++---- views/user.html | 17 +++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pages/user.go b/pages/user.go index d1fdc40..418e149 100644 --- a/pages/user.go +++ b/pages/user.go @@ -25,9 +25,12 @@ type User struct { Following int64 Followers int64 Link string + Link1 string + Link2 string + Link3 string + Link4 string Company string Type string - EwTwitter string Contributions string Readme string } @@ -136,12 +139,25 @@ func HandleUser(c *fiber.Ctx) error { sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") }) - // Metadata (Location/Workplace/Website/Twitter etc.) + // Metadata (Location/Workplace/Website/Social etc.) sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") - Scrape.EwTwitter = e.ChildText("a[href*='https://twitter.com/' i]") + // I know this is hacky, I'll get around to fixing it later -arya + e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { + switch i { + case 0: + Scrape.Link1 = el.ChildText("a.Link--primary") + case 1: + Scrape.Link2 = el.ChildText("a.Link--primary") + case 2: + Scrape.Link3 = el.ChildText("a.Link--primary") + case 3: + Scrape.Link4 = el.ChildText("a.Link--primary") + } + }) + }) // Followers/Following sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { @@ -166,7 +182,6 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") - Scrape.EwTwitter = strings.Trim(Scrape.EwTwitter, "@") userArray = append(userArray, Scrape) return c.Render("user", fiber.Map{ diff --git a/views/user.html b/views/user.html index e8d6fd0..8600dde 100644 --- a/views/user.html +++ b/views/user.html @@ -24,13 +24,22 @@

🕑️ {{.Timezone}}

{{ end }} {{ if .Link }} -

🔗 {{.Link}}

+

🌐 {{.Link}}

+ {{ end }} + {{ if .Link1 }} +

🔗 {{.Link1}}

+ {{ end }} + {{ if .Link2 }} +

🔗 {{.Link2}}

+ {{ end }} + {{ if .Link3 }} +

🔗 {{.Link3}}

+ {{ end }} + {{ if .Link4 }} +

🔗 {{.Link4}}

{{ end }} {{ if .Company }}

🏢 {{.Company}}

- {{ end }} - {{ if .EwTwitter }} -

🐦 {{.EwTwitter}}

{{ end }} {{ if eq .Type "http://schema.org/Person" }}

✏️ {{ if .Name }} {{.Name}} {{ else }} {{.Login}} {{ end }} has made {{.Contributions}}

From ba642bf695591e2bab7b58fd579df69d6d22f011 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Thu, 9 Feb 2023 22:11:53 +0530 Subject: [PATCH 12/24] simplify social code --- pages/user.go | 17 ++--------------- views/user.html | 17 +++++------------ 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/pages/user.go b/pages/user.go index 418e149..b60dd8e 100644 --- a/pages/user.go +++ b/pages/user.go @@ -25,10 +25,7 @@ type User struct { Following int64 Followers int64 Link string - Link1 string - Link2 string - Link3 string - Link4 string + Social []string Company string Type string Contributions string @@ -144,18 +141,8 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") - // I know this is hacky, I'll get around to fixing it later -arya e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { - switch i { - case 0: - Scrape.Link1 = el.ChildText("a.Link--primary") - case 1: - Scrape.Link2 = el.ChildText("a.Link--primary") - case 2: - Scrape.Link3 = el.ChildText("a.Link--primary") - case 3: - Scrape.Link4 = el.ChildText("a.Link--primary") - } + Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary")) }) }) diff --git a/views/user.html b/views/user.html index 8600dde..aad9a14 100644 --- a/views/user.html +++ b/views/user.html @@ -26,18 +26,11 @@ {{ if .Link }}

🌐 {{.Link}}

{{ end }} - {{ if .Link1 }} -

🔗 {{.Link1}}

- {{ end }} - {{ if .Link2 }} -

🔗 {{.Link2}}

- {{ end }} - {{ if .Link3 }} -

🔗 {{.Link3}}

- {{ end }} - {{ if .Link4 }} -

🔗 {{.Link4}}

- {{ end }} + {{ if .Social }} + {{range .Social}} +

🔗 {{.}}

+ {{ end }} + {{ end }} {{ if .Company }}

🏢 {{.Company}}

{{ end }} From 4e57c47eb0e117b4ac86f90b30fd114ad77197c4 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Fri, 10 Feb 2023 14:43:06 +0530 Subject: [PATCH 13/24] user agent --- README.md | 4 +++- pages/user.go | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5ef46e5..5b579ce 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,11 @@ go build -o gothub If you don't want to use port 3000 for GotHub, set the GOTHUB_PORT environment variable to the port you want to use. +## Scraping +Variables: 1) GOTHUB_USER_AGENT - Sets the User Agent to use for making the requests. Defaults to Chrome 110 on Windows 10. ## DMCA/Legal notice All content shown on a GotHub instance is from GitHub. Any issues with content shown on a GotHub instance needs to be reported to GitHub, not the instance host's internet or domain provider. -GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc. \ No newline at end of file +GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc. diff --git a/pages/user.go b/pages/user.go index b60dd8e..df1a884 100644 --- a/pages/user.go +++ b/pages/user.go @@ -13,6 +13,7 @@ import ( "regexp" "strconv" "strings" + "os" ) type User struct { @@ -116,16 +117,18 @@ func HandleUser(c *fiber.Ctx) error { Readme: string(readmeOutput), } - sc1 := colly.NewCollector(colly.AllowedDomains("github.com")) + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + sc1 := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent),) sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { Scrape.Type = e.Attr("itemtype") }) sc1.Visit("https://github.com/" + c.Params("user") + "/") - sc := colly.NewCollector( - colly.AllowedDomains("github.com"), - ) + sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent),) if Scrape.Type == "http://schema.org/Person" { // Bio From 4319d9d835470e5e1dfd19be5e96465c21397def Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Fri, 10 Feb 2023 16:01:57 +0530 Subject: [PATCH 14/24] use avatar proxy instead of directly serving from GH --- pages/user.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pages/user.go b/pages/user.go index df1a884..8ee296e 100644 --- a/pages/user.go +++ b/pages/user.go @@ -172,6 +172,8 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") + Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/") + Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl userArray = append(userArray, Scrape) return c.Render("user", fiber.Map{ From b0fa5a50fb5a01ade9020030861ecde486e6797a Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Sat, 11 Feb 2023 18:37:47 +0530 Subject: [PATCH 15/24] add organizations support --- pages/user.go | 14 +++++++++++--- views/user.html | 7 +++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pages/user.go b/pages/user.go index 8ee296e..c85e794 100644 --- a/pages/user.go +++ b/pages/user.go @@ -10,10 +10,10 @@ import ( "github.com/microcosm-cc/bluemonday" "log" "net/http" + "os" "regexp" "strconv" "strings" - "os" ) type User struct { @@ -27,6 +27,7 @@ type User struct { Followers int64 Link string Social []string + Organization []string Company string Type string Contributions string @@ -121,14 +122,14 @@ func HandleUser(c *fiber.Ctx) error { if !ok { UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" } - sc1 := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent),) + sc1 := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) sc1.OnHTML("div[itemtype]", func(e *colly.HTMLElement) { Scrape.Type = e.Attr("itemtype") }) sc1.Visit("https://github.com/" + c.Params("user") + "/") - sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent),) + sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) if Scrape.Type == "http://schema.org/Person" { // Bio @@ -165,6 +166,13 @@ func HandleUser(c *fiber.Ctx) error { sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { Scrape.Contributions = e.ChildText("h2") }) + // Organizations + sc.OnHTML("div.mt-3", func(e *colly.HTMLElement) { + e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { + Scrape.Organization = append(Scrape.Organization, el.Attr("aria-label")) + }) + }) + } else { sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") diff --git a/views/user.html b/views/user.html index aad9a14..b71f35f 100644 --- a/views/user.html +++ b/views/user.html @@ -31,6 +31,13 @@

🔗 {{.}}

{{ end }} {{ end }} + {{ if .Organization }} +

Organizations: + {{range .Organization}} + {{.}} + {{ end }} +

+ {{ end }} {{ if .Company }}

🏢 {{.Company}}

{{ end }} From ca17adad2a94314a73f33f120679143fe54b2623 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 12:58:10 +0530 Subject: [PATCH 16/24] add user status --- go.mod | 1 + go.sum | 2 ++ pages/user.go | 12 ++++++++++++ views/user.html | 3 +++ 4 files changed, 18 insertions(+) diff --git a/go.mod b/go.mod index 6124a9b..d0c3642 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/antchfx/xpath v1.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/enescakir/emoji v1.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gocolly/colly v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index 369c681..de5ddd8 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,8 @@ github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55k github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= +github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= diff --git a/pages/user.go b/pages/user.go index c85e794..3fa32b7 100644 --- a/pages/user.go +++ b/pages/user.go @@ -4,6 +4,7 @@ import ( "codeberg.org/gothub/gothub/utils" "context" "github.com/carlmjohnson/requests" + "github.com/enescakir/emoji" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" @@ -20,6 +21,8 @@ type User struct { Login string Name string Bio string + Status string + StatusEmoji string AvatarUrl string Location string Timezone string @@ -162,6 +165,14 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Login = e.ChildText("span[itemprop*='additionalName']") Scrape.Name = e.ChildText("span[itemprop*='name']") }) + // User Status + sc.OnHTML("div.user-status-circle-badge", func(e *colly.HTMLElement) { + Scrape.Status = e.ChildText("div.user-status-message-wrapper div") + }) + // User Status Emoji + sc.OnHTML("div.user-status-circle-badge div.user-status-emoji-container g-emoji", func(e *colly.HTMLElement) { + Scrape.StatusEmoji = e.Attr("alias") + }) // Contributions sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { Scrape.Contributions = e.ChildText("h2") @@ -182,6 +193,7 @@ func HandleUser(c *fiber.Ctx) error { sc.Visit("https://github.com/" + c.Params("user") + "/") Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/") Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl + Scrape.StatusEmoji = emoji.Parse(":"+Scrape.StatusEmoji+":") userArray = append(userArray, Scrape) return c.Render("user", fiber.Map{ diff --git a/views/user.html b/views/user.html index b71f35f..8a5eabd 100644 --- a/views/user.html +++ b/views/user.html @@ -17,6 +17,9 @@ {{ else }}

{{.Followers}} followers

{{ end }} + {{ if .StatusEmoji }} +

{{.StatusEmoji}} {{.Status}}

+ {{ end }} {{ if .Location }}

🌍 {{.Location}}

{{ end }} From 7f3b788bbf9b856b17f429aced9798329e01b51f Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 14:36:53 +0530 Subject: [PATCH 17/24] use same UA for both colly and api --- utils/getrequest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/getrequest.go b/utils/getrequest.go index 6edf2c0..01b6936 100644 --- a/utils/getrequest.go +++ b/utils/getrequest.go @@ -14,7 +14,7 @@ func GetRequest(url string) gjson.Result { } r.Header.Set("Accept", "application/vnd.github.v3+json") - r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36") + r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36") client := &http.Client{} res, err := client.Do(r) From b22cecdc820af9b72b7494f85ef3b5cb0fb5cdb9 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 14:55:39 +0530 Subject: [PATCH 18/24] move ugcpolicy to utils --- pages/repo.go | 2 +- pages/user.go | 198 +-------------------------------------------- utils/ugcpolicy.go | 197 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 196 deletions(-) create mode 100644 utils/ugcpolicy.go diff --git a/pages/repo.go b/pages/repo.go index 8e7792c..987b6f3 100644 --- a/pages/repo.go +++ b/pages/repo.go @@ -69,7 +69,7 @@ func HandleRepo(c *fiber.Ctx) error { mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) // Trust Nobody - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) + readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) repoArray = append(repoArray, Repo{ Fullname: repo.Get("full_name").String(), diff --git a/pages/user.go b/pages/user.go index 3fa32b7..a59ef7a 100644 --- a/pages/user.go +++ b/pages/user.go @@ -8,11 +8,9 @@ import ( "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" - "github.com/microcosm-cc/bluemonday" "log" "net/http" "os" - "regexp" "strconv" "strings" ) @@ -113,7 +111,7 @@ func HandleUser(c *fiber.Ctx) error { } mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) // Sanitize the user README incase there is any weird shit in it - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) + readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) // scraping Scrape := User{ @@ -191,9 +189,11 @@ func HandleUser(c *fiber.Ctx) error { log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") + // Fixing the output a bit Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/") Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl Scrape.StatusEmoji = emoji.Parse(":"+Scrape.StatusEmoji+":") + // Add scrape-based info to userArray userArray = append(userArray, Scrape) return c.Render("user", fiber.Map{ @@ -201,195 +201,3 @@ func HandleUser(c *fiber.Ctx) error { "user": userArray, }) } - -// copied from bluemonday's GitHub repostiory, with some adaptations -func UGCPolicy() *bluemonday.Policy { - - p := bluemonday.NewPolicy() - - /////////////////////// - // Global attributes // - /////////////////////// - - // "class" is not permitted as we are not allowing users to style their own - // content - - p.AllowStandardAttributes() - - ////////////////////////////// - // Global URL format policy // - ////////////////////////////// - - p.AllowStandardURLs() - - //////////////////////////////// - // Declarations and structure // - //////////////////////////////// - - // "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are - // expecting user generated content to be a fragment of HTML and not a full - // document. - - ////////////////////////// - // Sectioning root tags // - ////////////////////////// - - // "article" and "aside" are permitted and takes no attributes - p.AllowElements("article", "aside") - - // "body" is not permitted as we are expecting user generated content to be a fragment - // of HTML and not a full document. - - // "details" is permitted, including the "open" attribute which can either - // be blank or the value "open". - p.AllowAttrs( - "open", - ).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details") - - // "fieldset" is not permitted as we are not allowing forms to be created. - - // "figure" is permitted and takes no attributes - p.AllowElements("figure") - - // "nav" is not permitted as it is assumed that the site (and not the user) - // has defined navigation elements - - // "section" is permitted and takes no attributes - p.AllowElements("section") - - // "summary" is permitted and takes no attributes - p.AllowElements("summary") - - ////////////////////////// - // Headings and footers // - ////////////////////////// - - // "footer" is not permitted as we expect user content to be a fragment and - // not structural to this extent - - // "h1" through "h6" are permitted and take no attributes - p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6") - - // "header" is not permitted as we expect user content to be a fragment and - // not structural to this extent - - // "hgroup" is permitted and takes no attributes - p.AllowElements("hgroup") - - ///////////////////////////////////// - // Content grouping and separating // - ///////////////////////////////////// - - // "blockquote" is permitted, including the "cite" attribute which must be - // a standard URL. - p.AllowAttrs("cite").OnElements("blockquote") - - // "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes - p.AllowElements("br", "div", "hr", "p", "span", "wbr") - - /////////// - // Links // - /////////// - - // "a" is permitted - p.AllowAttrs("href").OnElements("a") - - // "area" is permitted along with the attributes that map image maps work - p.AllowAttrs("name").Matching( - regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`), - ).OnElements("map") - p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area") - p.AllowAttrs("coords").Matching( - regexp.MustCompile(`^([0-9]+,)+[0-9]+$`), - ).OnElements("area") - p.AllowAttrs("href").OnElements("area") - p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area") - p.AllowAttrs("shape").Matching( - regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`), - ).OnElements("area") - p.AllowAttrs("usemap").Matching( - regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`), - ).OnElements("img") - - // "link" is not permitted - - ///////////////////// - // Phrase elements // - ///////////////////// - - // The following are all inline phrasing elements - p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em", - "figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var") - - // "q" is permitted and "cite" is a URL and handled by URL policies - p.AllowAttrs("cite").OnElements("q") - - // "time" is permitted - p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time") - - //////////////////// - // Style elements // - //////////////////// - - // block and inline elements that impart no semantic meaning but style the - // document - p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u") - - // "style" is not permitted as we are not yet sanitising CSS and it is an - // XSS attack vector - - ////////////////////// - // HTML5 Formatting // - ////////////////////// - - // "bdi" "bdo" are permitted - p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo") - - // "rp" "rt" "ruby" are permitted - p.AllowElements("rp", "rt", "ruby") - - /////////////////////////// - // HTML5 Change tracking // - /////////////////////////// - - // "del" "ins" are permitted - p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins") - p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins") - - /////////// - // Lists // - /////////// - - p.AllowLists() - - //////////// - // Tables // - //////////// - - p.AllowTables() - - /////////// - // Forms // - /////////// - - // By and large, forms are not permitted. However there are some form - // elements that can be used to present data, and we do permit those - // - // "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist" - // "textarea" "optgroup" "option" are all not permitted - - // "meter" is permitted - p.AllowAttrs( - "value", - "min", - "max", - "low", - "high", - "optimum", - ).Matching(bluemonday.Number).OnElements("meter") - - // "progress" is permitted - p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress") - - return p -} diff --git a/utils/ugcpolicy.go b/utils/ugcpolicy.go new file mode 100644 index 0000000..665aa41 --- /dev/null +++ b/utils/ugcpolicy.go @@ -0,0 +1,197 @@ +package utils +import ( + "regexp" + "github.com/microcosm-cc/bluemonday" +) +// copied from bluemonday's GitHub repostiory, with some adaptations +func UGCPolicy() *bluemonday.Policy { + + p := bluemonday.NewPolicy() + + /////////////////////// + // Global attributes // + /////////////////////// + + // "class" is not permitted as we are not allowing users to style their own + // content + + p.AllowStandardAttributes() + + ////////////////////////////// + // Global URL format policy // + ////////////////////////////// + + p.AllowStandardURLs() + + //////////////////////////////// + // Declarations and structure // + //////////////////////////////// + + // "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are + // expecting user generated content to be a fragment of HTML and not a full + // document. + + ////////////////////////// + // Sectioning root tags // + ////////////////////////// + + // "article" and "aside" are permitted and takes no attributes + p.AllowElements("article", "aside") + + // "body" is not permitted as we are expecting user generated content to be a fragment + // of HTML and not a full document. + + // "details" is permitted, including the "open" attribute which can either + // be blank or the value "open". + p.AllowAttrs( + "open", + ).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details") + + // "fieldset" is not permitted as we are not allowing forms to be created. + + // "figure" is permitted and takes no attributes + p.AllowElements("figure") + + // "nav" is not permitted as it is assumed that the site (and not the user) + // has defined navigation elements + + // "section" is permitted and takes no attributes + p.AllowElements("section") + + // "summary" is permitted and takes no attributes + p.AllowElements("summary") + + ////////////////////////// + // Headings and footers // + ////////////////////////// + + // "footer" is not permitted as we expect user content to be a fragment and + // not structural to this extent + + // "h1" through "h6" are permitted and take no attributes + p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6") + + // "header" is not permitted as we expect user content to be a fragment and + // not structural to this extent + + // "hgroup" is permitted and takes no attributes + p.AllowElements("hgroup") + + ///////////////////////////////////// + // Content grouping and separating // + ///////////////////////////////////// + + // "blockquote" is permitted, including the "cite" attribute which must be + // a standard URL. + p.AllowAttrs("cite").OnElements("blockquote") + + // "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes + p.AllowElements("br", "div", "hr", "p", "span", "wbr") + + /////////// + // Links // + /////////// + + // "a" is permitted + p.AllowAttrs("href").OnElements("a") + + // "area" is permitted along with the attributes that map image maps work + p.AllowAttrs("name").Matching( + regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`), + ).OnElements("map") + p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area") + p.AllowAttrs("coords").Matching( + regexp.MustCompile(`^([0-9]+,)+[0-9]+$`), + ).OnElements("area") + p.AllowAttrs("href").OnElements("area") + p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area") + p.AllowAttrs("shape").Matching( + regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`), + ).OnElements("area") + p.AllowAttrs("usemap").Matching( + regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`), + ).OnElements("img") + + // "link" is not permitted + + ///////////////////// + // Phrase elements // + ///////////////////// + + // The following are all inline phrasing elements + p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em", + "figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var") + + // "q" is permitted and "cite" is a URL and handled by URL policies + p.AllowAttrs("cite").OnElements("q") + + // "time" is permitted + p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time") + + //////////////////// + // Style elements // + //////////////////// + + // block and inline elements that impart no semantic meaning but style the + // document + p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u") + + // "style" is not permitted as we are not yet sanitising CSS and it is an + // XSS attack vector + + ////////////////////// + // HTML5 Formatting // + ////////////////////// + + // "bdi" "bdo" are permitted + p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo") + + // "rp" "rt" "ruby" are permitted + p.AllowElements("rp", "rt", "ruby") + + /////////////////////////// + // HTML5 Change tracking // + /////////////////////////// + + // "del" "ins" are permitted + p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins") + p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins") + + /////////// + // Lists // + /////////// + + p.AllowLists() + + //////////// + // Tables // + //////////// + + p.AllowTables() + + /////////// + // Forms // + /////////// + + // By and large, forms are not permitted. However there are some form + // elements that can be used to present data, and we do permit those + // + // "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist" + // "textarea" "optgroup" "option" are all not permitted + + // "meter" is permitted + p.AllowAttrs( + "value", + "min", + "max", + "low", + "high", + "optimum", + ).Matching(bluemonday.Number).OnElements("meter") + + // "progress" is permitted + p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress") + + return p +} + From 924a41ddcaa09e88d01caf2b018208bbd4da9745 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 19:29:44 +0530 Subject: [PATCH 19/24] add support for organizations --- pages/user.go | 34 +++++++++++++++++++++++++++------- utils/ugcpolicy.go | 5 +++-- views/user.html | 7 +++++-- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/pages/user.go b/pages/user.go index a59ef7a..86a6b18 100644 --- a/pages/user.go +++ b/pages/user.go @@ -23,12 +23,13 @@ type User struct { StatusEmoji string AvatarUrl string Location string + Email string Timezone string Following int64 Followers int64 Link string Social []string - Organization []string + Organizations []string Company string Type string Contributions string @@ -115,7 +116,6 @@ func HandleUser(c *fiber.Ctx) error { // scraping Scrape := User{ - Link: link, Readme: string(readmeOutput), } @@ -131,7 +131,6 @@ func HandleUser(c *fiber.Ctx) error { sc1.Visit("https://github.com/" + c.Params("user") + "/") sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) - if Scrape.Type == "http://schema.org/Person" { // Bio sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { @@ -149,7 +148,7 @@ func HandleUser(c *fiber.Ctx) error { e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary")) }) - + Scrape.Link = e.ChildText("a[itemprop*='url']") }) // Followers/Following sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { @@ -178,21 +177,42 @@ func HandleUser(c *fiber.Ctx) error { // Organizations sc.OnHTML("div.mt-3", func(e *colly.HTMLElement) { e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { - Scrape.Organization = append(Scrape.Organization, el.Attr("aria-label")) + Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) }) }) } else { + sc.OnHTML("div.container-xl div.flex-md-items-center div.flex-1", func(e *colly.HTMLElement) { + Scrape.Bio = e.ChildText("div.color-fg-muted div") + Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64) + Scrape.Name = e.ChildText("h1.h2") + Scrape.Location = e.ChildText("span[itemprop*='location']") + Scrape.Link = e.ChildText("a[itemprop*='url']") + Scrape.Email = e.ChildText("a[itemprop*='email']") + e.ForEach("a.Link--primary", func(i int, el *colly.HTMLElement) { + Scrape.Social = append(Scrape.Social, el.Attr("href")) + }) + }) sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { Scrape.AvatarUrl = e.Attr("src") + Scrape.Login = e.Attr("alt") }) - log.Println("Bio and Location routes cannot be scraped for organizations") } sc.Visit("https://github.com/" + c.Params("user") + "/") // Fixing the output a bit Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/") Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl - Scrape.StatusEmoji = emoji.Parse(":"+Scrape.StatusEmoji+":") + if Scrape.StatusEmoji != "" { + Scrape.StatusEmoji = emoji.Parse(":" + Scrape.StatusEmoji + ":") + } + Scrape.Login = strings.TrimPrefix(Scrape.Login, "@") // Only for orgs + if strings.HasPrefix(Scrape.Link, "https://") { + Scrape.Link = strings.TrimPrefix(Scrape.Link, "https://") + } else if strings.HasPrefix(Scrape.Link, "http://") { + Scrape.Link = strings.TrimPrefix(Scrape.Link, "http://") + } else { + log.Println("Has no prefix") + } // Add scrape-based info to userArray userArray = append(userArray, Scrape) diff --git a/utils/ugcpolicy.go b/utils/ugcpolicy.go index 665aa41..ffcb4a8 100644 --- a/utils/ugcpolicy.go +++ b/utils/ugcpolicy.go @@ -1,8 +1,10 @@ package utils + import ( - "regexp" "github.com/microcosm-cc/bluemonday" + "regexp" ) + // copied from bluemonday's GitHub repostiory, with some adaptations func UGCPolicy() *bluemonday.Policy { @@ -194,4 +196,3 @@ func UGCPolicy() *bluemonday.Policy { return p } - diff --git a/views/user.html b/views/user.html index 8a5eabd..6d052f2 100644 --- a/views/user.html +++ b/views/user.html @@ -29,14 +29,17 @@ {{ if .Link }}

🌐 {{.Link}}

{{ end }} + {{ if .Email }} +

✉️ {{.Email}}

+ {{ end }} {{ if .Social }} {{range .Social}}

🔗 {{.}}

{{ end }} {{ end }} - {{ if .Organization }} + {{ if .Organizations }}

Organizations: - {{range .Organization}} + {{range .Organizations}} {{.}} {{ end }}

From 05793d776ad9d0ac4acfd1a27cead14c921bbda0 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 19:46:27 +0530 Subject: [PATCH 20/24] add support for listing public organization members --- pages/user.go | 7 +++++++ views/user.html | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/pages/user.go b/pages/user.go index 86a6b18..e7fa079 100644 --- a/pages/user.go +++ b/pages/user.go @@ -30,6 +30,7 @@ type User struct { Link string Social []string Organizations []string + OrgMembers []string Company string Type string Contributions string @@ -197,6 +198,11 @@ func HandleUser(c *fiber.Ctx) error { Scrape.AvatarUrl = e.Attr("src") Scrape.Login = e.Attr("alt") }) + sc.OnHTML("div.clearfix", func(e *colly.HTMLElement) { + e.ForEach("a[data-hovercard-type*='user'] img", func(i int, el *colly.HTMLElement) { + Scrape.OrgMembers = append(Scrape.OrgMembers, strings.TrimPrefix(el.Attr("alt"), "@")) + }) + }) } sc.Visit("https://github.com/" + c.Params("user") + "/") // Fixing the output a bit @@ -213,6 +219,7 @@ func HandleUser(c *fiber.Ctx) error { } else { log.Println("Has no prefix") } + // Add scrape-based info to userArray userArray = append(userArray, Scrape) diff --git a/views/user.html b/views/user.html index 6d052f2..dd47b53 100644 --- a/views/user.html +++ b/views/user.html @@ -44,6 +44,13 @@ {{ end }}

{{ end }} + {{ if .OrgMembers }} +

Members: + {{range .OrgMembers}} + @{{.}} + {{ end }} +

+ {{ end }} {{ if .Company }}

🏢 {{.Company}}

{{ end }} From e215d628e4138b589bb1793ec0090c63c2f47fea Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 20:08:36 +0530 Subject: [PATCH 21/24] simplify scraping and remove un-needed OnHTML calls --- pages/user.go | 72 +++++++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/pages/user.go b/pages/user.go index e7fa079..4049649 100644 --- a/pages/user.go +++ b/pages/user.go @@ -133,71 +133,53 @@ func HandleUser(c *fiber.Ctx) error { sc1.Visit("https://github.com/" + c.Params("user") + "/") sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) if Scrape.Type == "http://schema.org/Person" { - // Bio - sc.OnHTML("div[data-bio-text]", func(e *colly.HTMLElement) { - Scrape.Bio = e.Attr("data-bio-text") - }) - // Avatar - sc.OnHTML("img[alt*=Avatar]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - }) - // Metadata (Location/Workplace/Website/Social etc.) - sc.OnHTML("ul.vcard-details", func(e *colly.HTMLElement) { + sc.OnHTML("div.js-profile-editable-replace", func(e *colly.HTMLElement) { + // Main info + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") + Scrape.Bio = e.ChildText("div[data-bio-text] div") + Scrape.AvatarUrl = e.ChildAttr("img[alt*='Avatar']", "src") + // Metadata Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.Link = e.ChildText("a[itemprop*='url']") e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary")) }) - Scrape.Link = e.ChildText("a[itemprop*='url']") - }) - // Followers/Following - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i]", func(e *colly.HTMLElement) { - Scrape.Followers, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - sc.OnHTML("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i]", func(e *colly.HTMLElement) { - Scrape.Following, err = strconv.ParseInt(e.ChildText("span"), 10, 64) - }) - // User/Full Name - sc.OnHTML("h1.vcard-names", func(e *colly.HTMLElement) { - Scrape.Login = e.ChildText("span[itemprop*='additionalName']") - Scrape.Name = e.ChildText("span[itemprop*='name']") - }) - // User Status - sc.OnHTML("div.user-status-circle-badge", func(e *colly.HTMLElement) { + // Followers/Following + Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=followers' i] span"), 10, 64) + Scrape.Following, err = strconv.ParseInt(e.ChildText("a[href*='https://github.com/"+c.Params("user")+"?tab=following' i] span"), 10, 64) + // Organizations + e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { + Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) + }) + // User Status Scrape.Status = e.ChildText("div.user-status-message-wrapper div") - }) - // User Status Emoji - sc.OnHTML("div.user-status-circle-badge div.user-status-emoji-container g-emoji", func(e *colly.HTMLElement) { - Scrape.StatusEmoji = e.Attr("alias") + Scrape.StatusEmoji = e.ChildAttr("div.user-status-circle-badge div.user-status-emoji-container g-emoji" ,"alias") }) // Contributions sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { Scrape.Contributions = e.ChildText("h2") }) - // Organizations - sc.OnHTML("div.mt-3", func(e *colly.HTMLElement) { - e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { - Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) - }) - }) - } else { - sc.OnHTML("div.container-xl div.flex-md-items-center div.flex-1", func(e *colly.HTMLElement) { - Scrape.Bio = e.ChildText("div.color-fg-muted div") - Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64) + sc.OnHTML("div.container-xl div.flex-md-items-center", func(e *colly.HTMLElement) { + // Main info + Scrape.Login = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "alt") // Hacky Scrape.Name = e.ChildText("h1.h2") + Scrape.Bio = e.ChildText("div.color-fg-muted div") + Scrape.AvatarUrl = e.ChildAttr("img[alt*='@"+c.Params("user")+"' i]", "src") + // Metadata Scrape.Location = e.ChildText("span[itemprop*='location']") - Scrape.Link = e.ChildText("a[itemprop*='url']") Scrape.Email = e.ChildText("a[itemprop*='email']") + Scrape.Link = e.ChildText("a[itemprop*='url']") e.ForEach("a.Link--primary", func(i int, el *colly.HTMLElement) { Scrape.Social = append(Scrape.Social, el.Attr("href")) }) + // Followers + Scrape.Followers, err = strconv.ParseInt(e.ChildText("a[href*='/orgs/"+c.Params("user")+"/followers' i] span"), 10, 64) }) - sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { - Scrape.AvatarUrl = e.Attr("src") - Scrape.Login = e.Attr("alt") - }) + // Org Members sc.OnHTML("div.clearfix", func(e *colly.HTMLElement) { e.ForEach("a[data-hovercard-type*='user'] img", func(i int, el *colly.HTMLElement) { Scrape.OrgMembers = append(Scrape.OrgMembers, strings.TrimPrefix(el.Attr("alt"), "@")) From 2482a2454ad507755154cdc3fd9a726672695ae6 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 20:09:53 +0530 Subject: [PATCH 22/24] make getrequest lookup UA envvar --- utils/getrequest.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/getrequest.go b/utils/getrequest.go index 01b6936..8516537 100644 --- a/utils/getrequest.go +++ b/utils/getrequest.go @@ -13,8 +13,12 @@ func GetRequest(url string) gjson.Result { panic(err) } + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } r.Header.Set("Accept", "application/vnd.github.v3+json") - r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36") + r.Header.Set("User-Agent", UserAgent) client := &http.Client{} res, err := client.Do(r) From 9d84a830a900a7613aba393203efc067506b066f Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 20:15:58 +0530 Subject: [PATCH 23/24] we dont need gh api for users anymore --- pages/user.go | 56 ++++++--------------------------------------- utils/getrequest.go | 2 +- 2 files changed, 8 insertions(+), 50 deletions(-) diff --git a/pages/user.go b/pages/user.go index 4049649..75f3aa2 100644 --- a/pages/user.go +++ b/pages/user.go @@ -37,63 +37,22 @@ type User struct { Readme string } -type Ratelimit struct { - Remaining int64 - Limit int64 -} - // HandleUser handles the user page. func HandleUser(c *fiber.Ctx) error { // Declare Array used for displaying data var userArray []User + resp, err := http.Get("https://github.com/" + c.Params("user")) if err != nil { log.Println(err) } if resp.StatusCode == 404 { + // I need a better way to do this return c.Status(404).Render("error", fiber.Map{ - "error": "User " + c.Params("user") + " not found", + "error": "User " + c.Params("user") + " not found", }) } - // API - user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) - var link string - if user.Get("message").String() == "Not Found" { - return c.Status(404).Render("error", fiber.Map{ - "error": "User " + c.Params("user") + " not found", - }) - } - if strings.Contains(user.Get("message").String(), "rate limit") { // dont wanna get the status code so i'll just do this instead 👍 - ratelimitJSON := utils.GetRequest("https://api.github.com/rate_limit") - log.Println(ratelimitJSON) - var ratelimitArray []Ratelimit - ratelimitArray = append(ratelimitArray, Ratelimit{ - Remaining: ratelimitJSON.Get("resources.core.remaining").Int(), - Limit: ratelimitJSON.Get("resources.core.limit").Int(), - }) - - log.Println(ratelimitArray) - - return c.Render("ratelimit", fiber.Map{ - "Title": "GitHub API /users endpoint rate limit exceeded", - "ratelimit": ratelimitArray, - }) - } else { - - if user.Get("blog").String() == "" { - link = "" - } else { - link = user.Get("blog").String() - if strings.HasPrefix(link, "https://") { - link = strings.TrimPrefix(link, "https://") - } else if strings.HasPrefix(link, "http://") { - link = strings.TrimPrefix(link, "http://") - } else { - log.Println("Has no prefix") - } - } - } // User README var readmee string @@ -115,7 +74,7 @@ func HandleUser(c *fiber.Ctx) error { // Sanitize the user README incase there is any weird shit in it readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) - // scraping + // Scraping Scrape := User{ Readme: string(readmeOutput), } @@ -189,17 +148,16 @@ func HandleUser(c *fiber.Ctx) error { sc.Visit("https://github.com/" + c.Params("user") + "/") // Fixing the output a bit Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/") - Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl + Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl // Avatar needs to be in /avatar so its proxied if Scrape.StatusEmoji != "" { - Scrape.StatusEmoji = emoji.Parse(":" + Scrape.StatusEmoji + ":") + Scrape.StatusEmoji = emoji.Parse(":" + Scrape.StatusEmoji + ":") // Convert the emoji code to an actual emoji } Scrape.Login = strings.TrimPrefix(Scrape.Login, "@") // Only for orgs + // Remove HTTP(s) from user website url if it exists if strings.HasPrefix(Scrape.Link, "https://") { Scrape.Link = strings.TrimPrefix(Scrape.Link, "https://") } else if strings.HasPrefix(Scrape.Link, "http://") { Scrape.Link = strings.TrimPrefix(Scrape.Link, "http://") - } else { - log.Println("Has no prefix") } // Add scrape-based info to userArray diff --git a/utils/getrequest.go b/utils/getrequest.go index 8516537..1c486fa 100644 --- a/utils/getrequest.go +++ b/utils/getrequest.go @@ -3,7 +3,7 @@ package utils import ( "io" "net/http" - + "os" "github.com/tidwall/gjson" ) From 7b15df28d9a716d663e210d5a58edf6254e50866 Mon Sep 17 00:00:00 2001 From: Arya Kiran Date: Mon, 13 Feb 2023 20:20:08 +0530 Subject: [PATCH 24/24] fix status message --- pages/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/user.go b/pages/user.go index 75f3aa2..7e3cd1b 100644 --- a/pages/user.go +++ b/pages/user.go @@ -114,7 +114,7 @@ func HandleUser(c *fiber.Ctx) error { Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) }) // User Status - Scrape.Status = e.ChildText("div.user-status-message-wrapper div") + Scrape.Status = e.ChildText("div.user-status-circle-badge div.user-status-message-wrapper div") Scrape.StatusEmoji = e.ChildAttr("div.user-status-circle-badge div.user-status-emoji-container g-emoji" ,"alias") }) // Contributions