diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..6c209ec --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*.yml] +end_of_line = lf +insert_final_newline = false +indent_style = tab +indent_size = 2 +charset = utf-8 \ No newline at end of file diff --git a/.gitea/ISSUE_TEMPLATE/instance.yml b/.gitea/ISSUE_TEMPLATE/instance.yml new file mode 100644 index 0000000..a96f3a6 --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/instance.yml @@ -0,0 +1,51 @@ +name: "Instance Request" +about: "Have an instance you want to add to the list? Use this template." +title: "[New Instance] " +labels: +- instance +body: + - type: input + attributes: + label: Server Host + description: Where is your server hosted? (Input your ISP's name if you are hosting at home) + placeholder: OVHcloud, Hetzner, Telenor, etc. + validations: + required: true + - type: input + attributes: + label: Location + description: Which country is your server hosted? + placeholder: Germany, Norway, France etc. + validations: + required: true + - type: checkboxes + attributes: + label: Cloudflare + description: Is your server using Cloudflare's proxy/DDoS protection? + options: + - label: My server uses Cloudflare's proxy/DDoS protection + - type: input + attributes: + label: Instance URL + description: What is the URL of your instance? + placeholder: https://gothub.example.org + validations: + required: true + - type: dropdown + attributes: + label: Which branch of GotHub are you using? + options: + - master + - dev + - '"latest" docker image' + - type: textarea + attributes: + label: Other details + description: Anything else we should know? + placeholder: We use a custom fork over at https://git.example.org/foo/gothub + - type: input + attributes: + label: Contact details + description: Input your contact information should we need to contact you privately. We recommend Matrix. If you don't have a contact method, or don't want your contact method public, tell us. + placeholder: "@foo:example.org" + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2bfce47..6a8121a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,10 +18,13 @@ # Dependency directories (remove the comment below to include it) # vendor/ +# Gothub binary (on linux and mac) +/gothub + # Go workspace file go.work tmp/ -# Gothub binary. -gothub \ No newline at end of file +# Fiber compressed files +*.fiber.gz diff --git a/.woodpecker.yml b/.woodpecker.yml deleted file mode 100644 index bce6cb8..0000000 --- a/.woodpecker.yml +++ /dev/null @@ -1,72 +0,0 @@ -pipeline: - dockerize: - when: - branch: [main, master] - event: [push] - image: woodpeckerci/plugin-docker-buildx - settings: - dockerfile: Dockerfile - registry: https://codeberg.org/v2 - repo: codeberg.org/gothub/gothub - platforms: linux/amd64,linux/arm64 - tag: latest - username: - from_secret: user - password: - from_secret: passwd - secrets: [user, passwd] - #dockerize-arm64: - # environment: - # - GOOS=linux - # - GOARCH=arm64 - # when: - # branch: [main, master] - # event: [push] - # image: plugins/docker - # settings: - # registry: codeberg.org - # repo: codeberg.org/gothub/gothub - # username: - # from_secret: user - # password: - # from_secret: passwd - # # password = key or actual password - # tags: arm64 - - dockerize-dev: - when: - event: [push] - branch: [dev] - image: woodpeckerci/plugin-docker-buildx - settings: - dockerfile: Dockerfile - registry: https://codeberg.org/v2 - repo: codeberg.org/gothub/gothub - platforms: linux/amd64,linux/arm64 - tag: dev - username: - from_secret: user - password: - from_secret: passwd - secrets: [user, passwd] - - #dockerize-dev-arm64: - # environment: - # - ARCH= - # - GOOS=linux - # - GOARCH=arm64 - # when: - # event: [push] - # branch: [dev] - # image: plugins/docker - # settings: - # registry: codeberg.org - # repo: codeberg.org/gothub/gothub - # username: - # from_secret: user - # password: - # from_secret: passwd - # # password = key or actual password - # tags: ${CI_COMMIT_SHA:0:8}-arm64 - # dockerfile: Dockerfile-arm64 - # # You can thank the drone docker maintainer for not providing a buildx option. \ No newline at end of file diff --git a/.woodpecker/dockerize-dev.yml b/.woodpecker/dockerize-dev.yml new file mode 100644 index 0000000..030744c --- /dev/null +++ b/.woodpecker/dockerize-dev.yml @@ -0,0 +1,17 @@ +pipeline: + dockerize-dev: + when: + event: push + branch: dev + image: woodpeckerci/plugin-docker-buildx + settings: + dockerfile: Dockerfile + registry: https://codeberg.org/v2 + repo: codeberg.org/gothub/gothub + platforms: linux/amd64,linux/arm64 + tag: dev + username: + from_secret: user + password: + from_secret: passwd + secrets: [user, passwd] \ No newline at end of file diff --git a/.woodpecker/dockerize.yml b/.woodpecker/dockerize.yml new file mode 100644 index 0000000..efd83c7 --- /dev/null +++ b/.woodpecker/dockerize.yml @@ -0,0 +1,17 @@ +pipeline: + dockerize: + when: + branch: main, master + event: push + image: woodpeckerci/plugin-docker-buildx + settings: + dockerfile: Dockerfile + registry: https://codeberg.org/v2 + repo: codeberg.org/gothub/gothub + platforms: linux/amd64,linux/arm64 + tag: latest + username: + from_secret: user + password: + from_secret: passwd + secrets: [user, passwd] \ No newline at end of file diff --git a/.woodpecker/instancejson.yml b/.woodpecker/instancejson.yml new file mode 100644 index 0000000..5a852b3 --- /dev/null +++ b/.woodpecker/instancejson.yml @@ -0,0 +1,28 @@ +# Credit to Hygna (https://codeberg.org/Hygna) +pipeline: + instancejson: + image: python:3-bullseye + secrets: [ssh_key, mail] + when: + - event: cron + cron: instancejson + - path: + include: ["README.md", ".woodpecker/instancejson.yml"] + branch: dev + commands: + # SSH configuration + - mkdir ~/.ssh + - ssh-keyscan -t ed25519 codeberg.org >> ~/.ssh/known_hosts + - eval `ssh-agent` + - echo "$SSH_KEY" | tr -d '\r' | ssh-add - + # Running script + - git clone git@codeberg.org:gothub/gothub-instances.git instances + - cd instances + - pip install -r requirements.txt + - python3 main.py + # Git configuration + - git config --global user.email $MAIL + - git config --global user.name "Woodpecker CI" + - git add instances.json + - git commit -m "Updated instances" + - git push -u origin master diff --git a/README.md b/README.md index b5262d0..b8a68a3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # GotHub Alternative front-end for GitHub written with Go. (WIP) + [Matrix](https://matrix.to/#/%23gothub%3Atrygve.me?via=projectsegfau.lt&via=vern.cc&via=matrix.org&via=trygve.me) ## Features @@ -11,7 +12,27 @@ Alternative front-end for GitHub written with Go. (WIP) - Save bandwidth - no JavaScript/tracking bloat loaded! ## Instances -The instances list is in the [wiki](https://codeberg.org/gothub/gothub/wiki/Instances). +### Difference between branches +The ``master`` branch runs on the master branch of GotHub. This means that the instance is running a stable, tested version of GotHub that doesn't get updated as much as ``dev``. Doesn't mean they're not bad though! + +``dev`` instances run on the dev branch of GotHub. This might mean they are more up-to-date than other instances, but at the same time, they might not be (instance admin might not always update their instance, no auto-updates etc.). You can get the latest and greatest straight out of Git, but you could meet on bugs and other nasty things. +| Link | Cloudflare | Country | ISP | Branch | +| -------- | ---------- | ----------- | ----- | ----- | +| [gh.odyssey346.dev](https://gh.odyssey346.dev) | No | 🇫🇷 | OVHcloud (Trolling Solutions) | master | +| [gh.riverside.rocks](https://gh.riverside.rocks) | No | 🇺🇸 | Comcast | master | +| [gh.vern.cc](https://gh.vern.cc)[^1] | No | 🇺🇸 | Hetzner | master | +| [gh.akisblack.dev](https://gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | master | +| [gothub.lunar.icu](https://gothub.lunar.icu) | Yes | 🇩🇪 | Unesty | master | +| [gh.phreedom.club](https://gh.phreedom.club) | No | 🇮🇸 | Flokinet | master | +| [gh.dev.odyssey346.dev](https://gh.dev.odyssey346.dev) | No | 🇫🇷 | OVHcloud (Trolling Solutions) | dev | +| [gothub.esmailelbob.xyz](https://gothub.esmailelbob.xyz) | No | 🇨🇦 | OVHcloud | dev | +| [dev.gh.akisblack.dev](https://dev.gh.akisblack.dev) | No | 🇩🇪 | OVHcloud | dev | + +Have an instance you want on the list? Make an issue or PR! + +You can find a json-list of the same in [gothub/gothub-instances](https://codeberg.org/gothub/gothub-instances). + +[^1]: Has some modifications: https://git.vern.cc/vern/modifications/src/branch/master/gothub ## Setup ### Docker @@ -25,6 +46,9 @@ If you are using Portainer or Podman, you know how to create the GotHub containe ### Manual You will need an installation of Go 1.19+ installed on your system. + +We recommend setting the `GO_TELEMETRY` environment variable to `off`. In the future, the Go CLI will send telemetry to Google, which you might not want. + ```bash git clone https://codeberg.org/gothub/gothub.git cd gothub @@ -34,9 +58,24 @@ go build -o gothub If you don't want to use port 3000 for GotHub, set the GOTHUB_PORT environment variable to the port you want to use. +### Web server configuration (Caddy) +We recommend Caddy for the web server, as it provides automatic HTTPS, HTTP/3 and is easy to setup. +```caddyfile +gothub.example.com { + reverse_proxy localhost:(the port for GotHub) +} +``` +## Scraping +Other than for the explore page, gothub uses scraping in order to retrieve the information. + +This makes it hard for gothub to get ratelimited. + +You can also set the GOTHUB_USER_AGENT envvar in order to set the user agent with which GotHub makes requests to Github. + +By default, we use the user agent of Chrome 110 on Windows 10. ## DMCA/Legal notice All content shown on a GotHub instance is from GitHub. Any issues with content shown on a GotHub instance needs to be reported to GitHub, not the instance host's internet or domain provider. -GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc. \ No newline at end of file +GitHub is a registered trademark of GitHub, Inc. GotHub is not affiliated with GitHub, Inc. diff --git a/cmd/gothub/update.go b/cmd/gothub/update.go new file mode 100644 index 0000000..a9ed0c9 --- /dev/null +++ b/cmd/gothub/update.go @@ -0,0 +1,62 @@ +package gothub + +import ( + "bytes" + + "github.com/spf13/cobra" + + "os" + "os/exec" + "runtime" +) + +var updateCmd = &cobra.Command{ + Use: "update", + Short: "Update GotHub.", + Long: `This command checks for updates and re-builds GotHub. Does not work on Docker`, + Run: func(cmd *cobra.Command, args []string) { + update() + }, +} + +func init() { + rootCmd.AddCommand(updateCmd) +} + +func update() { + if os.Getenv("DOCKER") == "true" { + println("The GotHub update command does not work on Docker. Please re-create the container with a new image instead.") + os.Exit(1) + } + gitPull := exec.Command("git", "pull") + var out bytes.Buffer + gitPull.Stdout = &out + err := gitPull.Run() + if out.String() == "Already up to date.\r" { + println("GotHub is already up to date.") + os.Exit(0) + } + if err != nil { + println("Couldn't run Git pull. Are you using Git?", err) + os.Exit(1) + } + if runtime.GOOS == "windows" { + err := exec.Command("go", "build", "-o", "gothub.exe").Run() + if err != nil { + println("Couldn't build GotHub.", err) + os.Exit(1) + } else { + println("GotHub updated successfully.") + os.Exit(0) + } + } else { + err := exec.Command("go", "build").Run() + if err != nil { + println("Couldn't build GotHub.", err) + os.Exit(1) + } else { + println("GotHub updated successfully.") + os.Exit(0) + } + } +} diff --git a/go.mod b/go.mod index 9729f25..d0c3642 100644 --- a/go.mod +++ b/go.mod @@ -14,17 +14,32 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.3.0 // indirect + github.com/antchfx/xmlquery v1.3.15 // indirect + github.com/antchfx/xpath v1.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/enescakir/emoji v1.0.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/gocolly/colly v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jawher/mow.cli v1.2.0 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.15.14 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/rivo/uniseg v0.4.3 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -32,4 +47,7 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 8ba98c8..de5ddd8 100644 --- a/go.sum +++ b/go.sum @@ -54,7 +54,11 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw= +github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= +github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/alecthomas/chroma/v2 v2.4.0 h1:Loe2ZjT5x3q1bcWwemqyqEi8p11/IV/ncFCeLYDpWC4= github.com/alecthomas/chroma/v2 v2.4.0/go.mod h1:6kHzqF5O6FUSJzBXW7fXELjb+e+7OXW4UpoPqMO7IBQ= github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= @@ -64,6 +68,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= +github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= +github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= +github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= +github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -108,9 +120,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= +github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -136,6 +151,10 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.40.1/go.mod h1:Gko04sLksnHbzLSRBFWPFdzM9Ws9pRxvvIaohJK1dsk= github.com/gofiber/fiber/v2 v2.41.0 h1:YhNoUS/OTjEz+/WLYuQ01xI7RXgKEFnGBKMagAu5f0M= @@ -148,6 +167,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -174,6 +194,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK0Ja9a3OUa2Fo+EaN0cbLu0eKpBwPFzc8= @@ -258,6 +279,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jawher/mow.cli v1.2.0 h1:e6ViPPy+82A/NFF/cfbq3Lr6q4JHKT9tyHwTCcUQgQw= +github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -265,6 +288,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -347,6 +372,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -363,6 +390,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -370,6 +398,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -392,6 +422,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= @@ -416,6 +447,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -454,6 +486,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -496,8 +529,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= @@ -529,6 +564,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -595,12 +631,15 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -610,6 +649,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -668,6 +709,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -710,6 +752,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -815,6 +858,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pages/fileview.go b/pages/fileview.go new file mode 100644 index 0000000..b1256a6 --- /dev/null +++ b/pages/fileview.go @@ -0,0 +1,67 @@ +package pages + +import ( + "bufio" + "bytes" + "context" + "html/template" + + htmlfmt "github.com/alecthomas/chroma/v2/formatters/html" + "github.com/alecthomas/chroma/v2/lexers" + "github.com/alecthomas/chroma/v2/styles" + "github.com/carlmjohnson/requests" + "github.com/gofiber/fiber/v2" +) + +// FileView is the file view page +func FileView(c *fiber.Ctx) error { + var file string + url := "https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + c.Params("branch") + "/" + c.Params("+") + err := requests. + URL(url). + ToString(&file). + Fetch(context.Background()) + if err != nil { + return c.Status(404).Render("error", fiber.Map{ + "error": err, + }) + } + + lexer := lexers.Match(c.Params("file")) + if lexer == nil { + lexer = lexers.Fallback + } + + formatter := htmlfmt.New( + htmlfmt.WithClasses(true), + htmlfmt.WithLineNumbers(true), + htmlfmt.PreventSurroundingPre(false), + ) + + buf := bytes.Buffer{} + writer := bufio.NewWriter(&buf) + style := styles.Get("native") + + tokens, err := lexer.Tokenise(nil, file) + if err != nil { + return c.Status(500).Render("error", fiber.Map{ + "error": err, + }) + } + formatter.Format(writer, style, tokens) + + cssbuf := bytes.Buffer{} + cssw := bufio.NewWriter(&cssbuf) + formatter.WriteCSS(cssw, style) + + writer.Flush() + cssw.Flush() + + return c.Render("file", fiber.Map{ + "file": template.HTML(buf.String()), + "css": template.CSS(cssbuf.String()), + "fullname": c.Params("user") + "/" + c.Params("repo"), + "name": c.Params("file"), + "branch": c.Params("branch"), + }) +} diff --git a/pages/repo.go b/pages/repo.go index 8e7792c..84705ed 100644 --- a/pages/repo.go +++ b/pages/repo.go @@ -1,27 +1,28 @@ package pages import ( - "context" - "log" - "codeberg.org/gothub/gothub/utils" + "context" "github.com/carlmjohnson/requests" + "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "github.com/gomarkdown/markdown" + "log" + "net/http" + "os" ) type Repo struct { Fullname string Description string - HtmlUrl string - Fork bool Parent string - Stars int64 - Forks int64 - Watchers int64 - Language string + Stars string + Forks string + Watchers string + Language []string License string DefaultBranch string + Readme string } type RepoFiles struct { @@ -35,55 +36,85 @@ type RepoFiles struct { func HandleRepo(c *fiber.Ctx) error { var repoArray []Repo var repoFilesArray []RepoFiles - // get repo - repo := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo")) - if repo.Get("message").String() == "Not Found" { + + resp, statusErr := http.Get("https://github.com/" + c.Params("user") + "/" + c.Params("repo")) + if statusErr != nil { + log.Println(statusErr) + } + if resp.StatusCode == 404 { + // I need a better way to do this return c.Status(404).Render("error", fiber.Map{ "title": "Error", "error": "Repository " + c.Params("user") + "/" + c.Params("repo") + " not found", }) } - repoFiles := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo") + "/contents") - bruh := repoFiles.Get("#.@pretty").Array() - for _, item := range bruh { - repoFilesArray = append(repoFilesArray, RepoFiles{ - Name: item.Get("path").String(), - Path: item.Get("path").String(), - Type: item.Get("type").String(), - Fullname: repo.Get("full_name").String(), - DefaultBranch: repo.Get("default_branch").String(), - }) + + // Scraping + Scrape := Repo{} + + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" } - var readmee string + sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) + sc.OnHTML("div.Layout-sidebar", func(e *colly.HTMLElement) { + Scrape.Fullname = c.Params("user") + "/" + c.Params("repo") + Scrape.Description = e.ChildText("p.f4") + Scrape.Stars = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/stargazers' i] strong") + Scrape.Watchers = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/watchers' i] strong") + Scrape.Forks = e.ChildText("a[href*='/" + c.Params("user") + "/" + c.Params("repo") + "/network/members' i] strong") + Scrape.License = e.ChildText("a[data-analytics-event*='{\"category\":\"Repository Overview\",\"action\":\"click\",\"label\":\"location:sidebar;file:license\"}']") + }) + sc.OnHTML("div#readme", func(e *colly.HTMLElement) { + Scrape.Readme = e.ChildText("a[href*='#readme']") + }) + sc.OnHTML("div.BorderGrid-cell ul.list-style-none", func(e *colly.HTMLElement) { + e.ForEach("li.d-inline .d-inline-flex", func(i int, el *colly.HTMLElement) { + Scrape.Language = append(Scrape.Language, el.ChildText("span.text-bold")+" "+el.ChildText("span:contains('%')")) + }) + }) + sc.OnHTML("div#repository-container-header", func(e *colly.HTMLElement) { + Scrape.Parent = e.ChildText("span.text-small a") + }) + sc.OnHTML("summary[title*='Switch branches or tags']", func(e *colly.HTMLElement) { + Scrape.DefaultBranch = e.ChildText("span.css-truncate-target") + }) + sc.OnHTML("div.js-details-container div.Details-content--hidden-not-important", func(e *colly.HTMLElement) { + e.ForEach("div.js-navigation-item", func(i int, el *colly.HTMLElement) { + var FileType string + if el.ChildAttr("div.flex-shrink-0 svg", "aria-label") == "Directory" { + FileType = "dir" + } else { + FileType = "file" + } + repoFilesArray = append(repoFilesArray, RepoFiles{ + Name: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"), + Path: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"), + Type: FileType, + Fullname: Scrape.Fullname, + DefaultBranch: Scrape.DefaultBranch, + }) + }) + }) + sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/") + // Add scrape-based info to repoArray + repoArray = append(repoArray, Scrape) + + // README + var readmee string err := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + repo.Get("default_branch").String() + "/README.md"). + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + Scrape.DefaultBranch + "/" + Scrape.Readme). ToString(&readmee). Fetch(context.Background()) if err != nil { readmee = "" log.Println(err) } - mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) - // Trust Nobody - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) - - repoArray = append(repoArray, Repo{ - Fullname: repo.Get("full_name").String(), - Description: repo.Get("description").String(), - HtmlUrl: repo.Get("html_url").String(), - Fork: repo.Get("fork").Bool(), - Stars: repo.Get("stargazers_count").Int(), - Forks: repo.Get("forks_count").Int(), - Watchers: repo.Get("watchers_count").Int(), - Language: repo.Get("language").String(), - License: repo.Get("license").Get("name").String(), - Parent: repo.Get("parent").Get("full_name").String(), - DefaultBranch: repo.Get("default_branch").String(), - }) + readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) return c.Render("repo", fiber.Map{ "title": "Repository " + c.Params("user") + "/" + c.Params("repo"), diff --git a/pages/user.go b/pages/user.go index a09070a..59d26c8 100644 --- a/pages/user.go +++ b/pages/user.go @@ -1,311 +1,189 @@ package pages import ( - "fmt" + "context" "log" - - "github.com/carlmjohnson/requests" - "github.com/gofiber/fiber/v2" - "github.com/gomarkdown/markdown" - "github.com/microcosm-cc/bluemonday" + "os" + "strings" "codeberg.org/gothub/gothub/utils" - - "context" - "regexp" - "strings" + "github.com/carlmjohnson/requests" + "github.com/enescakir/emoji" + "github.com/gocolly/colly" + "github.com/gofiber/fiber/v2" + "github.com/gomarkdown/markdown" ) type User struct { - Login string - Name string - Bio string - AvatarUrl string - Location string - Following int64 - Followers int64 - Link string - Company string - Type string - EwTwitter string - Readme string + Login string + Name string + Bio string + Status string + StatusEmoji string + AvatarUrl string + Location string + Email string + Timezone string + Following string + Followers string + Link string + Social []string + Organizations []string + OrgMembers []string + Company string + Type string + Contributions string + Readme string + MainRepos []RepoList + PinOrPopular string } - -type Ratelimit struct { - Remaining int64 - Limit int64 +type RepoList struct { + Name string + Type string + Desc string + Lang string + Stars string + Forks string + ForkOf string } // HandleUser handles the user page. func HandleUser(c *fiber.Ctx) error { - user := utils.GetRequest("https://api.github.com/users/" + c.Params("user")) - if user.Get("message").String() == "Not Found" { + // Declare Array used for displaying data + var userArray []User + + // User README + var readmee string + err0 := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err0 != nil { + err2 := requests. + URL("https://raw.githubusercontent.com/" + c.Params("user") + "/.github/master/profile/README.md"). + ToString(&readmee). + Fetch(context.Background()) + if err2 != nil { + readmee = "" + log.Println(err0) + } + } + mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) + // Sanitize the user README incase there is any weird shit in it + readmeOutput := utils.UGCPolicy().SanitizeBytes(mightBeUnsafe) + + // Scraping + Scrape := User{ + Readme: string(readmeOutput), + } + + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + sc1 := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) + + sc1.OnHTML("div[itemscope]", func(e *colly.HTMLElement) { + Scrape.Type = e.Attr("itemtype") + }) + + sc1.Visit("https://github.com/" + c.Params("user") + "/") + sc := colly.NewCollector(colly.AllowedDomains("github.com"), colly.UserAgent(UserAgent)) + if Scrape.Type == "http://schema.org/Person" { + sc.OnHTML("div.js-profile-editable-replace", func(e *colly.HTMLElement) { + // Main info + Scrape.Login = e.ChildText("span[itemprop*='additionalName']") + Scrape.Name = e.ChildText("span[itemprop*='name']") + Scrape.Bio = e.ChildText("div[data-bio-text] div") + Scrape.AvatarUrl = e.ChildAttr("img[alt*='Avatar']", "src") + // Metadata + Scrape.Location = e.ChildText("li[itemprop*='homeLocation'] span") + Scrape.Timezone = e.ChildText("li[itemprop*='localTime'] span") + Scrape.Company = e.ChildText("li[itemprop*='worksFor'] span") + Scrape.Link = e.ChildText("li[itemprop*='url'] a") + e.ForEach("li[itemprop*='social']", func(i int, el *colly.HTMLElement) { + Scrape.Social = append(Scrape.Social, el.ChildText("a.Link--primary")) + }) + // Followers/Following + Scrape.Followers = e.ChildText("a[href*='https://github.com/" + c.Params("user") + "?tab=followers' i] span") + Scrape.Following = e.ChildText("a[href*='https://github.com/" + c.Params("user") + "?tab=following' i] span") + // Organizations + e.ForEach("a[data-hovercard-type*='organization']", func(i int, el *colly.HTMLElement) { + Scrape.Organizations = append(Scrape.Organizations, el.Attr("aria-label")) + }) + // User Status + Scrape.Status = e.ChildText("div.user-status-circle-badge div.user-status-message-wrapper div") + Scrape.StatusEmoji = e.ChildAttr("div.user-status-circle-badge div.user-status-emoji-container g-emoji", "alias") + }) + // Contributions + sc.OnHTML("div.js-yearly-contributions", func(e *colly.HTMLElement) { + Scrape.Contributions = e.ChildText("h2") + }) + } else if Scrape.Type == "http://schema.org/Code" { + sc.OnHTML("div.container-xl div.flex-md-items-center div.flex-1", func(e *colly.HTMLElement) { + // Main info + Scrape.Name = e.ChildText("h1.h2") + Scrape.Bio = e.ChildText("div.color-fg-muted div") + // Metadata + Scrape.Location = e.ChildText("span[itemprop*='location']") + Scrape.Email = e.ChildText("a[itemprop*='email']") + Scrape.Link = e.ChildText("a[itemprop*='url']") + e.ForEach("a.Link--primary", func(i int, el *colly.HTMLElement) { + Scrape.Social = append(Scrape.Social, el.Attr("href")) + }) + // Followers + Scrape.Followers = e.ChildText("a[href*='/orgs/" + c.Params("user") + "/followers' i] span") + }) + sc.OnHTML("img[alt*='@"+c.Params("user")+"' i]", func(e *colly.HTMLElement) { + Scrape.AvatarUrl = e.Attr("src") + Scrape.Login = e.Attr("alt") + }) + // Org Members + sc.OnHTML("div.clearfix", func(e *colly.HTMLElement) { + e.ForEach("a[data-hovercard-type*='user'] img", func(i int, el *colly.HTMLElement) { + Scrape.OrgMembers = append(Scrape.OrgMembers, strings.TrimPrefix(el.Attr("alt"), "@")) + }) + }) + } else { return c.Status(404).Render("error", fiber.Map{ "error": "User " + c.Params("user") + " not found", }) } - if strings.Contains(user.Get("message").String(), "rate limit") { // dont wanna get the status code so i'll just do this instead 👍 - ratelimitJSON := utils.GetRequest("https://api.github.com/rate_limit") - fmt.Println(ratelimitJSON) - var ratelimitArray []Ratelimit - - ratelimitArray = append(ratelimitArray, Ratelimit{ - Remaining: ratelimitJSON.Get("resources.core.remaining").Int(), - Limit: ratelimitJSON.Get("resources.core.limit").Int(), - }) - - fmt.Println(ratelimitArray) - - return c.Render("ratelimit", fiber.Map{ - "Title": "GitHub API /users endpoint rate limit exceeded", - "ratelimit": ratelimitArray, - }) - } else { - var userArray []User - - var readmee string - - err := requests. - URL("https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("user") + "/master/README.md"). - ToString(&readmee). - Fetch(context.Background()) - if err != nil { - readmee = "" - log.Println(err) - } - - mightBeUnsafe := markdown.ToHTML([]byte(readmee), nil, nil) - - // Trust Nobody - readmeOutput := UGCPolicy().SanitizeBytes(mightBeUnsafe) - - var link string - - if user.Get("blog").String() == "" { - link = "" - } else { - link = user.Get("blog").String() - if strings.HasPrefix(link, "https://") { - link = strings.TrimPrefix(link, "https://") - } else if strings.HasPrefix(link, "http://") { - link = strings.TrimPrefix(link, "http://") - } else { - log.Println("Has no prefix") - } - } - - userArray = append(userArray, User{ - Login: user.Get("login").String(), - Name: user.Get("name").String(), - Bio: user.Get("bio").String(), - AvatarUrl: "/avatar/" + user.Get("id").String(), - Location: user.Get("location").String(), - Following: user.Get("following").Int(), - Followers: user.Get("followers").Int(), - Link: link, - Company: user.Get("company").String(), - Type: user.Get("type").String(), - EwTwitter: user.Get("twitter_username").String(), - Readme: string(readmeOutput), - }) - - fmt.Println(userArray) - - return c.Render("user", fiber.Map{ - "Title": "User " + c.Params("user"), - "user": userArray, + sc.OnHTML("div.js-pinned-items-reorder-container", func(e *colly.HTMLElement) { + e.ForEach("div.pinned-item-list-item-content", func(i int, el *colly.HTMLElement) { + var MainRepo RepoList + MainRepo = RepoList{} // Clear data if old data is present + MainRepo.Name = el.ChildAttr("div.width-full a", "href") + MainRepo.Type = el.ChildText("div.width-full span.Label") + MainRepo.Desc = el.ChildText("p.pinned-item-desc") + MainRepo.Lang = el.ChildText("p.color-fg-muted span[itemprop*='programmingLanguage']") + MainRepo.Stars = el.ChildText("p.color-fg-muted a[href*='/stargazers' i]") + MainRepo.Forks = el.ChildText("p.color-fg-muted a[href*='/forks' i]") + MainRepo.ForkOf = el.ChildText("p.text-small a.Link--muted") + Scrape.MainRepos = append(Scrape.MainRepos, MainRepo) }) + Scrape.PinOrPopular = e.ChildText("h2.text-normal") + }) + sc.Visit("https://github.com/" + c.Params("user") + "/") + // Fixing the output a bit + Scrape.AvatarUrl = strings.TrimPrefix(Scrape.AvatarUrl, "https://avatars.githubusercontent.com/u/") + Scrape.AvatarUrl = "/avatar/" + Scrape.AvatarUrl // Avatar needs to be in /avatar so its proxied + if Scrape.StatusEmoji != "" { + Scrape.StatusEmoji = emoji.Parse(":" + Scrape.StatusEmoji + ":") // Convert the emoji code to an actual emoji } -} - -// copied from bluemonday's GitHub repostiory, with some adaptations -func UGCPolicy() *bluemonday.Policy { - - p := bluemonday.NewPolicy() - - /////////////////////// - // Global attributes // - /////////////////////// - - // "class" is not permitted as we are not allowing users to style their own - // content - - p.AllowStandardAttributes() - - ////////////////////////////// - // Global URL format policy // - ////////////////////////////// - - p.AllowStandardURLs() - - //////////////////////////////// - // Declarations and structure // - //////////////////////////////// - - // "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are - // expecting user generated content to be a fragment of HTML and not a full - // document. - - ////////////////////////// - // Sectioning root tags // - ////////////////////////// - - // "article" and "aside" are permitted and takes no attributes - p.AllowElements("article", "aside") - - // "body" is not permitted as we are expecting user generated content to be a fragment - // of HTML and not a full document. - - // "details" is permitted, including the "open" attribute which can either - // be blank or the value "open". - p.AllowAttrs( - "open", - ).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details") - - // "fieldset" is not permitted as we are not allowing forms to be created. - - // "figure" is permitted and takes no attributes - p.AllowElements("figure") - - // "nav" is not permitted as it is assumed that the site (and not the user) - // has defined navigation elements - - // "section" is permitted and takes no attributes - p.AllowElements("section") - - // "summary" is permitted and takes no attributes - p.AllowElements("summary") - - ////////////////////////// - // Headings and footers // - ////////////////////////// - - // "footer" is not permitted as we expect user content to be a fragment and - // not structural to this extent - - // "h1" through "h6" are permitted and take no attributes - p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6") - - // "header" is not permitted as we expect user content to be a fragment and - // not structural to this extent - - // "hgroup" is permitted and takes no attributes - p.AllowElements("hgroup") - - ///////////////////////////////////// - // Content grouping and separating // - ///////////////////////////////////// - - // "blockquote" is permitted, including the "cite" attribute which must be - // a standard URL. - p.AllowAttrs("cite").OnElements("blockquote") - - // "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes - p.AllowElements("br", "div", "hr", "p", "span", "wbr") - - /////////// - // Links // - /////////// - - // "a" is permitted - p.AllowAttrs("href").OnElements("a") - - // "area" is permitted along with the attributes that map image maps work - p.AllowAttrs("name").Matching( - regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`), - ).OnElements("map") - p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area") - p.AllowAttrs("coords").Matching( - regexp.MustCompile(`^([0-9]+,)+[0-9]+$`), - ).OnElements("area") - p.AllowAttrs("href").OnElements("area") - p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area") - p.AllowAttrs("shape").Matching( - regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`), - ).OnElements("area") - p.AllowAttrs("usemap").Matching( - regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`), - ).OnElements("img") - - // "link" is not permitted - - ///////////////////// - // Phrase elements // - ///////////////////// - - // The following are all inline phrasing elements - p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em", - "figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var") - - // "q" is permitted and "cite" is a URL and handled by URL policies - p.AllowAttrs("cite").OnElements("q") - - // "time" is permitted - p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time") - - //////////////////// - // Style elements // - //////////////////// - - // block and inline elements that impart no semantic meaning but style the - // document - p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u") - - // "style" is not permitted as we are not yet sanitising CSS and it is an - // XSS attack vector - - ////////////////////// - // HTML5 Formatting // - ////////////////////// - - // "bdi" "bdo" are permitted - p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo") - - // "rp" "rt" "ruby" are permitted - p.AllowElements("rp", "rt", "ruby") - - /////////////////////////// - // HTML5 Change tracking // - /////////////////////////// - - // "del" "ins" are permitted - p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins") - p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins") - - /////////// - // Lists // - /////////// - - p.AllowLists() - - //////////// - // Tables // - //////////// - - p.AllowTables() - - /////////// - // Forms // - /////////// - - // By and large, forms are not permitted. However there are some form - // elements that can be used to present data, and we do permit those - // - // "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist" - // "textarea" "optgroup" "option" are all not permitted - - // "meter" is permitted - p.AllowAttrs( - "value", - "min", - "max", - "low", - "high", - "optimum", - ).Matching(bluemonday.Number).OnElements("meter") - - // "progress" is permitted - p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress") - - return p + Scrape.Login = strings.TrimPrefix(Scrape.Login, "@") // Only for orgs + // Remove HTTP(s) from user website url if it exists + if strings.HasPrefix(Scrape.Link, "https://") { + Scrape.Link = strings.TrimPrefix(Scrape.Link, "https://") + } else if strings.HasPrefix(Scrape.Link, "http://") { + Scrape.Link = strings.TrimPrefix(Scrape.Link, "http://") + } + + // Add scrape-based info to userArray + userArray = append(userArray, Scrape) + + return c.Render("user", fiber.Map{ + "title": c.Params("user"), + "user": userArray, + }) } diff --git a/public/css/global.css b/public/css/global.css index ef3249b..55b627a 100644 --- a/public/css/global.css +++ b/public/css/global.css @@ -12,12 +12,15 @@ --secondary-background: #353535; --background-darker: #151515; --accent: #00b7c3; + color-scheme: dark; } body { font-family: 'Lato', sans-serif; color: #FFF; background-color: #151515; + transition: ease-in-out .25s; + margin: 0; } ::-webkit-scrollbar { @@ -40,10 +43,10 @@ main { justify-content: space-between; flex-direction: row; padding: 8px; -} -.navbarImg { - height: 30px; + margin-bottom: 8px; + + background-color: var(--background); } .indexGitHub { @@ -57,22 +60,24 @@ main { display: flex; align-items: center; gap: 8px; - transition: opactiy .25s; + transition: ease-in-out .25s; color: var(--text); } .navbarSlogan:hover { - color: var(--accent); + text-decoration: underline; + text-underline-offset: 5px; } .brand:hover { - opacity: .95; + color: var(--text); + text-decoration: underline; } .error { background-color: #252525; padding: 8px; - border-radius: 8px; + border-radius: 4px; font-family: 'Lato', sans-serif; color: #fff; } @@ -96,11 +101,20 @@ a:hover { .exploreIDontKnowWhatToNameThisDiv { background-color: #252525; padding: 8px; - border-radius: 8px; + border-radius: 4px; margin: 8px 0 8px 0; text-overflow: ellipsis; white-space: wrap; + text-decoration: none; + display: flex; + + transition: ease-in-out .25s; +} + +.exploreIDontKnowWhatToNameThisDiv:hover { + background-color: var(--secondary-background); + color: var(--text); } /* URI: /:user */ @@ -108,7 +122,7 @@ a:hover { .userProfile { background-color: var(--background); padding: 8px; - border-radius: 8px; + border-radius: 4px; margin: 8px; display: flex; flex-direction: column; @@ -138,14 +152,14 @@ a:hover { .userBio, .userReadme { background-color: var(--background); padding: 8px; - border-radius: 8px; + border-radius: 4x; margin: 8px; } .userBioText, .userReadmeText { background-color: var(--background-darker); padding: 8px; - border-radius: 8px; + border-radius: 4px; margin-bottom: 0; } @@ -153,11 +167,16 @@ a:hover { margin-top: 0; } +.sociallinks { + display: flex; + flex-direction: row; +} + /* URI: /:user/:repo */ .button { background-color: var(--background); padding: 8px; - border-radius: 8px; + border-radius: 4px; margin: 8px; color: var(--text); text-decoration: none; @@ -166,7 +185,7 @@ a:hover { } .button:hover { - background-color: var(--accent); + background-color: var(--secondary-background); color: var(--text); } @@ -192,7 +211,7 @@ a:hover { /* URI: /file/:user/:repo/:file */ .userReadme pre { padding: 8px; - border-radius: 8px; + border-radius: 4px; /*white-space: pre-wrap;*/ overflow-x: auto; } @@ -212,20 +231,15 @@ a:hover { @media screen and (prefers-color-scheme: light) { :root { --text: #000; - --background: #f1f1f1; - --secondary-background: #e1e1e1; - --background-darker: #fff; + --background: #ececec; + --secondary-background: #f5f5f5; + --background-darker: #dddddd; + color-scheme: light; } body { - background-color: #fff; + background-color: var(--background-darker); color: #000; } - .navbarSlogan { - color: #00b7c3; - } - .brand:hover { - opacity: .75; - } a { color: black; } @@ -233,18 +247,9 @@ a:hover { color: #00b7c3; } .exploreIDontKnowWhatToNameThisDiv { - background-color: #f1f1f1; + background-color: var(--background); color: #000; } - ::-webkit-scrollbar { - width: 8px; - height: 10px; - - } - ::-webkit-scrollbar-thumb { - border-radius: 8px; - background: #f1f1f1; - } } @media screen and (max-width: 900px) { @@ -265,4 +270,8 @@ a:hover { margin-bottom: 8px; margin-right: 0; } + + .sociallinks { + flex-direction: column; + } } diff --git a/serve/serve.go b/serve/serve.go index e876d21..8da421f 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -1,9 +1,6 @@ package serve import ( - "bufio" - "bytes" - "context" "html/template" "log" "os" @@ -12,17 +9,11 @@ import ( "codeberg.org/gothub/gothub/pages" "codeberg.org/gothub/gothub/utils" - "github.com/carlmjohnson/requests" - - htmlfmt "github.com/alecthomas/chroma/v2/formatters/html" - "github.com/alecthomas/chroma/v2/lexers" - "github.com/alecthomas/chroma/v2/styles" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/cache" "github.com/gofiber/fiber/v2/middleware/compress" "github.com/gofiber/fiber/v2/middleware/limiter" - "github.com/gofiber/fiber/v2/middleware/proxy" "github.com/gofiber/fiber/v2/middleware/recover" "github.com/gofiber/template/html" ) @@ -31,7 +22,7 @@ func Serve() { engine := html.New("./views", ".html") engine.AddFunc( - // add unescape function + // Add unescape function. This is needed to render HTML from Markdown. "unescape", func(s string) template.HTML { return template.HTML(s) }, @@ -41,7 +32,7 @@ func Serve() { Views: engine, Prefork: false, AppName: "GotHub", - // fucked up way to fix rate limits + // kind of screwed up way to fix rate limits EnableTrustedProxyCheck: true, TrustedProxies: []string{"0.0.0.0/0"}, ProxyHeader: fiber.HeaderXForwardedFor, @@ -67,8 +58,6 @@ func Serve() { Expiration: 5 * time.Minute, })) - app.Get("/", pages.HandleIndex) - app.Use(compress.New(compress.Config{ Level: compress.LevelBestSpeed, // 1 })) @@ -85,90 +74,44 @@ func Serve() { }, }) - app.Static("/css", "./public/css") - app.Static("/fonts", "./public/fonts") - app.Static("/robots.txt", "./public/robots.txt") - app.Static("/favicon.ico", "./public/assets/favicon.ico") - app.Static("/logo.svg", "./public/assets/logo.svg") + staticConfig := fiber.Static{ + Compress: true, + // Cache-Control: max-age=31536000 + MaxAge: 31536000, + } + + // add global headers + app.Use(func(c *fiber.Ctx) error { + c.Set("X-Frame-Options", "SAMEORIGIN") + c.Set("X-XSS-Protection", "1; mode=block") + c.Set("X-Content-Type-Options", "nosniff") + c.Set("Referrer-Policy", "no-referrer") + c.Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data; font-src 'self'; script-src 'self'; frame-ancestors 'self'; form-action 'self'; base-uri 'self'; connect-src 'self';") + c.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload") + + return c.Next() + }) + + app.Get("/", pages.HandleIndex) + app.Static("/css", "./public/css", staticConfig) + app.Static("/fonts", "./public/fonts", staticConfig) + app.Static("/robots.txt", "./public/robots.txt", staticConfig) + app.Static("/favicon.ico", "./public/assets/favicon.ico", staticConfig) + app.Static("/logo.svg", "./public/assets/logo.svg", staticConfig) app.Get("/explore", ratelimiter, pages.HandleExplore) app.Get("/:user", ratelimiter, pages.HandleUser) app.Get("/avatar/:id", func(c *fiber.Ctx) error { - url := "https://avatars.githubusercontent.com/u/" + c.Params("id") + "?v=4" - if err := proxy.Do(c, url); err != nil { - return err - } - // Remove Server header from response - c.Response().Header.Del(fiber.HeaderServer) + utils.ProxyRequest(c, "https://avatars.githubusercontent.com/u/"+c.Params("id")+"?v=4") return nil }) app.Get("/:user/:repo", ratelimiter, pages.HandleRepo) - app.Get("/file/:user/:repo/:branch/:file", func(c *fiber.Ctx) error { - var file string - url := "https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + c.Params("branch") + "/" + c.Params("file") - err := requests. - URL(url). - ToString(&file). - Fetch(context.Background()) - if err != nil { - return c.Status(404).Render("error", fiber.Map{ - "error": err, - }) - } - - lexer := lexers.Match(c.Params("file")) - if lexer == nil { - lexer = lexers.Fallback - } - - formatter := htmlfmt.New( - htmlfmt.WithClasses(true), - htmlfmt.WithLineNumbers(true), - htmlfmt.PreventSurroundingPre(false), - ) - - buf := bytes.Buffer{} - writer := bufio.NewWriter(&buf) - style := styles.Get("native") - - tokens, err := lexer.Tokenise(nil, file) - if err != nil { - return c.Status(500).Render("error", fiber.Map{ - "error": err, - }) - } - formatter.Format(writer, style, tokens) - - cssbuf := bytes.Buffer{} - cssw := bufio.NewWriter(&cssbuf) - formatter.WriteCSS(cssw, style) - - writer.Flush() - cssw.Flush() - - return c.Render("file", fiber.Map{ - "file": template.HTML(buf.String()), - "css": template.CSS(cssbuf.String()), - "fullname": c.Params("user") + "/" + c.Params("repo"), - "name": c.Params("file"), - "branch": c.Params("branch"), - }) - }) + app.Get("/:user/:repo/tree/:branch/+", pages.FileView) app.Get("/download/:user/:repo/:branch", ratelimiter, func(c *fiber.Ctx) error { - url := "https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/archive/" + c.Params("branch") + ".zip" - if err := proxy.Do(c, url); err != nil { - return err - } - // Remove Server header from response - c.Response().Header.Del(fiber.HeaderServer) + utils.ProxyRequest(c, "https://github.com/"+c.Params("user")+"/"+c.Params("repo")+"/archive/"+c.Params("branch")+".zip") return nil }) app.Get("/raw/:user/:repo/:branch/:file", ratelimiter, func(c *fiber.Ctx) error { - url := "https://raw.githubusercontent.com/" + c.Params("user") + "/" + c.Params("repo") + "/" + c.Params("branch") + "/" + c.Params("file") - if err := proxy.Do(c, url); err != nil { - return err - } - // Remove Server header from response - c.Response().Header.Del(fiber.HeaderServer) + utils.ProxyRequest(c, "https://raw.githubusercontent.com/"+c.Params("user")+"/"+c.Params("repo")+"/"+c.Params("branch")+"/"+c.Params("file")) return nil }) @@ -185,8 +128,6 @@ func Serve() { val, ok := os.LookupEnv("GOTHUB_PORT") if !ok { val = "3000" - } else { - log.Println("Using port from env variable") } log.Fatal(app.Listen(":" + val)) } diff --git a/utils/getrequest.go b/utils/getrequest.go index 6edf2c0..b3ed646 100644 --- a/utils/getrequest.go +++ b/utils/getrequest.go @@ -1,10 +1,10 @@ package utils import ( + "github.com/tidwall/gjson" "io" "net/http" - - "github.com/tidwall/gjson" + "os" ) func GetRequest(url string) gjson.Result { @@ -13,8 +13,12 @@ func GetRequest(url string) gjson.Result { panic(err) } + UserAgent, ok := os.LookupEnv("GOTHUB_USER_AGENT") + if !ok { + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } r.Header.Set("Accept", "application/vnd.github.v3+json") - r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36") + r.Header.Set("User-Agent", UserAgent) client := &http.Client{} res, err := client.Do(r) diff --git a/utils/proxyrequest.go b/utils/proxyrequest.go new file mode 100644 index 0000000..28d0266 --- /dev/null +++ b/utils/proxyrequest.go @@ -0,0 +1,16 @@ +package utils + +import ( + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/proxy" +) + +// ProxyRequest proxies requests to GitHub media +func ProxyRequest(c *fiber.Ctx, url string) error { + if err := proxy.Do(c, url); err != nil { + return err + } + // Remove Server header from response + c.Response().Header.Del(fiber.HeaderServer) + return nil +} diff --git a/utils/ugcpolicy.go b/utils/ugcpolicy.go new file mode 100644 index 0000000..ffcb4a8 --- /dev/null +++ b/utils/ugcpolicy.go @@ -0,0 +1,198 @@ +package utils + +import ( + "github.com/microcosm-cc/bluemonday" + "regexp" +) + +// copied from bluemonday's GitHub repostiory, with some adaptations +func UGCPolicy() *bluemonday.Policy { + + p := bluemonday.NewPolicy() + + /////////////////////// + // Global attributes // + /////////////////////// + + // "class" is not permitted as we are not allowing users to style their own + // content + + p.AllowStandardAttributes() + + ////////////////////////////// + // Global URL format policy // + ////////////////////////////// + + p.AllowStandardURLs() + + //////////////////////////////// + // Declarations and structure // + //////////////////////////////// + + // "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are + // expecting user generated content to be a fragment of HTML and not a full + // document. + + ////////////////////////// + // Sectioning root tags // + ////////////////////////// + + // "article" and "aside" are permitted and takes no attributes + p.AllowElements("article", "aside") + + // "body" is not permitted as we are expecting user generated content to be a fragment + // of HTML and not a full document. + + // "details" is permitted, including the "open" attribute which can either + // be blank or the value "open". + p.AllowAttrs( + "open", + ).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details") + + // "fieldset" is not permitted as we are not allowing forms to be created. + + // "figure" is permitted and takes no attributes + p.AllowElements("figure") + + // "nav" is not permitted as it is assumed that the site (and not the user) + // has defined navigation elements + + // "section" is permitted and takes no attributes + p.AllowElements("section") + + // "summary" is permitted and takes no attributes + p.AllowElements("summary") + + ////////////////////////// + // Headings and footers // + ////////////////////////// + + // "footer" is not permitted as we expect user content to be a fragment and + // not structural to this extent + + // "h1" through "h6" are permitted and take no attributes + p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6") + + // "header" is not permitted as we expect user content to be a fragment and + // not structural to this extent + + // "hgroup" is permitted and takes no attributes + p.AllowElements("hgroup") + + ///////////////////////////////////// + // Content grouping and separating // + ///////////////////////////////////// + + // "blockquote" is permitted, including the "cite" attribute which must be + // a standard URL. + p.AllowAttrs("cite").OnElements("blockquote") + + // "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes + p.AllowElements("br", "div", "hr", "p", "span", "wbr") + + /////////// + // Links // + /////////// + + // "a" is permitted + p.AllowAttrs("href").OnElements("a") + + // "area" is permitted along with the attributes that map image maps work + p.AllowAttrs("name").Matching( + regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`), + ).OnElements("map") + p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area") + p.AllowAttrs("coords").Matching( + regexp.MustCompile(`^([0-9]+,)+[0-9]+$`), + ).OnElements("area") + p.AllowAttrs("href").OnElements("area") + p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area") + p.AllowAttrs("shape").Matching( + regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`), + ).OnElements("area") + p.AllowAttrs("usemap").Matching( + regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`), + ).OnElements("img") + + // "link" is not permitted + + ///////////////////// + // Phrase elements // + ///////////////////// + + // The following are all inline phrasing elements + p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em", + "figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var") + + // "q" is permitted and "cite" is a URL and handled by URL policies + p.AllowAttrs("cite").OnElements("q") + + // "time" is permitted + p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time") + + //////////////////// + // Style elements // + //////////////////// + + // block and inline elements that impart no semantic meaning but style the + // document + p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u") + + // "style" is not permitted as we are not yet sanitising CSS and it is an + // XSS attack vector + + ////////////////////// + // HTML5 Formatting // + ////////////////////// + + // "bdi" "bdo" are permitted + p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo") + + // "rp" "rt" "ruby" are permitted + p.AllowElements("rp", "rt", "ruby") + + /////////////////////////// + // HTML5 Change tracking // + /////////////////////////// + + // "del" "ins" are permitted + p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins") + p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins") + + /////////// + // Lists // + /////////// + + p.AllowLists() + + //////////// + // Tables // + //////////// + + p.AllowTables() + + /////////// + // Forms // + /////////// + + // By and large, forms are not permitted. However there are some form + // elements that can be used to present data, and we do permit those + // + // "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist" + // "textarea" "optgroup" "option" are all not permitted + + // "meter" is permitted + p.AllowAttrs( + "value", + "min", + "max", + "low", + "high", + "optimum", + ).Matching(bluemonday.Number).OnElements("meter") + + // "progress" is permitted + p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress") + + return p +} diff --git a/views/explore.html b/views/explore.html index aaec28d..ae01c1a 100644 --- a/views/explore.html +++ b/views/explore.html @@ -5,8 +5,9 @@

25 repositories shown, sorted by most stars.

{{ if .repos}} {{ range $key, $value := .repos}} -
- {{ .Fullname }} + +
+

{{.Fullname}}

{{.Description}}

{{ if .Language }} {{ if .License }} @@ -21,7 +22,8 @@

⭐ {{.Stars}} 🍴 {{.Forks}} 👀 {{.Watchers}} ⚖️ No license

{{ end }} {{ end }} -
+
+ {{ end }} {{ else }}

There are no trending repositories at the moment.

diff --git a/views/header.html b/views/header.html index bf876bd..94d4ad4 100644 --- a/views/header.html +++ b/views/header.html @@ -13,7 +13,7 @@