mirror of
https://github.com/libredirect/instances.git
synced 2024-12-06 19:26:17 +01:00
Made code readable. Fixed nitter https://github.com/libredirect/browser_extension/issues/975
This commit is contained in:
ManeraKai
@@ -14,7 +14,7 @@ jobs:
|
||||
- run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install requests colorama pyyaml
|
||||
- run: python instances.py
|
||||
- run: python main.py
|
||||
- run: |
|
||||
git config --local user.email "action@github.com"
|
||||
git config --local user.name "GitHub Action"
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
+8
-10
@@ -1,6 +1,8 @@
|
||||
{
|
||||
"cloudflare": [
|
||||
"https://invidious.perennialte.ch",
|
||||
"https://watch.leptons.xyz",
|
||||
"https://piped.r4fo.com",
|
||||
"https://piped.video",
|
||||
"https://piped.coldforge.xyz",
|
||||
"https://schaun.ehwurscht.at",
|
||||
@@ -8,12 +10,9 @@
|
||||
"https://tiktok.wpme.pl",
|
||||
"https://send.aurorabilisim.com",
|
||||
"https://send.cyberjake.xyz",
|
||||
"https://nitter.lucabased.xyz",
|
||||
"https://nitter.rawbit.ninja",
|
||||
"https://n.sneed.network",
|
||||
"https://nitter.freedit.eu",
|
||||
"https://nitter.altgr.xyz",
|
||||
"https://jote.lile.cl",
|
||||
"https://nitter.perennialte.ch",
|
||||
"https://nitter.holo-mix.com",
|
||||
"https://redlib.freedit.eu",
|
||||
"https://redlib.frontendfriendly.xyz",
|
||||
"https://teddit.domain.glass",
|
||||
@@ -39,15 +38,14 @@
|
||||
"https://hp.iqbalrifai.eu.org",
|
||||
"https://hyperpipe.darkness.services",
|
||||
"https://www.openstreetmap.org",
|
||||
"https://paste.3server.de",
|
||||
"https://bonus01.hwb0307.com",
|
||||
"https://privatebin.io",
|
||||
"https://private.rmconsulting.com",
|
||||
"https://bin.oldgate.org",
|
||||
"https://privatebin.luckvintage.com",
|
||||
"https://pb.blackvoid.club",
|
||||
"https://paste.3server.de",
|
||||
"https://n0t.io",
|
||||
"https://bin.oldgate.org",
|
||||
"https://pb.blackvoid.club",
|
||||
"https://sendpass.ru",
|
||||
"https://privatebin.io",
|
||||
"https://wikiless.rawbit.ninja",
|
||||
"https://w.sneed.network",
|
||||
"https://biblioreads.mooo.com",
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
import traceback
|
||||
import logging
|
||||
import requests
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
import re
|
||||
from colorama import Fore, Style
|
||||
import socket
|
||||
|
||||
def get_cloudflare_ips():
|
||||
r = requests.get('https://www.cloudflare.com/ips-v4')
|
||||
return r.text.split('\n')
|
||||
|
||||
|
||||
cloudflare_ips = get_cloudflare_ips()
|
||||
|
||||
|
||||
|
||||
def ip2bin(ip): return "".join(
|
||||
map(
|
||||
str,
|
||||
[
|
||||
"{0:08b}".format(int(x)) for x in ip.split(".")
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def is_cloudflare(url):
|
||||
instance_ip = None
|
||||
try:
|
||||
instance_ip = socket.gethostbyname(urlparse(url).hostname)
|
||||
if instance_ip is None:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
instance_bin = ip2bin(instance_ip)
|
||||
|
||||
for cloudflare_ip_mask in cloudflare_ips:
|
||||
cloudflare_ip = cloudflare_ip_mask.split('/')[0]
|
||||
cloudflare_bin = ip2bin(cloudflare_ip)
|
||||
|
||||
mask = int(cloudflare_ip_mask.split('/')[1])
|
||||
cloudflare_bin_masked = cloudflare_bin[:mask]
|
||||
instance_bin_masked = instance_bin[:mask]
|
||||
|
||||
if cloudflare_bin_masked == instance_bin_masked:
|
||||
print(url + ' is behind ' + Fore.RED +
|
||||
'cloudflare' + Style.RESET_ALL)
|
||||
return True
|
||||
return False
|
||||
|
||||
is_cloudflare('https://pipedapi-libre.kavin.rocks')
|
||||
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"invidious": {
|
||||
"clearnet": [
|
||||
"https://inv.tux.pizza",
|
||||
"https://iv.ggtyler.dev",
|
||||
"https://inv.nadeko.net",
|
||||
"https://invidious.jing.rocks",
|
||||
@@ -31,8 +32,10 @@
|
||||
},
|
||||
"piped": {
|
||||
"clearnet": [
|
||||
"https://watch.leptons.xyz",
|
||||
"https://piped.lunar.icu",
|
||||
"http://ww25.yapi.vyper.me/?subid1=20240820-0828-04b5-b23a-516774864bee",
|
||||
"http://ww25.yapi.vyper.me/?subid1=20240821-0732-1439-ac1f-4980fa340f80",
|
||||
"https://piped.r4fo.com",
|
||||
"https://piped.video",
|
||||
"https://piped.projectsegfau.lt",
|
||||
"https://piped.in.projectsegfau.lt",
|
||||
@@ -50,6 +53,7 @@
|
||||
"https://piped.ngn.tf",
|
||||
"https://piped.coldforge.xyz",
|
||||
"https://piped.cdsp.cz",
|
||||
"https://piped.phoenixthrush.com",
|
||||
"https://piped.private.coffee",
|
||||
"https://schaun.ehwurscht.at",
|
||||
"https://piped.darkness.services"
|
||||
@@ -137,83 +141,23 @@
|
||||
},
|
||||
"nitter": {
|
||||
"clearnet": [
|
||||
"https://nitter:nitter@nitter.nixnet.services",
|
||||
"https://nitter.fdn.fr",
|
||||
"https://nitter.1d4.us",
|
||||
"https://nitter.kavin.rocks",
|
||||
"https://nitter.unixfox.eu",
|
||||
"https://nitter.moomoo.me",
|
||||
"https://bird.trom.tf",
|
||||
"https://nitter.it",
|
||||
"https://nitter.weiler.rocks",
|
||||
"https://nitter.sethforprivacy.com",
|
||||
"https://nitter.nl",
|
||||
"https://nitter.mint.lgbt",
|
||||
"https://nitter.esmailelbob.xyz",
|
||||
"https://nitter.tiekoetter.com",
|
||||
"https://nitter.privacy.com.de",
|
||||
"https://nitter.poast.org",
|
||||
"https://nitter.bird.froth.zone",
|
||||
"https://nitter.cz",
|
||||
"https://xcancel.com",
|
||||
"https://nitter.privacydev.net",
|
||||
"https://tweet.lambda.dance",
|
||||
"https://nitter.kylrth.com",
|
||||
"https://unofficialbird.com",
|
||||
"https://nitter.projectsegfau.lt",
|
||||
"https://nitter.eu.projectsegfau.lt",
|
||||
"https://nitter.in.projectsegfau.lt",
|
||||
"https://singapore.unofficialbird.com",
|
||||
"https://canada.unofficialbird.com",
|
||||
"https://india.unofficialbird.com",
|
||||
"https://nederland.unofficialbird.com",
|
||||
"https://uk.unofficialbird.com",
|
||||
"https://nitter.soopy.moe",
|
||||
"https://nitter.poast.org",
|
||||
"https://nitter.lucabased.xyz",
|
||||
"https://nitter.kavin.rocks",
|
||||
"https://nitter.tiekoetter.com",
|
||||
"https://nitter.qwik.space",
|
||||
"https://nitter.rawbit.ninja",
|
||||
"https://nt.vern.cc",
|
||||
"https://n.sneed.network",
|
||||
"https://nitter.smnz.de",
|
||||
"https://nitter.twei.space",
|
||||
"https://nitter.inpt.fr",
|
||||
"https://nitter.d420.de",
|
||||
"https://nitter.at",
|
||||
"https://bird.habedieeh.re",
|
||||
"https://nitter.hostux.net",
|
||||
"https://nitter.adminforge.de",
|
||||
"https://nitter.platypush.tech",
|
||||
"https://nitter.us.projectsegfau.lt",
|
||||
"https://t.com.sb",
|
||||
"https://nitter.kling.gg",
|
||||
"https://nitter.lunar.icu",
|
||||
"https://twitter.moe.ngo",
|
||||
"https://nitter.freedit.eu",
|
||||
"https://nitter.tux.pizza",
|
||||
"https://nitter.altgr.xyz",
|
||||
"https://jote.lile.cl",
|
||||
"https://nitter.one",
|
||||
"https://nitter.moomoo.me",
|
||||
"https://nitter.bird.froth.zone",
|
||||
"https://nitter.kylrth.com",
|
||||
"https://nitter.io.lol",
|
||||
"https://nitter.no-logs.com",
|
||||
"https://tweet.whateveritworks.org",
|
||||
"https://nitter.fediflix.org",
|
||||
"https://nitter.nohost.network",
|
||||
"https://nitter.simpleprivacy.fr",
|
||||
"https://nitter.onthescent.xyz",
|
||||
"https://nitter.x86-64-unknown-linux-gnu.zip",
|
||||
"https://nitter.private.coffee",
|
||||
"https://nitter.oksocial.net",
|
||||
"https://nitter.woodland.cafe",
|
||||
"https://nitter.dafriser.be",
|
||||
"https://nitter.catsarch.com",
|
||||
"https://nitter.perennialte.ch",
|
||||
"https://nitter.laserdisc.tokyo",
|
||||
"https://nitter.datura.network",
|
||||
"https://nitter.salastil.com",
|
||||
"https://nt.ggtyler.dev",
|
||||
"https://nitter.hyperreal.coffee",
|
||||
"https://nitter.uni-sonia.com",
|
||||
"https://n.opnxng.com",
|
||||
"https://nitter.ktachibana.party",
|
||||
"https://nitter.tinfoil-hat.net"
|
||||
"https://nitter.rawbit.ninja",
|
||||
"https://n.populas.no",
|
||||
"https://nitter.holo-mix.com"
|
||||
],
|
||||
"tor": [
|
||||
"http://nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion",
|
||||
@@ -253,7 +197,9 @@
|
||||
"http://twitter.cepyxplublbyw2f4axy4pyztfbxmf63lrt2c7uwv6wl4iixz53czload.onion",
|
||||
"http://nitter.coffee2m3bjsrrqqycx6ghkxrnejl2q6nl7pjw2j4clchjj6uk5zozad.onion",
|
||||
"http://nitter.daturab6drmkhyeia4ch5gvfc2f3wgo6bhjrv3pz6n7kxmvoznlkq4yd.onion",
|
||||
"http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion"
|
||||
"http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion",
|
||||
"http://6g7w5lt6qcidh6vtue64udzld2mjnvxlpgddvn34qywxafxdv6a3w5id.onion",
|
||||
"http://nitter.catsarchywsyuss6jdxlypsw5dc7owd5u5tr6bujxb7o6xw2hipqehyd.onion"
|
||||
],
|
||||
"i2p": [
|
||||
"http://axd6uavsstsrvstva4mzlzh4ct76rc6zdug3nxdgeitrzczhzf4q.b32.i2p",
|
||||
@@ -265,9 +211,7 @@
|
||||
"http://4kzgzonccxuv4ctforjqx565jgd4bocgnzsy3dg4fmdf3ye2iciq.b32.i2p",
|
||||
"http://pjsfs4ukb6prmfx3qx3a5ef2cpcupkvcrxdh72kqn2rxc2cw4nka.b32.i2p"
|
||||
],
|
||||
"loki": [
|
||||
"http://nitter.priv.loki"
|
||||
]
|
||||
"loki": []
|
||||
},
|
||||
"libreddit": {
|
||||
"clearnet": [
|
||||
@@ -617,7 +561,6 @@
|
||||
"https://xo.wtf"
|
||||
],
|
||||
"tor": [
|
||||
"http://lbmegc3rjnekmdxuisynqdc7y3m2tgyq7gj257ooddaobxqjw36bdayd.onion",
|
||||
"http://4n53nafyi77iplnbrpmxnp3x4exbswwxigujaxy3b37fvr7bvlopxeyd.onion",
|
||||
"http://whereissky27lxfdrs7ct7a5ievor3tl67qi77rv5luga7wod5jf2mad.onion",
|
||||
"http://searchpie7fejn4wctzm6fgm6oz5vnziltfzzuec6b4qmkr3wrqaoqqd.onion",
|
||||
@@ -832,14 +775,6 @@
|
||||
"i2p": [],
|
||||
"loki": []
|
||||
},
|
||||
"facil": {
|
||||
"clearnet": [
|
||||
"https://facilmap.org"
|
||||
],
|
||||
"tor": [],
|
||||
"i2p": [],
|
||||
"loki": []
|
||||
},
|
||||
"osm": {
|
||||
"clearnet": [
|
||||
"https://www.openstreetmap.org"
|
||||
@@ -905,49 +840,47 @@
|
||||
},
|
||||
"privateBin": {
|
||||
"clearnet": [
|
||||
"https://secret.hummel-web.at",
|
||||
"https://bin.0xfc.de",
|
||||
"https://paste.3server.de",
|
||||
"https://paste.tuxcloud.net",
|
||||
"https://snip.dssr.ch",
|
||||
"https://paste.ononoki.org",
|
||||
"https://p.vinci-concessions.com",
|
||||
"https://extrait.facil.services",
|
||||
"https://bonus01.hwb0307.com",
|
||||
"https://extrait.facil.services",
|
||||
"https://paste.momobako.com",
|
||||
"https://paste.waays.fr",
|
||||
"https://notebin.de",
|
||||
"https://vip.ci",
|
||||
"https://privatebin.wsmm.de",
|
||||
"https://paste.devsite.pl",
|
||||
"https://onetimesecret.trans-ix.nl",
|
||||
"https://secrets.l25.cloud",
|
||||
"https://secret.hummel-web.at",
|
||||
"https://p.darklab.sh",
|
||||
"https://privatebin.io",
|
||||
"https://paste.hackliberty.org",
|
||||
"https://0.jaegers.net",
|
||||
"https://bin.bus-hit.me",
|
||||
"https://paste.devsite.pl",
|
||||
"https://ots.kocsar.at",
|
||||
"https://onetimesecret.trans-ix.nl",
|
||||
"https://private.rmconsulting.com",
|
||||
"https://zerobin.thican.net",
|
||||
"https://bin.nixnet.services",
|
||||
"https://paste.momobako.com",
|
||||
"https://ots.kocsar.at",
|
||||
"https://privatebin.net",
|
||||
"https://vip.ci",
|
||||
"https://bin.oldgate.org",
|
||||
"https://paste.waays.fr",
|
||||
"https://0.0g.gg",
|
||||
"https://privatebin.luckvintage.com",
|
||||
"https://paste.encrypt.co.il",
|
||||
"https://paste.skynetcloud.site",
|
||||
"https://paste.techbag.app",
|
||||
"https://pb.blackvoid.club",
|
||||
"https://privatebin.arch-linux.cz",
|
||||
"https://paste.tildeverse.org",
|
||||
"https://n0t.io",
|
||||
"https://sendpass.ru",
|
||||
"https://secrets.true.nl",
|
||||
"https://notebin.de",
|
||||
"https://paste.hostux.net",
|
||||
"https://bin.ngn.tf",
|
||||
"https://encryp.ch/note",
|
||||
"https://paste.tuxcloud.net",
|
||||
"https://paste.3server.de",
|
||||
"https://n0t.io",
|
||||
"https://bin.oldgate.org",
|
||||
"https://paste.techbag.app",
|
||||
"https://bin.nixnet.services",
|
||||
"https://privatebin.arch-linux.cz",
|
||||
"https://paste.ottertelecom.com",
|
||||
"https://bin.bus-hit.me"
|
||||
"https://paste.encrypt.co.il",
|
||||
"https://encryp.ch/note",
|
||||
"https://bin.ngn.tf",
|
||||
"https://bin.0xfc.de",
|
||||
"https://pb.blackvoid.club",
|
||||
"https://paste.tildeverse.org",
|
||||
"https://paste.ononoki.org",
|
||||
"https://sendpass.ru",
|
||||
"https://snip.dssr.ch",
|
||||
"https://privatebin.net",
|
||||
"https://paste.hostux.net",
|
||||
"https://privatebin.io",
|
||||
"https://paste.hackliberty.org",
|
||||
"https://p.vinci-concessions.com",
|
||||
"https://0.jaegers.net",
|
||||
"https://secrets.l25.cloud",
|
||||
"https://privatebin.xi.ht"
|
||||
],
|
||||
"tor": [],
|
||||
"i2p": [],
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"clearnet": ["https://facilmap.org"],
|
||||
"tor": [],
|
||||
"i2p": [],
|
||||
"loki": []
|
||||
}
|
||||
-745
@@ -1,745 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import traceback
|
||||
import logging
|
||||
import requests
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
import re
|
||||
from colorama import Fore, Style
|
||||
import socket
|
||||
import yaml
|
||||
|
||||
mightyList = {}
|
||||
networks = {}
|
||||
|
||||
startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*"
|
||||
endRegex = "(?:\/[^\s\/]+)*\/?"
|
||||
torRegex = startRegex + "onion" + endRegex
|
||||
i2pRegex = startRegex + "i2p" + endRegex
|
||||
lokiRegex = startRegex + "loki" + endRegex
|
||||
authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex
|
||||
|
||||
with open('networks.json', 'rt') as tmp:
|
||||
networks = json.load(tmp)
|
||||
|
||||
|
||||
def filterLastSlash(urlList):
|
||||
tmp = {}
|
||||
for frontend in urlList:
|
||||
tmp[frontend] = {}
|
||||
for network in urlList[frontend]:
|
||||
tmp[frontend][network] = []
|
||||
for url in urlList[frontend][network]:
|
||||
if url.endswith('/'):
|
||||
tmp[frontend][network].append(url[:-1])
|
||||
print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
|
||||
else:
|
||||
tmp[frontend][network].append(url)
|
||||
return tmp
|
||||
|
||||
|
||||
def idnaEncode(urlList):
|
||||
tmp = {}
|
||||
for frontend in urlList:
|
||||
tmp[frontend] = {}
|
||||
for network in urlList[frontend]:
|
||||
tmp[frontend][network] = []
|
||||
for url in urlList[frontend][network]:
|
||||
try:
|
||||
encodedUrl = url.encode("idna").decode("utf8")
|
||||
tmp[frontend][network].append(encodedUrl)
|
||||
if (encodedUrl != url):
|
||||
print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
|
||||
except Exception:
|
||||
tmp[frontend][network].append(url)
|
||||
return tmp
|
||||
|
||||
|
||||
def ip2bin(ip): return "".join(
|
||||
map(
|
||||
str,
|
||||
[
|
||||
"{0:08b}".format(int(x)) for x in ip.split(".")
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_cloudflare_ips():
|
||||
r = requests.get('https://www.cloudflare.com/ips-v4')
|
||||
return r.text.split('\n')
|
||||
|
||||
|
||||
cloudflare_ips = get_cloudflare_ips()
|
||||
|
||||
|
||||
def is_cloudflare(url):
|
||||
instance_ip = None
|
||||
try:
|
||||
instance_ip = socket.gethostbyname(urlparse(url).hostname)
|
||||
if instance_ip is None:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
instance_bin = ip2bin(instance_ip)
|
||||
|
||||
for cloudflare_ip_mask in cloudflare_ips:
|
||||
cloudflare_ip = cloudflare_ip_mask.split('/')[0]
|
||||
cloudflare_bin = ip2bin(cloudflare_ip)
|
||||
|
||||
mask = int(cloudflare_ip_mask.split('/')[1])
|
||||
cloudflare_bin_masked = cloudflare_bin[:mask]
|
||||
instance_bin_masked = instance_bin[:mask]
|
||||
|
||||
if cloudflare_bin_masked == instance_bin_masked:
|
||||
print(url + ' is behind ' + Fore.RED +
|
||||
'cloudflare' + Style.RESET_ALL)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def fetchCache(frontend):
|
||||
try:
|
||||
with open('./data.json') as file:
|
||||
mightyList[frontend] = json.load(file)[frontend]
|
||||
print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + frontend)
|
||||
except Exception:
|
||||
print(Fore.RED + 'Failed' + Style.RESET_ALL +
|
||||
' to get cached ' + frontend)
|
||||
|
||||
|
||||
def fetchFromFile(frontend):
|
||||
with open('./fixed/' + frontend + '.json') as file:
|
||||
mightyList[frontend] = json.load(file)
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
|
||||
|
||||
def fetchJsonList(frontend, url, urlItem, jsonObject):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
rJson = json.loads(r.text)
|
||||
if jsonObject:
|
||||
rJson = rJson['instances']
|
||||
_list = {}
|
||||
for network in networks:
|
||||
_list[network] = []
|
||||
if type(urlItem) == dict:
|
||||
for item in rJson:
|
||||
for network in networks:
|
||||
if urlItem[network] is not None:
|
||||
if urlItem[network] in item and item[urlItem[network]] is not None:
|
||||
if type(item[urlItem[network]]) == list:
|
||||
for i in item[urlItem[network]]:
|
||||
if i.strip() != '':
|
||||
_list[network].append(i)
|
||||
elif item[urlItem[network]].strip() != '':
|
||||
_list[network].append(item[urlItem[network]])
|
||||
else:
|
||||
for item in rJson:
|
||||
tmpItem = item
|
||||
if urlItem is not None:
|
||||
tmpItem = item[urlItem]
|
||||
if tmpItem.strip() == '':
|
||||
continue
|
||||
elif re.search(torRegex, tmpItem):
|
||||
_list['tor'].append(tmpItem)
|
||||
elif re.search(i2pRegex, tmpItem):
|
||||
_list['i2p'].append(tmpItem)
|
||||
elif re.search(lokiRegex, tmpItem):
|
||||
_list['loki'].append(tmpItem)
|
||||
else:
|
||||
_list['clearnet'].append(tmpItem)
|
||||
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def fetchRegexList(frontend, url, regex):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
_list = {}
|
||||
for network in networks:
|
||||
_list[network] = []
|
||||
|
||||
tmp = re.findall(regex, r.text)
|
||||
|
||||
for item in tmp:
|
||||
if item.strip() == "":
|
||||
continue
|
||||
elif re.search(torRegex, item):
|
||||
_list['tor'].append(item)
|
||||
elif re.search(i2pRegex, item):
|
||||
_list['i2p'].append(item)
|
||||
elif re.search(lokiRegex, item):
|
||||
_list['loki'].append(item)
|
||||
else:
|
||||
_list['clearnet'].append(item)
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def fetchTextList(frontend, url, prepend):
|
||||
try:
|
||||
_list = {}
|
||||
for network in networks:
|
||||
_list[network] = []
|
||||
|
||||
if type(url) == dict:
|
||||
for network in networks:
|
||||
if url[network] is not None:
|
||||
r = requests.get(url[network])
|
||||
tmp = r.text.strip().split('\n')
|
||||
for item in tmp:
|
||||
item = prepend[network] + item
|
||||
_list[network].append(item)
|
||||
else:
|
||||
r = requests.get(url)
|
||||
tmp = r.text.strip().split('\n')
|
||||
|
||||
for item in tmp:
|
||||
item = prepend + item
|
||||
if re.search(torRegex, item):
|
||||
_list['tor'].append(item)
|
||||
elif re.search(i2pRegex, item):
|
||||
_list['i2p'].append(item)
|
||||
elif re.search(lokiRegex, item):
|
||||
_list['loki'].append(item)
|
||||
else:
|
||||
_list['clearnet'].append(item)
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def invidious():
|
||||
name = 'Invidious'
|
||||
frontend = 'invidious'
|
||||
url = 'https://api.invidious.io/instances.json'
|
||||
try:
|
||||
_list = {}
|
||||
_list['clearnet'] = []
|
||||
_list['tor'] = []
|
||||
_list['i2p'] = []
|
||||
_list['loki'] = []
|
||||
r = requests.get(url)
|
||||
rJson = json.loads(r.text)
|
||||
for instance in rJson:
|
||||
if instance[1]['type'] == 'https':
|
||||
_list['clearnet'].append(instance[1]['uri'])
|
||||
elif instance[1]['type'] == 'onion':
|
||||
_list['tor'].append(instance[1]['uri'])
|
||||
elif instance[1]['type'] == 'i2p':
|
||||
_list['i2p'].append(instance[1]['uri'])
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def piped():
|
||||
frontend = 'piped'
|
||||
try:
|
||||
_list = {}
|
||||
_list['clearnet'] = []
|
||||
_list['tor'] = []
|
||||
_list['i2p'] = []
|
||||
_list['loki'] = []
|
||||
r = requests.get(
|
||||
'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md')
|
||||
|
||||
tmp = re.findall(
|
||||
r' \| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text)
|
||||
for item in tmp:
|
||||
try:
|
||||
url = requests.get(item, timeout=5).url
|
||||
if url.strip("/") == item:
|
||||
continue
|
||||
else:
|
||||
_list['clearnet'].append(url)
|
||||
except Exception:
|
||||
logging.error(traceback.format_exc())
|
||||
continue
|
||||
mightyList[frontend] = _list
|
||||
_list['clearnet'].remove("https://piped.video")
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def materialious():
|
||||
fetchRegexList('materialious', 'https://raw.githubusercontent.com/Materialious/Materialious/main/docs/INSTANCES.md',
|
||||
r"- \[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)")
|
||||
|
||||
|
||||
def pipedMaterial():
|
||||
fetchRegexList('pipedMaterial', 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md',
|
||||
r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production")
|
||||
|
||||
|
||||
def cloudtube():
|
||||
fetchFromFile('cloudtube')
|
||||
|
||||
|
||||
def proxitok():
|
||||
fetchRegexList('proxiTok', 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md',
|
||||
r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))")
|
||||
|
||||
|
||||
def send():
|
||||
fetchRegexList('send', 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md',
|
||||
r"(https.*?) \|.*?\n")
|
||||
|
||||
|
||||
def nitter():
|
||||
frontend = 'nitter'
|
||||
try:
|
||||
r = requests.get(
|
||||
'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md')
|
||||
_list = {}
|
||||
|
||||
_list['clearnet'] = []
|
||||
public = re.findall(r"## Public((?:\n|.*)+?)##", r.text)
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^\| \[.*?\]\((https.*?)\)", line)
|
||||
if len(result) > 0:
|
||||
_list['clearnet'].append(result[0])
|
||||
|
||||
_list['tor'] = []
|
||||
public = re.findall(r"## Tor((?:\n|.*)+?)##", r.text)
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^\| <(http.*?)\/?>", line)
|
||||
if len(result) > 0:
|
||||
_list['tor'].append(result[0])
|
||||
|
||||
_list['i2p'] = []
|
||||
public = re.findall(r"## I2P((?:\n|.*)+?)##", r.text)
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^- <(http.*?)\/?>", line)
|
||||
if len(result) > 0:
|
||||
_list['i2p'].append(result[0])
|
||||
|
||||
_list['loki'] = []
|
||||
public = re.findall(r"## Lokinet((?:\n|.*)+?)##", r.text)
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^- <(http.*?)\/?>", line)
|
||||
if len(result) > 0:
|
||||
_list['loki'].append(result[0])
|
||||
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
# fetchRegexList('nitter', '',
|
||||
# r"(?:(?:\| )|(?:- ))\[(?:(?:\S+\.)+[a-zA-Z0-9]+)\/?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]+)\/?\)(?:(?: (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,4} +\|(?:(?:\n)|(?: ❌)|(?: ✅)|(?: ❓)|(?: \[)))|(?:\n))")
|
||||
|
||||
|
||||
def libreddit():
|
||||
fetchJsonList('libreddit', 'https://github.com/libreddit/libreddit-instances/raw/master/instances.json',
|
||||
{'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True)
|
||||
|
||||
|
||||
def redlib():
|
||||
fetchJsonList('redlib', 'https://github.com/redlib-org/redlib-instances/raw/main/instances.json',
|
||||
{'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True)
|
||||
|
||||
|
||||
def teddit():
|
||||
fetchJsonList('teddit', 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json',
|
||||
{'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False)
|
||||
|
||||
|
||||
def scribe():
|
||||
fetchJsonList(
|
||||
'scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', None, False)
|
||||
|
||||
|
||||
def quetre():
|
||||
fetchJsonList('quetre', 'https://raw.githubusercontent.com/zyachel/quetre/main/instances.json',
|
||||
{'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False)
|
||||
|
||||
|
||||
def libremdb():
|
||||
fetchJsonList('libremdb', 'https://raw.githubusercontent.com/zyachel/libremdb/main/instances.json',
|
||||
{'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False)
|
||||
|
||||
|
||||
def simplytranslate():
|
||||
fetchFromFile('simplyTranslate')
|
||||
|
||||
|
||||
def linvgatranslate():
|
||||
fetchJsonList(
|
||||
'lingva', 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False)
|
||||
|
||||
|
||||
def searxng():
|
||||
r = requests.get(
|
||||
'https://searx.space/data/instances.json')
|
||||
rJson = json.loads(r.text)
|
||||
searxngList = {}
|
||||
searxngList['clearnet'] = []
|
||||
searxngList['tor'] = []
|
||||
searxngList['i2p'] = []
|
||||
searxngList['loki'] = []
|
||||
|
||||
for item in rJson['instances']:
|
||||
if re.search(torRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
|
||||
searxngList['tor'].append(item[:-1])
|
||||
elif re.search(i2pRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
|
||||
searxngList['i2p'].append(item[:-1])
|
||||
elif rJson['instances'][item].get('generator') == 'searxng':
|
||||
searxngList['clearnet'].append(item[:-1])
|
||||
|
||||
mightyList['searxng'] = searxngList
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
|
||||
|
||||
|
||||
def searx():
|
||||
searxList = {}
|
||||
searxList['clearnet'] = []
|
||||
searxList['tor'] = []
|
||||
searxList['i2p'] = []
|
||||
searxList['loki'] = []
|
||||
r = requests.get(
|
||||
'https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml')
|
||||
data = yaml.safe_load(r.text)
|
||||
for key in data:
|
||||
searxList['clearnet'].append(key)
|
||||
if 'additional_urls' in data[key]:
|
||||
for additional_url in data[key]['additional_urls']:
|
||||
if data[key]['additional_urls'][additional_url] == "Hidden Service":
|
||||
searxList['tor'].append(additional_url)
|
||||
mightyList['searx'] = searxList
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
|
||||
|
||||
|
||||
def whoogle():
|
||||
fetchRegexList('whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md',
|
||||
r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ")
|
||||
|
||||
|
||||
def librex():
|
||||
fetchJsonList('librex', 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json',
|
||||
{'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, True)
|
||||
|
||||
|
||||
def rimgo():
|
||||
try:
|
||||
r = requests.get('https://rimgo.codeberg.page/api.json')
|
||||
rJson = json.loads(r.text)
|
||||
_list = {
|
||||
'clearnet': [],
|
||||
'tor': [],
|
||||
}
|
||||
for instance in rJson['clearnet']:
|
||||
_list['clearnet'].append(instance['url'])
|
||||
|
||||
for instance in rJson['tor']:
|
||||
_list['tor'].append(instance['url'])
|
||||
|
||||
mightyList['rimgo'] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'rimgo')
|
||||
except Exception:
|
||||
fetchCache('rimgo')
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def pixivFe():
|
||||
fetchRegexList('pixivFe', 'https://codeberg.org/VnPower/pixivfe/raw/branch/main/README.md',
|
||||
r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/? +\|")
|
||||
|
||||
|
||||
def safetwitch():
|
||||
fetchRegexList('safetwitch', 'https://codeberg.org/dragongoose/safetwitch/raw/branch/master/README.md',
|
||||
re.compile(r"^\| \[.*?\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)",re.MULTILINE))
|
||||
|
||||
|
||||
def hyperpipe():
|
||||
fetchJsonList(
|
||||
'hyperpipe', 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', 'url', False)
|
||||
|
||||
|
||||
def facil():
|
||||
fetchFromFile('facil')
|
||||
|
||||
|
||||
def osm():
|
||||
fetchFromFile('osm')
|
||||
|
||||
|
||||
def libreTranslate():
|
||||
fetchRegexList('libreTranslate', 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md',
|
||||
r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|")
|
||||
|
||||
|
||||
def breezeWiki():
|
||||
fetchJsonList(
|
||||
'breezeWiki', 'https://docs.breezewiki.com/files/instances.json', 'instance', False)
|
||||
|
||||
|
||||
def binternet():
|
||||
fetchRegexList('binternet', 'https://raw.githubusercontent.com/Ahwxorg/Binternet/main/README.md',
|
||||
r"\| \[[\w\.]+!?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/?\)")
|
||||
|
||||
|
||||
def privateBin():
|
||||
fetchJsonList('privateBin', 'https://privatebin.info/directory/api?top=100&https_redirect=true&min_rating=A&csp_header=true&min_uptime=100&attachments=true', 'url', False)
|
||||
|
||||
|
||||
def neuters():
|
||||
fetchFromFile('neuters')
|
||||
|
||||
|
||||
def libMedium():
|
||||
fetchRegexList('libMedium', 'https://raw.githubusercontent.com/realaravinth/libmedium/master/README.md',
|
||||
r"\| (https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/? +\|")
|
||||
|
||||
|
||||
def dumb():
|
||||
fetchJsonList('dumb', 'https://raw.githubusercontent.com/rramiachraf/dumb/main/instances.json',
|
||||
{'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False)
|
||||
|
||||
|
||||
def ruralDictionary():
|
||||
fetchJsonList('ruralDictionary',
|
||||
'https://codeberg.org/zortazert/rural-dictionary/raw/branch/master/instances.json',
|
||||
{'clearnet': 'clearnet', 'tor': 'tor',
|
||||
'i2p': 'i2p', 'loki': None},
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
def anonymousOverflow():
|
||||
fetchRegexList('anonymousOverflow', 'https://raw.githubusercontent.com/httpjamesm/AnonymousOverflow/main/README.md',
|
||||
r"\| \[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) +\|")
|
||||
|
||||
|
||||
def wikiless():
|
||||
fetchFromFile('wikiless')
|
||||
|
||||
|
||||
def biblioReads():
|
||||
fetchJsonList(
|
||||
'biblioReads',
|
||||
'https://raw.githubusercontent.com/nesaku/BiblioReads/main/instances.json',
|
||||
{'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None},
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
def suds():
|
||||
fetchJsonList(
|
||||
'suds', 'https://git.vern.cc/cobra/Suds/raw/branch/main/instances.json',
|
||||
{
|
||||
'clearnet': 'clearnet',
|
||||
'tor': 'tor',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
def poketube():
|
||||
frontend = 'poketube'
|
||||
try:
|
||||
r = requests.get(
|
||||
'https://codeberg.org/Ashley/poketube/raw/branch/main/instances.json')
|
||||
rJson = json.loads(r.text)
|
||||
_list = {
|
||||
'clearnet': [],
|
||||
'tor': [],
|
||||
'i2p': [],
|
||||
'loki': []
|
||||
|
||||
}
|
||||
for element in rJson:
|
||||
_list['clearnet'].append(element[1]['uri'])
|
||||
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def gothub():
|
||||
fetchJsonList(
|
||||
'gothub',
|
||||
'https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json',
|
||||
{
|
||||
'clearnet': 'link',
|
||||
'tor': None,
|
||||
'i2p': None,
|
||||
'loki': None
|
||||
},
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
def mikuInvidious():
|
||||
fetchFromFile('mikuInvidious')
|
||||
|
||||
|
||||
def tent():
|
||||
fetchJsonList('tent', 'https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json', 'url', False)
|
||||
|
||||
|
||||
def laboratory():
|
||||
fetchRegexList('laboratory', 'https://git.vitali64.duckdns.org/utils/laboratory.git/plain/README.md',
|
||||
r"\| (https:\/{2}.*?) \|")
|
||||
|
||||
|
||||
wolfreeAlpha_url_list = [
|
||||
"https://gqq.gitlab.io",
|
||||
"https://jqq.gitlab.io",
|
||||
"https://rqq.gitlab.io",
|
||||
"https://sqq.gitlab.io",
|
||||
"https://uqq.gitlab.io"
|
||||
]
|
||||
wolfreeAlpha_url_list_i = 0
|
||||
|
||||
|
||||
def wolfreeAlpha(i):
|
||||
global wolfreeAlpha_url_list_i
|
||||
frontend = 'wolfreeAlpha'
|
||||
try:
|
||||
r = requests.get(wolfreeAlpha_url_list[i]+"/instances.json")
|
||||
if r.status_code != 200:
|
||||
wolfreeAlpha_url_list_i += 1
|
||||
wolfreeAlpha(wolfreeAlpha_url_list_i)
|
||||
else:
|
||||
rJson = json.loads(r.text)
|
||||
networks = rJson['wolfree']
|
||||
_list = {}
|
||||
for i in networks.keys():
|
||||
_list[i] = networks[i]
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except:
|
||||
wolfreeAlpha_url_list_i += 1
|
||||
wolfreeAlpha(wolfreeAlpha_url_list_i)
|
||||
|
||||
|
||||
def jiti():
|
||||
fetchRegexList('jitsi', "https://raw.githubusercontent.com/jitsi/handbook/master/docs/community/instances.md",
|
||||
r"\|(?:(?: | )+)+((?:[a-z]+\.)+[a-z]+)(?:(?: | )+)+\|")
|
||||
mightyList['jitsi']['clearnet'] = list(
|
||||
map(lambda x: "https://"+x, mightyList['jitsi']['clearnet']))
|
||||
mightyList['jitsi']['clearnet'].insert(0, 'https://8x8.vc')
|
||||
mightyList['jitsi']['clearnet'].insert(0, 'https://meet.jit.si')
|
||||
|
||||
|
||||
def proxigram():
|
||||
fetchRegexList('proxigram', 'https://codeberg.org/ThePenguinDev/Proxigram/wiki/raw/Instances.md',
|
||||
r"\[(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\]")
|
||||
|
||||
|
||||
def twineo():
|
||||
fetchFromFile('twineo')
|
||||
|
||||
|
||||
def priviblur():
|
||||
fetchRegexList('priviblur', 'https://raw.githubusercontent.com/syeopite/priviblur/master/instances.md',
|
||||
r"\| ?\[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\) ?|")
|
||||
|
||||
|
||||
def isValid(url): # This code is contributed by avanitrachhadiya2155
|
||||
try:
|
||||
result = urlparse(url)
|
||||
return all([result.scheme, result.netloc])
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
invidious()
|
||||
materialious()
|
||||
piped()
|
||||
pipedMaterial()
|
||||
cloudtube()
|
||||
proxitok()
|
||||
send()
|
||||
nitter()
|
||||
libreddit()
|
||||
redlib()
|
||||
teddit()
|
||||
scribe()
|
||||
quetre()
|
||||
libremdb()
|
||||
simplytranslate()
|
||||
linvgatranslate()
|
||||
libreTranslate()
|
||||
searxng()
|
||||
searx()
|
||||
whoogle()
|
||||
librex()
|
||||
rimgo()
|
||||
pixivFe()
|
||||
safetwitch()
|
||||
hyperpipe()
|
||||
facil()
|
||||
osm()
|
||||
breezeWiki()
|
||||
binternet()
|
||||
privateBin()
|
||||
neuters()
|
||||
ruralDictionary()
|
||||
libMedium()
|
||||
dumb()
|
||||
anonymousOverflow()
|
||||
wikiless()
|
||||
biblioReads()
|
||||
suds()
|
||||
poketube()
|
||||
gothub()
|
||||
mikuInvidious()
|
||||
wolfreeAlpha(wolfreeAlpha_url_list_i)
|
||||
jiti()
|
||||
proxigram()
|
||||
tent()
|
||||
laboratory()
|
||||
twineo()
|
||||
priviblur()
|
||||
|
||||
mightyList = filterLastSlash(mightyList)
|
||||
mightyList = idnaEncode(mightyList)
|
||||
|
||||
cloudflare = []
|
||||
for k1, v1 in mightyList.items():
|
||||
if type(mightyList[k1]) is dict:
|
||||
for k2, v2 in mightyList[k1].items():
|
||||
for instance in mightyList[k1][k2]:
|
||||
if (not isValid(instance)):
|
||||
mightyList[k1][k2].remove(instance)
|
||||
print("removed " + instance)
|
||||
else:
|
||||
if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance):
|
||||
cloudflare.append(instance)
|
||||
blacklist = {
|
||||
'cloudflare': cloudflare
|
||||
}
|
||||
|
||||
# Writing to file
|
||||
json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
|
||||
with open('./data.json', 'w') as outfile:
|
||||
outfile.write(json_object)
|
||||
print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json')
|
||||
|
||||
json_object = json.dumps(blacklist, ensure_ascii=False, indent=2)
|
||||
with open('./blacklist.json', 'w') as outfile:
|
||||
outfile.write(json_object)
|
||||
print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json')
|
||||
@@ -0,0 +1,89 @@
|
||||
import json
|
||||
from colorama import Fore, Style
|
||||
from services.from_file import *
|
||||
from services.medium import *
|
||||
from services.other import *
|
||||
from services.reddit import *
|
||||
from services.search import *
|
||||
from services.translate import *
|
||||
from services.utils import *
|
||||
from services.wolfram import *
|
||||
from services.youtube import *
|
||||
|
||||
mightyList = {}
|
||||
|
||||
invidious(mightyList)
|
||||
materialious(mightyList)
|
||||
piped(mightyList)
|
||||
pipedMaterial(mightyList)
|
||||
cloudtube(mightyList)
|
||||
proxitok(mightyList)
|
||||
send(mightyList)
|
||||
nitter(mightyList)
|
||||
libreddit(mightyList)
|
||||
redlib(mightyList)
|
||||
teddit(mightyList)
|
||||
scribe(mightyList)
|
||||
quetre(mightyList)
|
||||
libremdb(mightyList)
|
||||
simplytranslate(mightyList)
|
||||
linvgatranslate(mightyList)
|
||||
libreTranslate(mightyList)
|
||||
searxng(mightyList)
|
||||
searx(mightyList)
|
||||
whoogle(mightyList)
|
||||
librex(mightyList)
|
||||
rimgo(mightyList)
|
||||
pixivFe(mightyList)
|
||||
safetwitch(mightyList)
|
||||
hyperpipe(mightyList)
|
||||
osm(mightyList)
|
||||
breezeWiki(mightyList)
|
||||
binternet(mightyList)
|
||||
privateBin(mightyList)
|
||||
neuters(mightyList)
|
||||
ruralDictionary(mightyList)
|
||||
libMedium(mightyList)
|
||||
dumb(mightyList)
|
||||
anonymousOverflow(mightyList)
|
||||
wikiless(mightyList)
|
||||
biblioReads(mightyList)
|
||||
suds(mightyList)
|
||||
poketube(mightyList)
|
||||
gothub(mightyList)
|
||||
mikuInvidious(mightyList)
|
||||
wolfreeAlpha(mightyList)
|
||||
jiti(mightyList)
|
||||
proxigram(mightyList)
|
||||
tent(mightyList)
|
||||
laboratory(mightyList)
|
||||
twineo(mightyList)
|
||||
priviblur(mightyList)
|
||||
|
||||
mightyList = filterLastSlash(mightyList)
|
||||
mightyList = idnaEncode(mightyList)
|
||||
|
||||
cloudflare = []
|
||||
for k1, v1 in mightyList.items():
|
||||
if type(mightyList[k1]) is dict:
|
||||
for k2, v2 in mightyList[k1].items():
|
||||
for instance in mightyList[k1][k2]:
|
||||
if not isValid(instance):
|
||||
mightyList[k1][k2].remove(instance)
|
||||
print("removed " + instance)
|
||||
else:
|
||||
if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance):
|
||||
cloudflare.append(instance)
|
||||
blacklist = {
|
||||
'cloudflare': cloudflare
|
||||
}
|
||||
|
||||
json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
|
||||
with open('./data.json', 'w') as outfile:
|
||||
outfile.write(json_object)
|
||||
print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json')
|
||||
|
||||
json_object = json.dumps(blacklist, ensure_ascii=False, indent=2)
|
||||
with open('./blacklist.json', 'w') as outfile:
|
||||
outfile.write(json_object)
|
||||
print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json')
|
||||
@@ -0,0 +1,3 @@
|
||||
requests
|
||||
colorama
|
||||
pyyaml
|
||||
@@ -0,0 +1,29 @@
|
||||
from .utils import fetchFromFile
|
||||
|
||||
|
||||
def osm(mightyList):
|
||||
fetchFromFile('osm', mightyList)
|
||||
|
||||
|
||||
def neuters(mightyList):
|
||||
fetchFromFile('neuters', mightyList)
|
||||
|
||||
|
||||
def wikiless(mightyList):
|
||||
fetchFromFile('wikiless', mightyList)
|
||||
|
||||
|
||||
def mikuInvidious(mightyList):
|
||||
fetchFromFile('mikuInvidious', mightyList)
|
||||
|
||||
|
||||
def twineo(mightyList):
|
||||
fetchFromFile('twineo', mightyList)
|
||||
|
||||
|
||||
def cloudtube(mightyList):
|
||||
fetchFromFile('cloudtube', mightyList)
|
||||
|
||||
|
||||
def simplytranslate(mightyList):
|
||||
fetchFromFile('simplyTranslate', mightyList)
|
||||
@@ -0,0 +1,19 @@
|
||||
from .utils import fetchJsonList, fetchRegexList
|
||||
|
||||
|
||||
def scribe(mightyList):
|
||||
fetchJsonList(
|
||||
'scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json',
|
||||
None,
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def libMedium(mightyList):
|
||||
fetchRegexList(
|
||||
'libMedium',
|
||||
'https://raw.githubusercontent.com/realaravinth/libmedium/master/README.md',
|
||||
r"\| (https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/? +\|",
|
||||
mightyList
|
||||
)
|
||||
@@ -0,0 +1,310 @@
|
||||
import requests
|
||||
import json
|
||||
from .utils import fetchJsonList, fetchRegexList, fetchCache
|
||||
from colorama import Fore, Style
|
||||
import logging
|
||||
import traceback
|
||||
import re
|
||||
|
||||
|
||||
def tent(mightyList):
|
||||
fetchJsonList(
|
||||
'tent',
|
||||
'https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json',
|
||||
'url',
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def ruralDictionary(mightyList):
|
||||
fetchJsonList(
|
||||
'ruralDictionary',
|
||||
'https://codeberg.org/zortazert/rural-dictionary/raw/branch/master/instances.json',
|
||||
{
|
||||
'clearnet': 'clearnet',
|
||||
'tor': 'tor',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def laboratory(mightyList):
|
||||
fetchRegexList(
|
||||
'laboratory',
|
||||
'https://git.vitali64.duckdns.org/utils/laboratory.git/plain/README.md',
|
||||
r"\| (https:\/{2}.*?) \|",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def gothub(mightyList):
|
||||
fetchJsonList(
|
||||
'gothub',
|
||||
'https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json',
|
||||
{
|
||||
'clearnet': 'link',
|
||||
'tor': None,
|
||||
'i2p': None,
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def biblioReads(mightyList):
|
||||
fetchJsonList(
|
||||
'biblioReads',
|
||||
'https://raw.githubusercontent.com/nesaku/BiblioReads/main/instances.json',
|
||||
{
|
||||
'clearnet': 'url',
|
||||
'tor': 'onion',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def libremdb(mightyList):
|
||||
fetchJsonList(
|
||||
'libremdb',
|
||||
'https://raw.githubusercontent.com/zyachel/libremdb/main/instances.json',
|
||||
{
|
||||
'clearnet': 'clearnet',
|
||||
'tor': 'tor',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def breezeWiki(mightyList):
|
||||
fetchJsonList(
|
||||
'breezeWiki',
|
||||
'https://docs.breezewiki.com/files/instances.json',
|
||||
'instance',
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def binternet(mightyList):
|
||||
fetchRegexList(
|
||||
'binternet',
|
||||
'https://raw.githubusercontent.com/Ahwxorg/Binternet/main/README.md',
|
||||
r"\| \[[\w\.]+!?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/?\)",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def dumb(mightyList):
|
||||
fetchJsonList(
|
||||
'dumb',
|
||||
'https://raw.githubusercontent.com/rramiachraf/dumb/main/instances.json',
|
||||
{
|
||||
'clearnet': 'clearnet',
|
||||
'tor': 'tor',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def suds(mightyList):
|
||||
fetchJsonList(
|
||||
'suds', 'https://git.vern.cc/cobra/Suds/raw/branch/main/instances.json',
|
||||
{
|
||||
'clearnet': 'clearnet',
|
||||
'tor': 'tor',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def proxigram(mightyList):
|
||||
fetchRegexList(
|
||||
'proxigram',
|
||||
'https://codeberg.org/ThePenguinDev/Proxigram/wiki/raw/Instances.md',
|
||||
r"\[(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\]",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def rimgo(mightyList):
|
||||
try:
|
||||
r = requests.get('https://rimgo.codeberg.page/api.json')
|
||||
rJson = json.loads(r.text)
|
||||
_list = {
|
||||
'clearnet': [],
|
||||
'tor': [],
|
||||
}
|
||||
for instance in rJson['clearnet']:
|
||||
_list['clearnet'].append(instance['url'])
|
||||
|
||||
for instance in rJson['tor']:
|
||||
_list['tor'].append(instance['url'])
|
||||
|
||||
mightyList['rimgo'] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'rimgo')
|
||||
except Exception:
|
||||
fetchCache('rimgo')
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def jiti(mightyList):
|
||||
fetchRegexList(
|
||||
'jitsi',
|
||||
"https://raw.githubusercontent.com/jitsi/handbook/master/docs/community/instances.md",
|
||||
r"\|(?:(?: | )+)+((?:[a-z]+\.)+[a-z]+)(?:(?: | )+)+\|",
|
||||
mightyList
|
||||
)
|
||||
mightyList['jitsi']['clearnet'] = list(
|
||||
map(
|
||||
lambda x: "https://"+x,
|
||||
mightyList['jitsi']['clearnet']
|
||||
)
|
||||
)
|
||||
mightyList['jitsi']['clearnet'].insert(0, 'https://8x8.vc')
|
||||
mightyList['jitsi']['clearnet'].insert(0, 'https://meet.jit.si')
|
||||
|
||||
|
||||
def anonymousOverflow(mightyList):
|
||||
fetchRegexList(
|
||||
'anonymousOverflow',
|
||||
'https://raw.githubusercontent.com/httpjamesm/AnonymousOverflow/main/README.md',
|
||||
r"\| \[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) +\|",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def proxitok(mightyList):
|
||||
fetchRegexList(
|
||||
'proxiTok',
|
||||
'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md',
|
||||
r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def priviblur(mightyList):
|
||||
fetchRegexList(
|
||||
'priviblur',
|
||||
'https://raw.githubusercontent.com/syeopite/priviblur/master/instances.md',
|
||||
r"\| ?\[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\) ?|",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def safetwitch(mightyList):
|
||||
fetchRegexList(
|
||||
'safetwitch',
|
||||
'https://codeberg.org/dragongoose/safetwitch/raw/branch/master/README.md',
|
||||
re.compile(
|
||||
r"^\| \[.*?\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)",
|
||||
re.MULTILINE
|
||||
),
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def nitter(mightyList):
|
||||
frontend = 'nitter'
|
||||
try:
|
||||
r = requests.get(
|
||||
'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md')
|
||||
_list = {}
|
||||
|
||||
_list['clearnet'] = []
|
||||
public = re.findall(r"## Public((?:\n|.*)+?)##", r.text)
|
||||
if public:
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^\| \[.*?\]\((https.*?)\)", line)
|
||||
if len(result) > 0:
|
||||
_list['clearnet'].append(result[0])
|
||||
|
||||
_list['tor'] = []
|
||||
public = re.findall(r"## Tor((?:\n|.*)+?)##", r.text)
|
||||
if public:
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^\| <(http.*?)\/?>", line)
|
||||
if len(result) > 0:
|
||||
_list['tor'].append(result[0])
|
||||
|
||||
_list['i2p'] = []
|
||||
public = re.findall(r"## I2P((?:\n|.*)+?)##", r.text)
|
||||
if public:
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^- <(http.*?)\/?>", line)
|
||||
if len(result) > 0:
|
||||
_list['i2p'].append(result[0])
|
||||
|
||||
_list['loki'] = []
|
||||
public = re.findall(r"## Lokinet((?:\n|.*)+?)##", r.text)
|
||||
if public:
|
||||
for line in public[0].split('\n'):
|
||||
result = re.findall(r"^- <(http.*?)\/?>", line)
|
||||
if len(result) > 0:
|
||||
_list['loki'].append(result[0])
|
||||
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
|
||||
except Exception:
|
||||
fetchCache(frontend, mightyList)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def send(mightyList):
|
||||
fetchRegexList(
|
||||
'send',
|
||||
'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md',
|
||||
r"(https.*?) \|.*?\n",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def quetre(mightyList):
|
||||
fetchJsonList(
|
||||
'quetre',
|
||||
'https://raw.githubusercontent.com/zyachel/quetre/main/instances.json',
|
||||
{
|
||||
'clearnet': 'clearnet',
|
||||
'tor': 'tor',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def privateBin(mightyList):
|
||||
fetchJsonList(
|
||||
'privateBin',
|
||||
'https://privatebin.info/directory/api?top=100&https_redirect=true&min_rating=A&csp_header=true&min_uptime=100&attachments=true',
|
||||
'url',
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def pixivFe(mightyList):
|
||||
fetchRegexList(
|
||||
'pixivFe',
|
||||
'https://codeberg.org/VnPower/pixivfe/raw/branch/main/README.md',
|
||||
r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/? +\|",
|
||||
mightyList
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
from .utils import fetchJsonList
|
||||
|
||||
|
||||
def libreddit(mightyList):
|
||||
fetchJsonList(
|
||||
'libreddit',
|
||||
'https://github.com/libreddit/libreddit-instances/raw/master/instances.json',
|
||||
{
|
||||
'clearnet': 'url',
|
||||
'tor': 'onion',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
True,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def redlib(mightyList):
|
||||
fetchJsonList(
|
||||
'redlib',
|
||||
'https://github.com/redlib-org/redlib-instances/raw/main/instances.json',
|
||||
{
|
||||
'clearnet': 'url',
|
||||
'tor': 'onion',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
True,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def teddit(mightyList):
|
||||
fetchJsonList(
|
||||
'teddit',
|
||||
'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json',
|
||||
{
|
||||
'clearnet': 'url',
|
||||
'tor': 'onion',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
@@ -0,0 +1,71 @@
|
||||
import requests
|
||||
from colorama import Fore, Style
|
||||
import json
|
||||
import re
|
||||
from .utils import torRegex, i2pRegex, fetchRegexList, fetchJsonList
|
||||
import yaml
|
||||
|
||||
|
||||
def searxng(mightyList):
|
||||
r = requests.get(
|
||||
'https://searx.space/data/instances.json')
|
||||
rJson = json.loads(r.text)
|
||||
searxngList = {}
|
||||
searxngList['clearnet'] = []
|
||||
searxngList['tor'] = []
|
||||
searxngList['i2p'] = []
|
||||
searxngList['loki'] = []
|
||||
|
||||
for item in rJson['instances']:
|
||||
if re.search(torRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
|
||||
searxngList['tor'].append(item[:-1])
|
||||
elif re.search(i2pRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
|
||||
searxngList['i2p'].append(item[:-1])
|
||||
elif rJson['instances'][item].get('generator') == 'searxng':
|
||||
searxngList['clearnet'].append(item[:-1])
|
||||
|
||||
mightyList['searxng'] = searxngList
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
|
||||
|
||||
|
||||
def searx(mightyList):
|
||||
searxList = {}
|
||||
searxList['clearnet'] = []
|
||||
searxList['tor'] = []
|
||||
searxList['i2p'] = []
|
||||
searxList['loki'] = []
|
||||
r = requests.get(
|
||||
'https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml')
|
||||
data = yaml.safe_load(r.text)
|
||||
for key in data:
|
||||
searxList['clearnet'].append(key)
|
||||
if 'additional_urls' in data[key]:
|
||||
for additional_url in data[key]['additional_urls']:
|
||||
if data[key]['additional_urls'][additional_url] == "Hidden Service":
|
||||
searxList['tor'].append(additional_url)
|
||||
mightyList['searx'] = searxList
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
|
||||
|
||||
|
||||
def whoogle(mightyList):
|
||||
fetchRegexList(
|
||||
'whoogle',
|
||||
'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md',
|
||||
r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def librex(mightyList):
|
||||
fetchJsonList(
|
||||
'librex',
|
||||
'https://raw.githubusercontent.com/hnhx/librex/main/instances.json',
|
||||
{
|
||||
'clearnet': 'clearnet',
|
||||
'tor': 'tor',
|
||||
'i2p': 'i2p',
|
||||
'loki': None
|
||||
},
|
||||
True,
|
||||
mightyList
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
from .utils import fetchJsonList, fetchRegexList
|
||||
|
||||
|
||||
def linvgatranslate(mightyList):
|
||||
fetchJsonList(
|
||||
'lingva',
|
||||
'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json',
|
||||
None,
|
||||
False,
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def libreTranslate(mightyList):
|
||||
fetchRegexList(
|
||||
'libreTranslate',
|
||||
'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md',
|
||||
r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|",
|
||||
mightyList
|
||||
)
|
||||
@@ -0,0 +1,229 @@
|
||||
import traceback
|
||||
import logging
|
||||
import requests
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
import re
|
||||
from colorama import Fore, Style
|
||||
import socket
|
||||
|
||||
|
||||
startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*"
|
||||
endRegex = "(?:\\/[^\\s\\/]+)*\\/?"
|
||||
torRegex = startRegex + "onion" + endRegex
|
||||
i2pRegex = startRegex + "i2p" + endRegex
|
||||
lokiRegex = startRegex + "loki" + endRegex
|
||||
authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex
|
||||
|
||||
|
||||
def get_cloudflare_ips():
|
||||
r = requests.get('https://www.cloudflare.com/ips-v4')
|
||||
return r.text.split('\n')
|
||||
|
||||
|
||||
cloudflare_ips = get_cloudflare_ips()
|
||||
|
||||
|
||||
def ip2bin(ip): return "".join(
|
||||
map(
|
||||
str,
|
||||
[
|
||||
"{0:08b}".format(int(x)) for x in ip.split(".")
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def is_cloudflare(url):
|
||||
instance_ip = None
|
||||
try:
|
||||
instance_ip = socket.gethostbyname(urlparse(url).hostname)
|
||||
if instance_ip is None:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
instance_bin = ip2bin(instance_ip)
|
||||
|
||||
for cloudflare_ip_mask in cloudflare_ips:
|
||||
cloudflare_ip = cloudflare_ip_mask.split('/')[0]
|
||||
cloudflare_bin = ip2bin(cloudflare_ip)
|
||||
|
||||
mask = int(cloudflare_ip_mask.split('/')[1])
|
||||
cloudflare_bin_masked = cloudflare_bin[:mask]
|
||||
instance_bin_masked = instance_bin[:mask]
|
||||
|
||||
if cloudflare_bin_masked == instance_bin_masked:
|
||||
print(url + ' is behind ' + Fore.RED +
|
||||
'cloudflare' + Style.RESET_ALL)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
is_cloudflare('https://pipedapi-libre.kavin.rocks')
|
||||
|
||||
|
||||
def filterLastSlash(urlList):
|
||||
tmp = {}
|
||||
for frontend in urlList:
|
||||
tmp[frontend] = {}
|
||||
for network in urlList[frontend]:
|
||||
tmp[frontend][network] = []
|
||||
for url in urlList[frontend][network]:
|
||||
if url.endswith('/'):
|
||||
tmp[frontend][network].append(url[:-1])
|
||||
print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
|
||||
else:
|
||||
tmp[frontend][network].append(url)
|
||||
return tmp
|
||||
|
||||
|
||||
def idnaEncode(urlList):
|
||||
tmp = {}
|
||||
for frontend in urlList:
|
||||
tmp[frontend] = {}
|
||||
for network in urlList[frontend]:
|
||||
tmp[frontend][network] = []
|
||||
for url in urlList[frontend][network]:
|
||||
try:
|
||||
encodedUrl = url.encode("idna").decode("utf8")
|
||||
tmp[frontend][network].append(encodedUrl)
|
||||
if (encodedUrl != url):
|
||||
print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
|
||||
except Exception:
|
||||
tmp[frontend][network].append(url)
|
||||
return tmp
|
||||
|
||||
|
||||
def fetchCache(frontend, mightyList):
|
||||
try:
|
||||
with open('./data.json') as file:
|
||||
mightyList[frontend] = json.load(file)[frontend]
|
||||
print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + frontend)
|
||||
except Exception:
|
||||
print(Fore.RED + 'Failed' + Style.RESET_ALL +
|
||||
' to get cached ' + frontend)
|
||||
|
||||
|
||||
def fetchFromFile(frontend, mightyList):
|
||||
with open('./fixed/' + frontend + '.json') as file:
|
||||
mightyList[frontend] = json.load(file)
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
|
||||
|
||||
networks = {}
|
||||
|
||||
with open('networks.json', 'rt') as tmp:
|
||||
networks = json.load(tmp)
|
||||
|
||||
|
||||
def fetchJsonList(frontend, url, urlItem, jsonObject, mightyList):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
rJson = json.loads(r.text)
|
||||
if jsonObject:
|
||||
rJson = rJson['instances']
|
||||
_list = {}
|
||||
for network in networks:
|
||||
_list[network] = []
|
||||
if type(urlItem) == dict:
|
||||
for item in rJson:
|
||||
for network in networks:
|
||||
if urlItem[network] is not None:
|
||||
if urlItem[network] in item and item[urlItem[network]] is not None:
|
||||
if type(item[urlItem[network]]) == list:
|
||||
for i in item[urlItem[network]]:
|
||||
if i.strip() != '':
|
||||
_list[network].append(i)
|
||||
elif item[urlItem[network]].strip() != '':
|
||||
_list[network].append(item[urlItem[network]])
|
||||
else:
|
||||
for item in rJson:
|
||||
tmpItem = item
|
||||
if urlItem is not None:
|
||||
tmpItem = item[urlItem]
|
||||
if tmpItem.strip() == '':
|
||||
continue
|
||||
elif re.search(torRegex, tmpItem):
|
||||
_list['tor'].append(tmpItem)
|
||||
elif re.search(i2pRegex, tmpItem):
|
||||
_list['i2p'].append(tmpItem)
|
||||
elif re.search(lokiRegex, tmpItem):
|
||||
_list['loki'].append(tmpItem)
|
||||
else:
|
||||
_list['clearnet'].append(tmpItem)
|
||||
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def fetchRegexList(frontend, url, regex, mightyList):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
_list = {}
|
||||
for network in networks:
|
||||
_list[network] = []
|
||||
|
||||
tmp = re.findall(regex, r.text)
|
||||
|
||||
for item in tmp:
|
||||
if item.strip() == "":
|
||||
continue
|
||||
elif re.search(torRegex, item):
|
||||
_list['tor'].append(item)
|
||||
elif re.search(i2pRegex, item):
|
||||
_list['i2p'].append(item)
|
||||
elif re.search(lokiRegex, item):
|
||||
_list['loki'].append(item)
|
||||
else:
|
||||
_list['clearnet'].append(item)
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def fetchTextList(frontend, url, prepend):
|
||||
try:
|
||||
_list = {}
|
||||
for network in networks:
|
||||
_list[network] = []
|
||||
|
||||
if type(url) == dict:
|
||||
for network in networks:
|
||||
if url[network] is not None:
|
||||
r = requests.get(url[network])
|
||||
tmp = r.text.strip().split('\n')
|
||||
for item in tmp:
|
||||
item = prepend[network] + item
|
||||
_list[network].append(item)
|
||||
else:
|
||||
r = requests.get(url)
|
||||
tmp = r.text.strip().split('\n')
|
||||
|
||||
for item in tmp:
|
||||
item = prepend + item
|
||||
if re.search(torRegex, item):
|
||||
_list['tor'].append(item)
|
||||
elif re.search(i2pRegex, item):
|
||||
_list['i2p'].append(item)
|
||||
elif re.search(lokiRegex, item):
|
||||
_list['loki'].append(item)
|
||||
else:
|
||||
_list['clearnet'].append(item)
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def isValid(url): # by avanitrachhadiya2155
|
||||
try:
|
||||
result = urlparse(url)
|
||||
return all([result.scheme, result.netloc])
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,33 @@
|
||||
import requests
|
||||
from colorama import Fore, Style
|
||||
import json
|
||||
|
||||
wolfreeAlpha_url_list = [
|
||||
"https://gqq.gitlab.io",
|
||||
"https://jqq.gitlab.io",
|
||||
"https://rqq.gitlab.io",
|
||||
"https://sqq.gitlab.io",
|
||||
"https://uqq.gitlab.io"
|
||||
]
|
||||
|
||||
|
||||
def wolfreeAlpha(mightyList):
|
||||
global wolfreeAlpha_url_list_i
|
||||
frontend = 'wolfreeAlpha'
|
||||
for instance in wolfreeAlpha_url_list:
|
||||
try:
|
||||
r = requests.get(instance+"/instances.json")
|
||||
if r.status_code != 200:
|
||||
continue
|
||||
else:
|
||||
rJson = json.loads(r.text)
|
||||
networks = rJson['wolfree']
|
||||
_list = {}
|
||||
for i in networks.keys():
|
||||
_list[i] = networks[i]
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
break
|
||||
except:
|
||||
wolfreeAlpha_url_list_i += 1
|
||||
wolfreeAlpha(wolfreeAlpha_url_list_i)
|
||||
@@ -0,0 +1,114 @@
|
||||
from .utils import fetchRegexList, fetchCache, fetchJsonList
|
||||
import requests
|
||||
import json
|
||||
from colorama import Fore, Style
|
||||
import logging
|
||||
import traceback
|
||||
import re
|
||||
|
||||
|
||||
def invidious(mightyList):
|
||||
name = 'Invidious'
|
||||
frontend = 'invidious'
|
||||
url = 'https://api.invidious.io/instances.json'
|
||||
try:
|
||||
_list = {}
|
||||
_list['clearnet'] = []
|
||||
_list['tor'] = []
|
||||
_list['i2p'] = []
|
||||
_list['loki'] = []
|
||||
r = requests.get(url)
|
||||
rJson = json.loads(r.text)
|
||||
for instance in rJson:
|
||||
if instance[1]['type'] == 'https':
|
||||
_list['clearnet'].append(instance[1]['uri'])
|
||||
elif instance[1]['type'] == 'onion':
|
||||
_list['tor'].append(instance[1]['uri'])
|
||||
elif instance[1]['type'] == 'i2p':
|
||||
_list['i2p'].append(instance[1]['uri'])
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name)
|
||||
except Exception:
|
||||
fetchCache(frontend, mightyList)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def piped(mightyList):
|
||||
frontend = 'piped'
|
||||
try:
|
||||
_list = {}
|
||||
_list['clearnet'] = []
|
||||
_list['tor'] = []
|
||||
_list['i2p'] = []
|
||||
_list['loki'] = []
|
||||
r = requests.get(
|
||||
'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md')
|
||||
|
||||
tmp = re.findall(
|
||||
r' \| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text)
|
||||
for item in tmp:
|
||||
try:
|
||||
url = requests.get(item, timeout=5).url
|
||||
if url.strip("/") == item:
|
||||
continue
|
||||
else:
|
||||
_list['clearnet'].append(url)
|
||||
except Exception:
|
||||
logging.error(traceback.format_exc())
|
||||
continue
|
||||
mightyList[frontend] = _list
|
||||
_list['clearnet'].remove("https://piped.video")
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def materialious(mightyList):
|
||||
fetchRegexList(
|
||||
'materialious',
|
||||
'https://raw.githubusercontent.com/Materialious/Materialious/main/docs/INSTANCES.md',
|
||||
r"- \[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def pipedMaterial(mightyList):
|
||||
fetchRegexList(
|
||||
'pipedMaterial',
|
||||
'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md',
|
||||
r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production",
|
||||
mightyList
|
||||
)
|
||||
|
||||
|
||||
def poketube(mightyList):
|
||||
frontend = 'poketube'
|
||||
try:
|
||||
r = requests.get(
|
||||
'https://codeberg.org/Ashley/poketube/raw/branch/main/instances.json')
|
||||
rJson = json.loads(r.text)
|
||||
_list = {
|
||||
'clearnet': [],
|
||||
'tor': [],
|
||||
'i2p': [],
|
||||
'loki': []
|
||||
|
||||
}
|
||||
for element in rJson:
|
||||
_list['clearnet'].append(element[1]['uri'])
|
||||
|
||||
mightyList[frontend] = _list
|
||||
print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
|
||||
except Exception:
|
||||
fetchCache(frontend)
|
||||
logging.error(traceback.format_exc())
|
||||
|
||||
|
||||
def hyperpipe(mightyList):
|
||||
fetchJsonList(
|
||||
'hyperpipe',
|
||||
'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json',
|
||||
'url',
|
||||
False,
|
||||
mightyList)
|
||||
Reference in New Issue
Block a user