From fb75d74ce0a9ada8946525dc396bfe58aa35200b Mon Sep 17 00:00:00 2001 From: ManeraKai Date: Wed, 21 Aug 2024 00:38:55 +0300 Subject: [PATCH] Made code readable. Fixed nitter https://github.com/libredirect/browser_extension/issues/975 --- .github/workflows/main.yml | 2 +- .gitignore | 162 ++++++++ blacklist.json | 18 +- cloudflare.py | 53 --- data.json | 175 +++------ fixed/facil.json | 6 - instances.py | 745 ------------------------------------- main.py | 89 +++++ requirements.txt | 3 + services/from_file.py | 29 ++ services/medium.py | 19 + services/other.py | 310 +++++++++++++++ services/reddit.py | 46 +++ services/search.py | 71 ++++ services/translate.py | 20 + services/utils.py | 229 ++++++++++++ services/wolfram.py | 33 ++ services/youtube.py | 114 ++++++ 18 files changed, 1188 insertions(+), 936 deletions(-) create mode 100644 .gitignore delete mode 100644 cloudflare.py delete mode 100644 fixed/facil.json delete mode 100755 instances.py create mode 100755 main.py create mode 100644 requirements.txt create mode 100644 services/from_file.py create mode 100644 services/medium.py create mode 100644 services/other.py create mode 100644 services/reddit.py create mode 100644 services/search.py create mode 100644 services/translate.py create mode 100644 services/utils.py create mode 100644 services/wolfram.py create mode 100644 services/youtube.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 30c19a6..fa6f789 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: - run: | python -m pip install --upgrade pip pip install requests colorama pyyaml - - run: python instances.py + - run: python main.py - run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..efa407c --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/blacklist.json b/blacklist.json index a1ca2de..17220da 100644 --- a/blacklist.json +++ b/blacklist.json @@ -1,6 +1,8 @@ { "cloudflare": [ "https://invidious.perennialte.ch", + "https://watch.leptons.xyz", + "https://piped.r4fo.com", "https://piped.video", "https://piped.coldforge.xyz", "https://schaun.ehwurscht.at", @@ -8,12 +10,9 @@ "https://tiktok.wpme.pl", "https://send.aurorabilisim.com", "https://send.cyberjake.xyz", + "https://nitter.lucabased.xyz", "https://nitter.rawbit.ninja", - "https://n.sneed.network", - "https://nitter.freedit.eu", - "https://nitter.altgr.xyz", - "https://jote.lile.cl", - "https://nitter.perennialte.ch", + "https://nitter.holo-mix.com", "https://redlib.freedit.eu", "https://redlib.frontendfriendly.xyz", "https://teddit.domain.glass", @@ -39,15 +38,14 @@ "https://hp.iqbalrifai.eu.org", "https://hyperpipe.darkness.services", "https://www.openstreetmap.org", - "https://paste.3server.de", "https://bonus01.hwb0307.com", - "https://privatebin.io", "https://private.rmconsulting.com", - "https://bin.oldgate.org", - "https://privatebin.luckvintage.com", - "https://pb.blackvoid.club", + "https://paste.3server.de", "https://n0t.io", + "https://bin.oldgate.org", + "https://pb.blackvoid.club", "https://sendpass.ru", + "https://privatebin.io", "https://wikiless.rawbit.ninja", "https://w.sneed.network", "https://biblioreads.mooo.com", diff --git a/cloudflare.py b/cloudflare.py deleted file mode 100644 index e36aeab..0000000 --- a/cloudflare.py +++ /dev/null @@ -1,53 +0,0 @@ -import traceback -import logging -import requests -import json -from urllib.parse import urlparse -import re -from colorama import Fore, Style -import socket - -def get_cloudflare_ips(): - r = requests.get('https://www.cloudflare.com/ips-v4') - return r.text.split('\n') - - -cloudflare_ips = get_cloudflare_ips() - - - -def ip2bin(ip): return "".join( - map( - str, - [ - "{0:08b}".format(int(x)) for x in ip.split(".") - ] - ) -) - - -def is_cloudflare(url): - instance_ip = None - try: - instance_ip = socket.gethostbyname(urlparse(url).hostname) - if instance_ip is None: - return False - except Exception: - return False - instance_bin = ip2bin(instance_ip) - - for cloudflare_ip_mask in cloudflare_ips: - cloudflare_ip = cloudflare_ip_mask.split('/')[0] - cloudflare_bin = ip2bin(cloudflare_ip) - - mask = int(cloudflare_ip_mask.split('/')[1]) - cloudflare_bin_masked = cloudflare_bin[:mask] - instance_bin_masked = instance_bin[:mask] - - if cloudflare_bin_masked == instance_bin_masked: - print(url + ' is behind ' + Fore.RED + - 'cloudflare' + Style.RESET_ALL) - return True - return False - -is_cloudflare('https://pipedapi-libre.kavin.rocks') \ No newline at end of file diff --git a/data.json b/data.json index 016d3de..e5cc115 100644 --- a/data.json +++ b/data.json @@ -1,6 +1,7 @@ { "invidious": { "clearnet": [ + "https://inv.tux.pizza", "https://iv.ggtyler.dev", "https://inv.nadeko.net", "https://invidious.jing.rocks", @@ -31,8 +32,10 @@ }, "piped": { "clearnet": [ + "https://watch.leptons.xyz", "https://piped.lunar.icu", - "http://ww25.yapi.vyper.me/?subid1=20240820-0828-04b5-b23a-516774864bee", + "http://ww25.yapi.vyper.me/?subid1=20240821-0732-1439-ac1f-4980fa340f80", + "https://piped.r4fo.com", "https://piped.video", "https://piped.projectsegfau.lt", "https://piped.in.projectsegfau.lt", @@ -50,6 +53,7 @@ "https://piped.ngn.tf", "https://piped.coldforge.xyz", "https://piped.cdsp.cz", + "https://piped.phoenixthrush.com", "https://piped.private.coffee", "https://schaun.ehwurscht.at", "https://piped.darkness.services" @@ -137,83 +141,23 @@ }, "nitter": { "clearnet": [ - "https://nitter:nitter@nitter.nixnet.services", - "https://nitter.fdn.fr", - "https://nitter.1d4.us", - "https://nitter.kavin.rocks", - "https://nitter.unixfox.eu", - "https://nitter.moomoo.me", - "https://bird.trom.tf", - "https://nitter.it", - "https://nitter.weiler.rocks", - "https://nitter.sethforprivacy.com", - "https://nitter.nl", - "https://nitter.mint.lgbt", - "https://nitter.esmailelbob.xyz", - "https://nitter.tiekoetter.com", - "https://nitter.privacy.com.de", - "https://nitter.poast.org", - "https://nitter.bird.froth.zone", - "https://nitter.cz", + "https://xcancel.com", "https://nitter.privacydev.net", - "https://tweet.lambda.dance", - "https://nitter.kylrth.com", - "https://unofficialbird.com", - "https://nitter.projectsegfau.lt", - "https://nitter.eu.projectsegfau.lt", - "https://nitter.in.projectsegfau.lt", - "https://singapore.unofficialbird.com", - "https://canada.unofficialbird.com", - "https://india.unofficialbird.com", - "https://nederland.unofficialbird.com", - "https://uk.unofficialbird.com", - "https://nitter.soopy.moe", + "https://nitter.poast.org", + "https://nitter.lucabased.xyz", + "https://nitter.kavin.rocks", + "https://nitter.tiekoetter.com", "https://nitter.qwik.space", - "https://nitter.rawbit.ninja", - "https://nt.vern.cc", - "https://n.sneed.network", - "https://nitter.smnz.de", - "https://nitter.twei.space", - "https://nitter.inpt.fr", - "https://nitter.d420.de", - "https://nitter.at", "https://bird.habedieeh.re", - "https://nitter.hostux.net", - "https://nitter.adminforge.de", - "https://nitter.platypush.tech", - "https://nitter.us.projectsegfau.lt", "https://t.com.sb", - "https://nitter.kling.gg", "https://nitter.lunar.icu", - "https://twitter.moe.ngo", - "https://nitter.freedit.eu", - "https://nitter.tux.pizza", - "https://nitter.altgr.xyz", - "https://jote.lile.cl", - "https://nitter.one", + "https://nitter.moomoo.me", + "https://nitter.bird.froth.zone", + "https://nitter.kylrth.com", "https://nitter.io.lol", - "https://nitter.no-logs.com", - "https://tweet.whateveritworks.org", - "https://nitter.fediflix.org", - "https://nitter.nohost.network", - "https://nitter.simpleprivacy.fr", - "https://nitter.onthescent.xyz", - "https://nitter.x86-64-unknown-linux-gnu.zip", - "https://nitter.private.coffee", - "https://nitter.oksocial.net", - "https://nitter.woodland.cafe", - "https://nitter.dafriser.be", - "https://nitter.catsarch.com", - "https://nitter.perennialte.ch", - "https://nitter.laserdisc.tokyo", - "https://nitter.datura.network", - "https://nitter.salastil.com", - "https://nt.ggtyler.dev", - "https://nitter.hyperreal.coffee", - "https://nitter.uni-sonia.com", - "https://n.opnxng.com", - "https://nitter.ktachibana.party", - "https://nitter.tinfoil-hat.net" + "https://nitter.rawbit.ninja", + "https://n.populas.no", + "https://nitter.holo-mix.com" ], "tor": [ "http://nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion", @@ -253,7 +197,9 @@ "http://twitter.cepyxplublbyw2f4axy4pyztfbxmf63lrt2c7uwv6wl4iixz53czload.onion", "http://nitter.coffee2m3bjsrrqqycx6ghkxrnejl2q6nl7pjw2j4clchjj6uk5zozad.onion", "http://nitter.daturab6drmkhyeia4ch5gvfc2f3wgo6bhjrv3pz6n7kxmvoznlkq4yd.onion", - "http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion" + "http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion", + "http://6g7w5lt6qcidh6vtue64udzld2mjnvxlpgddvn34qywxafxdv6a3w5id.onion", + "http://nitter.catsarchywsyuss6jdxlypsw5dc7owd5u5tr6bujxb7o6xw2hipqehyd.onion" ], "i2p": [ "http://axd6uavsstsrvstva4mzlzh4ct76rc6zdug3nxdgeitrzczhzf4q.b32.i2p", @@ -265,9 +211,7 @@ "http://4kzgzonccxuv4ctforjqx565jgd4bocgnzsy3dg4fmdf3ye2iciq.b32.i2p", "http://pjsfs4ukb6prmfx3qx3a5ef2cpcupkvcrxdh72kqn2rxc2cw4nka.b32.i2p" ], - "loki": [ - "http://nitter.priv.loki" - ] + "loki": [] }, "libreddit": { "clearnet": [ @@ -617,7 +561,6 @@ "https://xo.wtf" ], "tor": [ - "http://lbmegc3rjnekmdxuisynqdc7y3m2tgyq7gj257ooddaobxqjw36bdayd.onion", "http://4n53nafyi77iplnbrpmxnp3x4exbswwxigujaxy3b37fvr7bvlopxeyd.onion", "http://whereissky27lxfdrs7ct7a5ievor3tl67qi77rv5luga7wod5jf2mad.onion", "http://searchpie7fejn4wctzm6fgm6oz5vnziltfzzuec6b4qmkr3wrqaoqqd.onion", @@ -832,14 +775,6 @@ "i2p": [], "loki": [] }, - "facil": { - "clearnet": [ - "https://facilmap.org" - ], - "tor": [], - "i2p": [], - "loki": [] - }, "osm": { "clearnet": [ "https://www.openstreetmap.org" @@ -905,49 +840,47 @@ }, "privateBin": { "clearnet": [ - "https://secret.hummel-web.at", - "https://bin.0xfc.de", - "https://paste.3server.de", - "https://paste.tuxcloud.net", - "https://snip.dssr.ch", - "https://paste.ononoki.org", - "https://p.vinci-concessions.com", - "https://extrait.facil.services", "https://bonus01.hwb0307.com", + "https://extrait.facil.services", + "https://paste.momobako.com", + "https://paste.waays.fr", + "https://notebin.de", + "https://vip.ci", "https://privatebin.wsmm.de", - "https://paste.devsite.pl", - "https://onetimesecret.trans-ix.nl", - "https://secrets.l25.cloud", + "https://secret.hummel-web.at", "https://p.darklab.sh", - "https://privatebin.io", - "https://paste.hackliberty.org", - "https://0.jaegers.net", + "https://bin.bus-hit.me", + "https://paste.devsite.pl", + "https://ots.kocsar.at", + "https://onetimesecret.trans-ix.nl", "https://private.rmconsulting.com", "https://zerobin.thican.net", - "https://bin.nixnet.services", - "https://paste.momobako.com", - "https://ots.kocsar.at", - "https://privatebin.net", - "https://vip.ci", - "https://bin.oldgate.org", - "https://paste.waays.fr", - "https://0.0g.gg", - "https://privatebin.luckvintage.com", - "https://paste.encrypt.co.il", - "https://paste.skynetcloud.site", - "https://paste.techbag.app", - "https://pb.blackvoid.club", - "https://privatebin.arch-linux.cz", - "https://paste.tildeverse.org", - "https://n0t.io", - "https://sendpass.ru", "https://secrets.true.nl", - "https://notebin.de", - "https://paste.hostux.net", - "https://bin.ngn.tf", - "https://encryp.ch/note", + "https://paste.tuxcloud.net", + "https://paste.3server.de", + "https://n0t.io", + "https://bin.oldgate.org", + "https://paste.techbag.app", + "https://bin.nixnet.services", + "https://privatebin.arch-linux.cz", "https://paste.ottertelecom.com", - "https://bin.bus-hit.me" + "https://paste.encrypt.co.il", + "https://encryp.ch/note", + "https://bin.ngn.tf", + "https://bin.0xfc.de", + "https://pb.blackvoid.club", + "https://paste.tildeverse.org", + "https://paste.ononoki.org", + "https://sendpass.ru", + "https://snip.dssr.ch", + "https://privatebin.net", + "https://paste.hostux.net", + "https://privatebin.io", + "https://paste.hackliberty.org", + "https://p.vinci-concessions.com", + "https://0.jaegers.net", + "https://secrets.l25.cloud", + "https://privatebin.xi.ht" ], "tor": [], "i2p": [], diff --git a/fixed/facil.json b/fixed/facil.json deleted file mode 100644 index 6c89eb1..0000000 --- a/fixed/facil.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "clearnet": ["https://facilmap.org"], - "tor": [], - "i2p": [], - "loki": [] -} diff --git a/instances.py b/instances.py deleted file mode 100755 index 62952ec..0000000 --- a/instances.py +++ /dev/null @@ -1,745 +0,0 @@ -#!/usr/bin/python3 - -import traceback -import logging -import requests -import json -from urllib.parse import urlparse -import re -from colorama import Fore, Style -import socket -import yaml - -mightyList = {} -networks = {} - -startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*" -endRegex = "(?:\/[^\s\/]+)*\/?" -torRegex = startRegex + "onion" + endRegex -i2pRegex = startRegex + "i2p" + endRegex -lokiRegex = startRegex + "loki" + endRegex -authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex - -with open('networks.json', 'rt') as tmp: - networks = json.load(tmp) - - -def filterLastSlash(urlList): - tmp = {} - for frontend in urlList: - tmp[frontend] = {} - for network in urlList[frontend]: - tmp[frontend][network] = [] - for url in urlList[frontend][network]: - if url.endswith('/'): - tmp[frontend][network].append(url[:-1]) - print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) - else: - tmp[frontend][network].append(url) - return tmp - - -def idnaEncode(urlList): - tmp = {} - for frontend in urlList: - tmp[frontend] = {} - for network in urlList[frontend]: - tmp[frontend][network] = [] - for url in urlList[frontend][network]: - try: - encodedUrl = url.encode("idna").decode("utf8") - tmp[frontend][network].append(encodedUrl) - if (encodedUrl != url): - print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) - except Exception: - tmp[frontend][network].append(url) - return tmp - - -def ip2bin(ip): return "".join( - map( - str, - [ - "{0:08b}".format(int(x)) for x in ip.split(".") - ] - ) -) - - -def get_cloudflare_ips(): - r = requests.get('https://www.cloudflare.com/ips-v4') - return r.text.split('\n') - - -cloudflare_ips = get_cloudflare_ips() - - -def is_cloudflare(url): - instance_ip = None - try: - instance_ip = socket.gethostbyname(urlparse(url).hostname) - if instance_ip is None: - return False - except Exception: - return False - instance_bin = ip2bin(instance_ip) - - for cloudflare_ip_mask in cloudflare_ips: - cloudflare_ip = cloudflare_ip_mask.split('/')[0] - cloudflare_bin = ip2bin(cloudflare_ip) - - mask = int(cloudflare_ip_mask.split('/')[1]) - cloudflare_bin_masked = cloudflare_bin[:mask] - instance_bin_masked = instance_bin[:mask] - - if cloudflare_bin_masked == instance_bin_masked: - print(url + ' is behind ' + Fore.RED + - 'cloudflare' + Style.RESET_ALL) - return True - return False - - -def fetchCache(frontend): - try: - with open('./data.json') as file: - mightyList[frontend] = json.load(file)[frontend] - print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + frontend) - except Exception: - print(Fore.RED + 'Failed' + Style.RESET_ALL + - ' to get cached ' + frontend) - - -def fetchFromFile(frontend): - with open('./fixed/' + frontend + '.json') as file: - mightyList[frontend] = json.load(file) - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - - -def fetchJsonList(frontend, url, urlItem, jsonObject): - try: - r = requests.get(url) - rJson = json.loads(r.text) - if jsonObject: - rJson = rJson['instances'] - _list = {} - for network in networks: - _list[network] = [] - if type(urlItem) == dict: - for item in rJson: - for network in networks: - if urlItem[network] is not None: - if urlItem[network] in item and item[urlItem[network]] is not None: - if type(item[urlItem[network]]) == list: - for i in item[urlItem[network]]: - if i.strip() != '': - _list[network].append(i) - elif item[urlItem[network]].strip() != '': - _list[network].append(item[urlItem[network]]) - else: - for item in rJson: - tmpItem = item - if urlItem is not None: - tmpItem = item[urlItem] - if tmpItem.strip() == '': - continue - elif re.search(torRegex, tmpItem): - _list['tor'].append(tmpItem) - elif re.search(i2pRegex, tmpItem): - _list['i2p'].append(tmpItem) - elif re.search(lokiRegex, tmpItem): - _list['loki'].append(tmpItem) - else: - _list['clearnet'].append(tmpItem) - - mightyList[frontend] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - except Exception: - fetchCache(frontend) - logging.error(traceback.format_exc()) - - -def fetchRegexList(frontend, url, regex): - try: - r = requests.get(url) - _list = {} - for network in networks: - _list[network] = [] - - tmp = re.findall(regex, r.text) - - for item in tmp: - if item.strip() == "": - continue - elif re.search(torRegex, item): - _list['tor'].append(item) - elif re.search(i2pRegex, item): - _list['i2p'].append(item) - elif re.search(lokiRegex, item): - _list['loki'].append(item) - else: - _list['clearnet'].append(item) - mightyList[frontend] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - except Exception: - fetchCache(frontend) - logging.error(traceback.format_exc()) - - -def fetchTextList(frontend, url, prepend): - try: - _list = {} - for network in networks: - _list[network] = [] - - if type(url) == dict: - for network in networks: - if url[network] is not None: - r = requests.get(url[network]) - tmp = r.text.strip().split('\n') - for item in tmp: - item = prepend[network] + item - _list[network].append(item) - else: - r = requests.get(url) - tmp = r.text.strip().split('\n') - - for item in tmp: - item = prepend + item - if re.search(torRegex, item): - _list['tor'].append(item) - elif re.search(i2pRegex, item): - _list['i2p'].append(item) - elif re.search(lokiRegex, item): - _list['loki'].append(item) - else: - _list['clearnet'].append(item) - mightyList[frontend] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - except Exception: - fetchCache(frontend) - logging.error(traceback.format_exc()) - - -def invidious(): - name = 'Invidious' - frontend = 'invidious' - url = 'https://api.invidious.io/instances.json' - try: - _list = {} - _list['clearnet'] = [] - _list['tor'] = [] - _list['i2p'] = [] - _list['loki'] = [] - r = requests.get(url) - rJson = json.loads(r.text) - for instance in rJson: - if instance[1]['type'] == 'https': - _list['clearnet'].append(instance[1]['uri']) - elif instance[1]['type'] == 'onion': - _list['tor'].append(instance[1]['uri']) - elif instance[1]['type'] == 'i2p': - _list['i2p'].append(instance[1]['uri']) - mightyList[frontend] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) - except Exception: - fetchCache(frontend) - logging.error(traceback.format_exc()) - - -def piped(): - frontend = 'piped' - try: - _list = {} - _list['clearnet'] = [] - _list['tor'] = [] - _list['i2p'] = [] - _list['loki'] = [] - r = requests.get( - 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md') - - tmp = re.findall( - r' \| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) - for item in tmp: - try: - url = requests.get(item, timeout=5).url - if url.strip("/") == item: - continue - else: - _list['clearnet'].append(url) - except Exception: - logging.error(traceback.format_exc()) - continue - mightyList[frontend] = _list - _list['clearnet'].remove("https://piped.video") - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - except Exception: - fetchCache(frontend) - logging.error(traceback.format_exc()) - - -def materialious(): - fetchRegexList('materialious', 'https://raw.githubusercontent.com/Materialious/Materialious/main/docs/INSTANCES.md', - r"- \[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)") - - -def pipedMaterial(): - fetchRegexList('pipedMaterial', 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md', - r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production") - - -def cloudtube(): - fetchFromFile('cloudtube') - - -def proxitok(): - fetchRegexList('proxiTok', 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md', - r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))") - - -def send(): - fetchRegexList('send', 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md', - r"(https.*?) \|.*?\n") - - -def nitter(): - frontend = 'nitter' - try: - r = requests.get( - 'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md') - _list = {} - - _list['clearnet'] = [] - public = re.findall(r"## Public((?:\n|.*)+?)##", r.text) - for line in public[0].split('\n'): - result = re.findall(r"^\| \[.*?\]\((https.*?)\)", line) - if len(result) > 0: - _list['clearnet'].append(result[0]) - - _list['tor'] = [] - public = re.findall(r"## Tor((?:\n|.*)+?)##", r.text) - for line in public[0].split('\n'): - result = re.findall(r"^\| <(http.*?)\/?>", line) - if len(result) > 0: - _list['tor'].append(result[0]) - - _list['i2p'] = [] - public = re.findall(r"## I2P((?:\n|.*)+?)##", r.text) - for line in public[0].split('\n'): - result = re.findall(r"^- <(http.*?)\/?>", line) - if len(result) > 0: - _list['i2p'].append(result[0]) - - _list['loki'] = [] - public = re.findall(r"## Lokinet((?:\n|.*)+?)##", r.text) - for line in public[0].split('\n'): - result = re.findall(r"^- <(http.*?)\/?>", line) - if len(result) > 0: - _list['loki'].append(result[0]) - - mightyList[frontend] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - - except Exception: - fetchCache(frontend) - logging.error(traceback.format_exc()) - - # fetchRegexList('nitter', '', - # r"(?:(?:\| )|(?:- ))\[(?:(?:\S+\.)+[a-zA-Z0-9]+)\/?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]+)\/?\)(?:(?: (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,4} +\|(?:(?:\n)|(?: ❌)|(?: ✅)|(?: ❓)|(?: \[)))|(?:\n))") - - -def libreddit(): - fetchJsonList('libreddit', 'https://github.com/libreddit/libreddit-instances/raw/master/instances.json', - {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True) - - -def redlib(): - fetchJsonList('redlib', 'https://github.com/redlib-org/redlib-instances/raw/main/instances.json', - {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True) - - -def teddit(): - fetchJsonList('teddit', 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json', - {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) - - -def scribe(): - fetchJsonList( - 'scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', None, False) - - -def quetre(): - fetchJsonList('quetre', 'https://raw.githubusercontent.com/zyachel/quetre/main/instances.json', - {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False) - - -def libremdb(): - fetchJsonList('libremdb', 'https://raw.githubusercontent.com/zyachel/libremdb/main/instances.json', - {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False) - - -def simplytranslate(): - fetchFromFile('simplyTranslate') - - -def linvgatranslate(): - fetchJsonList( - 'lingva', 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False) - - -def searxng(): - r = requests.get( - 'https://searx.space/data/instances.json') - rJson = json.loads(r.text) - searxngList = {} - searxngList['clearnet'] = [] - searxngList['tor'] = [] - searxngList['i2p'] = [] - searxngList['loki'] = [] - - for item in rJson['instances']: - if re.search(torRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng': - searxngList['tor'].append(item[:-1]) - elif re.search(i2pRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng': - searxngList['i2p'].append(item[:-1]) - elif rJson['instances'][item].get('generator') == 'searxng': - searxngList['clearnet'].append(item[:-1]) - - mightyList['searxng'] = searxngList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG') - - -def searx(): - searxList = {} - searxList['clearnet'] = [] - searxList['tor'] = [] - searxList['i2p'] = [] - searxList['loki'] = [] - r = requests.get( - 'https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml') - data = yaml.safe_load(r.text) - for key in data: - searxList['clearnet'].append(key) - if 'additional_urls' in data[key]: - for additional_url in data[key]['additional_urls']: - if data[key]['additional_urls'][additional_url] == "Hidden Service": - searxList['tor'].append(additional_url) - mightyList['searx'] = searxList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG') - - -def whoogle(): - fetchRegexList('whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md', - r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ") - - -def librex(): - fetchJsonList('librex', 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json', - {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, True) - - -def rimgo(): - try: - r = requests.get('https://rimgo.codeberg.page/api.json') - rJson = json.loads(r.text) - _list = { - 'clearnet': [], - 'tor': [], - } - for instance in rJson['clearnet']: - _list['clearnet'].append(instance['url']) - - for instance in rJson['tor']: - _list['tor'].append(instance['url']) - - mightyList['rimgo'] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'rimgo') - except Exception: - fetchCache('rimgo') - logging.error(traceback.format_exc()) - - -def pixivFe(): - fetchRegexList('pixivFe', 'https://codeberg.org/VnPower/pixivfe/raw/branch/main/README.md', - r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/? +\|") - - -def safetwitch(): - fetchRegexList('safetwitch', 'https://codeberg.org/dragongoose/safetwitch/raw/branch/master/README.md', - re.compile(r"^\| \[.*?\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)",re.MULTILINE)) - - -def hyperpipe(): - fetchJsonList( - 'hyperpipe', 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', 'url', False) - - -def facil(): - fetchFromFile('facil') - - -def osm(): - fetchFromFile('osm') - - -def libreTranslate(): - fetchRegexList('libreTranslate', 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md', - r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|") - - -def breezeWiki(): - fetchJsonList( - 'breezeWiki', 'https://docs.breezewiki.com/files/instances.json', 'instance', False) - - -def binternet(): - fetchRegexList('binternet', 'https://raw.githubusercontent.com/Ahwxorg/Binternet/main/README.md', - r"\| \[[\w\.]+!?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/?\)") - - -def privateBin(): - fetchJsonList('privateBin', 'https://privatebin.info/directory/api?top=100&https_redirect=true&min_rating=A&csp_header=true&min_uptime=100&attachments=true', 'url', False) - - -def neuters(): - fetchFromFile('neuters') - - -def libMedium(): - fetchRegexList('libMedium', 'https://raw.githubusercontent.com/realaravinth/libmedium/master/README.md', - r"\| (https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/? +\|") - - -def dumb(): - fetchJsonList('dumb', 'https://raw.githubusercontent.com/rramiachraf/dumb/main/instances.json', - {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False) - - -def ruralDictionary(): - fetchJsonList('ruralDictionary', - 'https://codeberg.org/zortazert/rural-dictionary/raw/branch/master/instances.json', - {'clearnet': 'clearnet', 'tor': 'tor', - 'i2p': 'i2p', 'loki': None}, - False - ) - - -def anonymousOverflow(): - fetchRegexList('anonymousOverflow', 'https://raw.githubusercontent.com/httpjamesm/AnonymousOverflow/main/README.md', - r"\| \[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) +\|") - - -def wikiless(): - fetchFromFile('wikiless') - - -def biblioReads(): - fetchJsonList( - 'biblioReads', - 'https://raw.githubusercontent.com/nesaku/BiblioReads/main/instances.json', - {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, - False - ) - - -def suds(): - fetchJsonList( - 'suds', 'https://git.vern.cc/cobra/Suds/raw/branch/main/instances.json', - { - 'clearnet': 'clearnet', - 'tor': 'tor', - 'i2p': 'i2p', - 'loki': None - }, - False, - ) - - -def poketube(): - frontend = 'poketube' - try: - r = requests.get( - 'https://codeberg.org/Ashley/poketube/raw/branch/main/instances.json') - rJson = json.loads(r.text) - _list = { - 'clearnet': [], - 'tor': [], - 'i2p': [], - 'loki': [] - - } - for element in rJson: - _list['clearnet'].append(element[1]['uri']) - - mightyList[frontend] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - except Exception: - fetchCache(frontend) - logging.error(traceback.format_exc()) - - -def gothub(): - fetchJsonList( - 'gothub', - 'https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json', - { - 'clearnet': 'link', - 'tor': None, - 'i2p': None, - 'loki': None - }, - False - ) - - -def mikuInvidious(): - fetchFromFile('mikuInvidious') - - -def tent(): - fetchJsonList('tent', 'https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json', 'url', False) - - -def laboratory(): - fetchRegexList('laboratory', 'https://git.vitali64.duckdns.org/utils/laboratory.git/plain/README.md', - r"\| (https:\/{2}.*?) \|") - - -wolfreeAlpha_url_list = [ - "https://gqq.gitlab.io", - "https://jqq.gitlab.io", - "https://rqq.gitlab.io", - "https://sqq.gitlab.io", - "https://uqq.gitlab.io" -] -wolfreeAlpha_url_list_i = 0 - - -def wolfreeAlpha(i): - global wolfreeAlpha_url_list_i - frontend = 'wolfreeAlpha' - try: - r = requests.get(wolfreeAlpha_url_list[i]+"/instances.json") - if r.status_code != 200: - wolfreeAlpha_url_list_i += 1 - wolfreeAlpha(wolfreeAlpha_url_list_i) - else: - rJson = json.loads(r.text) - networks = rJson['wolfree'] - _list = {} - for i in networks.keys(): - _list[i] = networks[i] - mightyList[frontend] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) - except: - wolfreeAlpha_url_list_i += 1 - wolfreeAlpha(wolfreeAlpha_url_list_i) - - -def jiti(): - fetchRegexList('jitsi', "https://raw.githubusercontent.com/jitsi/handbook/master/docs/community/instances.md", - r"\|(?:(?: | )+)+((?:[a-z]+\.)+[a-z]+)(?:(?: | )+)+\|") - mightyList['jitsi']['clearnet'] = list( - map(lambda x: "https://"+x, mightyList['jitsi']['clearnet'])) - mightyList['jitsi']['clearnet'].insert(0, 'https://8x8.vc') - mightyList['jitsi']['clearnet'].insert(0, 'https://meet.jit.si') - - -def proxigram(): - fetchRegexList('proxigram', 'https://codeberg.org/ThePenguinDev/Proxigram/wiki/raw/Instances.md', - r"\[(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\]") - - -def twineo(): - fetchFromFile('twineo') - - -def priviblur(): - fetchRegexList('priviblur', 'https://raw.githubusercontent.com/syeopite/priviblur/master/instances.md', - r"\| ?\[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\) ?|") - - -def isValid(url): # This code is contributed by avanitrachhadiya2155 - try: - result = urlparse(url) - return all([result.scheme, result.netloc]) - except Exception: - return False - - -invidious() -materialious() -piped() -pipedMaterial() -cloudtube() -proxitok() -send() -nitter() -libreddit() -redlib() -teddit() -scribe() -quetre() -libremdb() -simplytranslate() -linvgatranslate() -libreTranslate() -searxng() -searx() -whoogle() -librex() -rimgo() -pixivFe() -safetwitch() -hyperpipe() -facil() -osm() -breezeWiki() -binternet() -privateBin() -neuters() -ruralDictionary() -libMedium() -dumb() -anonymousOverflow() -wikiless() -biblioReads() -suds() -poketube() -gothub() -mikuInvidious() -wolfreeAlpha(wolfreeAlpha_url_list_i) -jiti() -proxigram() -tent() -laboratory() -twineo() -priviblur() - -mightyList = filterLastSlash(mightyList) -mightyList = idnaEncode(mightyList) - -cloudflare = [] -for k1, v1 in mightyList.items(): - if type(mightyList[k1]) is dict: - for k2, v2 in mightyList[k1].items(): - for instance in mightyList[k1][k2]: - if (not isValid(instance)): - mightyList[k1][k2].remove(instance) - print("removed " + instance) - else: - if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance): - cloudflare.append(instance) -blacklist = { - 'cloudflare': cloudflare -} - -# Writing to file -json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) -with open('./data.json', 'w') as outfile: - outfile.write(json_object) -print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json') - -json_object = json.dumps(blacklist, ensure_ascii=False, indent=2) -with open('./blacklist.json', 'w') as outfile: - outfile.write(json_object) -print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json') diff --git a/main.py b/main.py new file mode 100755 index 0000000..9f168c0 --- /dev/null +++ b/main.py @@ -0,0 +1,89 @@ +import json +from colorama import Fore, Style +from services.from_file import * +from services.medium import * +from services.other import * +from services.reddit import * +from services.search import * +from services.translate import * +from services.utils import * +from services.wolfram import * +from services.youtube import * + +mightyList = {} + +invidious(mightyList) +materialious(mightyList) +piped(mightyList) +pipedMaterial(mightyList) +cloudtube(mightyList) +proxitok(mightyList) +send(mightyList) +nitter(mightyList) +libreddit(mightyList) +redlib(mightyList) +teddit(mightyList) +scribe(mightyList) +quetre(mightyList) +libremdb(mightyList) +simplytranslate(mightyList) +linvgatranslate(mightyList) +libreTranslate(mightyList) +searxng(mightyList) +searx(mightyList) +whoogle(mightyList) +librex(mightyList) +rimgo(mightyList) +pixivFe(mightyList) +safetwitch(mightyList) +hyperpipe(mightyList) +osm(mightyList) +breezeWiki(mightyList) +binternet(mightyList) +privateBin(mightyList) +neuters(mightyList) +ruralDictionary(mightyList) +libMedium(mightyList) +dumb(mightyList) +anonymousOverflow(mightyList) +wikiless(mightyList) +biblioReads(mightyList) +suds(mightyList) +poketube(mightyList) +gothub(mightyList) +mikuInvidious(mightyList) +wolfreeAlpha(mightyList) +jiti(mightyList) +proxigram(mightyList) +tent(mightyList) +laboratory(mightyList) +twineo(mightyList) +priviblur(mightyList) + +mightyList = filterLastSlash(mightyList) +mightyList = idnaEncode(mightyList) + +cloudflare = [] +for k1, v1 in mightyList.items(): + if type(mightyList[k1]) is dict: + for k2, v2 in mightyList[k1].items(): + for instance in mightyList[k1][k2]: + if not isValid(instance): + mightyList[k1][k2].remove(instance) + print("removed " + instance) + else: + if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance): + cloudflare.append(instance) +blacklist = { + 'cloudflare': cloudflare +} + +json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) +with open('./data.json', 'w') as outfile: + outfile.write(json_object) +print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json') + +json_object = json.dumps(blacklist, ensure_ascii=False, indent=2) +with open('./blacklist.json', 'w') as outfile: + outfile.write(json_object) +print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0dfc2c3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests +colorama +pyyaml \ No newline at end of file diff --git a/services/from_file.py b/services/from_file.py new file mode 100644 index 0000000..e09e5c8 --- /dev/null +++ b/services/from_file.py @@ -0,0 +1,29 @@ +from .utils import fetchFromFile + + +def osm(mightyList): + fetchFromFile('osm', mightyList) + + +def neuters(mightyList): + fetchFromFile('neuters', mightyList) + + +def wikiless(mightyList): + fetchFromFile('wikiless', mightyList) + + +def mikuInvidious(mightyList): + fetchFromFile('mikuInvidious', mightyList) + + +def twineo(mightyList): + fetchFromFile('twineo', mightyList) + + +def cloudtube(mightyList): + fetchFromFile('cloudtube', mightyList) + + +def simplytranslate(mightyList): + fetchFromFile('simplyTranslate', mightyList) diff --git a/services/medium.py b/services/medium.py new file mode 100644 index 0000000..e19ca65 --- /dev/null +++ b/services/medium.py @@ -0,0 +1,19 @@ +from .utils import fetchJsonList, fetchRegexList + + +def scribe(mightyList): + fetchJsonList( + 'scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', + None, + False, + mightyList + ) + + +def libMedium(mightyList): + fetchRegexList( + 'libMedium', + 'https://raw.githubusercontent.com/realaravinth/libmedium/master/README.md', + r"\| (https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/? +\|", + mightyList + ) diff --git a/services/other.py b/services/other.py new file mode 100644 index 0000000..5b0c2fd --- /dev/null +++ b/services/other.py @@ -0,0 +1,310 @@ +import requests +import json +from .utils import fetchJsonList, fetchRegexList, fetchCache +from colorama import Fore, Style +import logging +import traceback +import re + + +def tent(mightyList): + fetchJsonList( + 'tent', + 'https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json', + 'url', + False, + mightyList + ) + + +def ruralDictionary(mightyList): + fetchJsonList( + 'ruralDictionary', + 'https://codeberg.org/zortazert/rural-dictionary/raw/branch/master/instances.json', + { + 'clearnet': 'clearnet', + 'tor': 'tor', + 'i2p': 'i2p', + 'loki': None + }, + False, + mightyList + ) + + +def laboratory(mightyList): + fetchRegexList( + 'laboratory', + 'https://git.vitali64.duckdns.org/utils/laboratory.git/plain/README.md', + r"\| (https:\/{2}.*?) \|", + mightyList + ) + + +def gothub(mightyList): + fetchJsonList( + 'gothub', + 'https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json', + { + 'clearnet': 'link', + 'tor': None, + 'i2p': None, + 'loki': None + }, + False, + mightyList + ) + + +def biblioReads(mightyList): + fetchJsonList( + 'biblioReads', + 'https://raw.githubusercontent.com/nesaku/BiblioReads/main/instances.json', + { + 'clearnet': 'url', + 'tor': 'onion', + 'i2p': 'i2p', + 'loki': None + }, + False, + mightyList + ) + + +def libremdb(mightyList): + fetchJsonList( + 'libremdb', + 'https://raw.githubusercontent.com/zyachel/libremdb/main/instances.json', + { + 'clearnet': 'clearnet', + 'tor': 'tor', + 'i2p': 'i2p', + 'loki': None + }, + False, + mightyList + ) + + +def breezeWiki(mightyList): + fetchJsonList( + 'breezeWiki', + 'https://docs.breezewiki.com/files/instances.json', + 'instance', + False, + mightyList + ) + + +def binternet(mightyList): + fetchRegexList( + 'binternet', + 'https://raw.githubusercontent.com/Ahwxorg/Binternet/main/README.md', + r"\| \[[\w\.]+!?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/?\)", + mightyList + ) + + +def dumb(mightyList): + fetchJsonList( + 'dumb', + 'https://raw.githubusercontent.com/rramiachraf/dumb/main/instances.json', + { + 'clearnet': 'clearnet', + 'tor': 'tor', + 'i2p': 'i2p', + 'loki': None + }, + False, + mightyList + ) + + +def suds(mightyList): + fetchJsonList( + 'suds', 'https://git.vern.cc/cobra/Suds/raw/branch/main/instances.json', + { + 'clearnet': 'clearnet', + 'tor': 'tor', + 'i2p': 'i2p', + 'loki': None + }, + False, + mightyList + ) + + +def proxigram(mightyList): + fetchRegexList( + 'proxigram', + 'https://codeberg.org/ThePenguinDev/Proxigram/wiki/raw/Instances.md', + r"\[(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\]", + mightyList + ) + + +def rimgo(mightyList): + try: + r = requests.get('https://rimgo.codeberg.page/api.json') + rJson = json.loads(r.text) + _list = { + 'clearnet': [], + 'tor': [], + } + for instance in rJson['clearnet']: + _list['clearnet'].append(instance['url']) + + for instance in rJson['tor']: + _list['tor'].append(instance['url']) + + mightyList['rimgo'] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'rimgo') + except Exception: + fetchCache('rimgo') + logging.error(traceback.format_exc()) + + +def jiti(mightyList): + fetchRegexList( + 'jitsi', + "https://raw.githubusercontent.com/jitsi/handbook/master/docs/community/instances.md", + r"\|(?:(?: | )+)+((?:[a-z]+\.)+[a-z]+)(?:(?: | )+)+\|", + mightyList + ) + mightyList['jitsi']['clearnet'] = list( + map( + lambda x: "https://"+x, + mightyList['jitsi']['clearnet'] + ) + ) + mightyList['jitsi']['clearnet'].insert(0, 'https://8x8.vc') + mightyList['jitsi']['clearnet'].insert(0, 'https://meet.jit.si') + + +def anonymousOverflow(mightyList): + fetchRegexList( + 'anonymousOverflow', + 'https://raw.githubusercontent.com/httpjamesm/AnonymousOverflow/main/README.md', + r"\| \[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) +\|", + mightyList + ) + + +def proxitok(mightyList): + fetchRegexList( + 'proxiTok', + 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md', + r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))", + mightyList + ) + + +def priviblur(mightyList): + fetchRegexList( + 'priviblur', + 'https://raw.githubusercontent.com/syeopite/priviblur/master/instances.md', + r"\| ?\[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\) ?|", + mightyList + ) + + +def safetwitch(mightyList): + fetchRegexList( + 'safetwitch', + 'https://codeberg.org/dragongoose/safetwitch/raw/branch/master/README.md', + re.compile( + r"^\| \[.*?\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)", + re.MULTILINE + ), + mightyList + ) + + +def nitter(mightyList): + frontend = 'nitter' + try: + r = requests.get( + 'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md') + _list = {} + + _list['clearnet'] = [] + public = re.findall(r"## Public((?:\n|.*)+?)##", r.text) + if public: + for line in public[0].split('\n'): + result = re.findall(r"^\| \[.*?\]\((https.*?)\)", line) + if len(result) > 0: + _list['clearnet'].append(result[0]) + + _list['tor'] = [] + public = re.findall(r"## Tor((?:\n|.*)+?)##", r.text) + if public: + for line in public[0].split('\n'): + result = re.findall(r"^\| <(http.*?)\/?>", line) + if len(result) > 0: + _list['tor'].append(result[0]) + + _list['i2p'] = [] + public = re.findall(r"## I2P((?:\n|.*)+?)##", r.text) + if public: + for line in public[0].split('\n'): + result = re.findall(r"^- <(http.*?)\/?>", line) + if len(result) > 0: + _list['i2p'].append(result[0]) + + _list['loki'] = [] + public = re.findall(r"## Lokinet((?:\n|.*)+?)##", r.text) + if public: + for line in public[0].split('\n'): + result = re.findall(r"^- <(http.*?)\/?>", line) + if len(result) > 0: + _list['loki'].append(result[0]) + + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + + except Exception: + fetchCache(frontend, mightyList) + logging.error(traceback.format_exc()) + + +def send(mightyList): + fetchRegexList( + 'send', + 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md', + r"(https.*?) \|.*?\n", + mightyList + ) + + +def quetre(mightyList): + fetchJsonList( + 'quetre', + 'https://raw.githubusercontent.com/zyachel/quetre/main/instances.json', + { + 'clearnet': 'clearnet', + 'tor': 'tor', + 'i2p': 'i2p', + 'loki': None + }, + False, + mightyList + ) + + +def privateBin(mightyList): + fetchJsonList( + 'privateBin', + 'https://privatebin.info/directory/api?top=100&https_redirect=true&min_rating=A&csp_header=true&min_uptime=100&attachments=true', + 'url', + False, + mightyList + ) + + +def pixivFe(mightyList): + fetchRegexList( + 'pixivFe', + 'https://codeberg.org/VnPower/pixivfe/raw/branch/main/README.md', + r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/? +\|", + mightyList + ) diff --git a/services/reddit.py b/services/reddit.py new file mode 100644 index 0000000..5a07d19 --- /dev/null +++ b/services/reddit.py @@ -0,0 +1,46 @@ +from .utils import fetchJsonList + + +def libreddit(mightyList): + fetchJsonList( + 'libreddit', + 'https://github.com/libreddit/libreddit-instances/raw/master/instances.json', + { + 'clearnet': 'url', + 'tor': 'onion', + 'i2p': 'i2p', + 'loki': None + }, + True, + mightyList + ) + + +def redlib(mightyList): + fetchJsonList( + 'redlib', + 'https://github.com/redlib-org/redlib-instances/raw/main/instances.json', + { + 'clearnet': 'url', + 'tor': 'onion', + 'i2p': 'i2p', + 'loki': None + }, + True, + mightyList + ) + + +def teddit(mightyList): + fetchJsonList( + 'teddit', + 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json', + { + 'clearnet': 'url', + 'tor': 'onion', + 'i2p': 'i2p', + 'loki': None + }, + False, + mightyList + ) diff --git a/services/search.py b/services/search.py new file mode 100644 index 0000000..cdfb943 --- /dev/null +++ b/services/search.py @@ -0,0 +1,71 @@ +import requests +from colorama import Fore, Style +import json +import re +from .utils import torRegex, i2pRegex, fetchRegexList, fetchJsonList +import yaml + + +def searxng(mightyList): + r = requests.get( + 'https://searx.space/data/instances.json') + rJson = json.loads(r.text) + searxngList = {} + searxngList['clearnet'] = [] + searxngList['tor'] = [] + searxngList['i2p'] = [] + searxngList['loki'] = [] + + for item in rJson['instances']: + if re.search(torRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng': + searxngList['tor'].append(item[:-1]) + elif re.search(i2pRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng': + searxngList['i2p'].append(item[:-1]) + elif rJson['instances'][item].get('generator') == 'searxng': + searxngList['clearnet'].append(item[:-1]) + + mightyList['searxng'] = searxngList + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG') + + +def searx(mightyList): + searxList = {} + searxList['clearnet'] = [] + searxList['tor'] = [] + searxList['i2p'] = [] + searxList['loki'] = [] + r = requests.get( + 'https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml') + data = yaml.safe_load(r.text) + for key in data: + searxList['clearnet'].append(key) + if 'additional_urls' in data[key]: + for additional_url in data[key]['additional_urls']: + if data[key]['additional_urls'][additional_url] == "Hidden Service": + searxList['tor'].append(additional_url) + mightyList['searx'] = searxList + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG') + + +def whoogle(mightyList): + fetchRegexList( + 'whoogle', + 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md', + r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ", + mightyList + ) + + +def librex(mightyList): + fetchJsonList( + 'librex', + 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json', + { + 'clearnet': 'clearnet', + 'tor': 'tor', + 'i2p': 'i2p', + 'loki': None + }, + True, + mightyList + ) diff --git a/services/translate.py b/services/translate.py new file mode 100644 index 0000000..0b38490 --- /dev/null +++ b/services/translate.py @@ -0,0 +1,20 @@ +from .utils import fetchJsonList, fetchRegexList + + +def linvgatranslate(mightyList): + fetchJsonList( + 'lingva', + 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', + None, + False, + mightyList + ) + + +def libreTranslate(mightyList): + fetchRegexList( + 'libreTranslate', + 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md', + r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|", + mightyList + ) diff --git a/services/utils.py b/services/utils.py new file mode 100644 index 0000000..1845d06 --- /dev/null +++ b/services/utils.py @@ -0,0 +1,229 @@ +import traceback +import logging +import requests +import json +from urllib.parse import urlparse +import re +from colorama import Fore, Style +import socket + + +startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*" +endRegex = "(?:\\/[^\\s\\/]+)*\\/?" +torRegex = startRegex + "onion" + endRegex +i2pRegex = startRegex + "i2p" + endRegex +lokiRegex = startRegex + "loki" + endRegex +authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex + + +def get_cloudflare_ips(): + r = requests.get('https://www.cloudflare.com/ips-v4') + return r.text.split('\n') + + +cloudflare_ips = get_cloudflare_ips() + + +def ip2bin(ip): return "".join( + map( + str, + [ + "{0:08b}".format(int(x)) for x in ip.split(".") + ] + ) +) + + +def is_cloudflare(url): + instance_ip = None + try: + instance_ip = socket.gethostbyname(urlparse(url).hostname) + if instance_ip is None: + return False + except Exception: + return False + instance_bin = ip2bin(instance_ip) + + for cloudflare_ip_mask in cloudflare_ips: + cloudflare_ip = cloudflare_ip_mask.split('/')[0] + cloudflare_bin = ip2bin(cloudflare_ip) + + mask = int(cloudflare_ip_mask.split('/')[1]) + cloudflare_bin_masked = cloudflare_bin[:mask] + instance_bin_masked = instance_bin[:mask] + + if cloudflare_bin_masked == instance_bin_masked: + print(url + ' is behind ' + Fore.RED + + 'cloudflare' + Style.RESET_ALL) + return True + return False + + +is_cloudflare('https://pipedapi-libre.kavin.rocks') + + +def filterLastSlash(urlList): + tmp = {} + for frontend in urlList: + tmp[frontend] = {} + for network in urlList[frontend]: + tmp[frontend][network] = [] + for url in urlList[frontend][network]: + if url.endswith('/'): + tmp[frontend][network].append(url[:-1]) + print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) + else: + tmp[frontend][network].append(url) + return tmp + + +def idnaEncode(urlList): + tmp = {} + for frontend in urlList: + tmp[frontend] = {} + for network in urlList[frontend]: + tmp[frontend][network] = [] + for url in urlList[frontend][network]: + try: + encodedUrl = url.encode("idna").decode("utf8") + tmp[frontend][network].append(encodedUrl) + if (encodedUrl != url): + print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) + except Exception: + tmp[frontend][network].append(url) + return tmp + + +def fetchCache(frontend, mightyList): + try: + with open('./data.json') as file: + mightyList[frontend] = json.load(file)[frontend] + print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + frontend) + except Exception: + print(Fore.RED + 'Failed' + Style.RESET_ALL + + ' to get cached ' + frontend) + + +def fetchFromFile(frontend, mightyList): + with open('./fixed/' + frontend + '.json') as file: + mightyList[frontend] = json.load(file) + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + + +networks = {} + +with open('networks.json', 'rt') as tmp: + networks = json.load(tmp) + + +def fetchJsonList(frontend, url, urlItem, jsonObject, mightyList): + try: + r = requests.get(url) + rJson = json.loads(r.text) + if jsonObject: + rJson = rJson['instances'] + _list = {} + for network in networks: + _list[network] = [] + if type(urlItem) == dict: + for item in rJson: + for network in networks: + if urlItem[network] is not None: + if urlItem[network] in item and item[urlItem[network]] is not None: + if type(item[urlItem[network]]) == list: + for i in item[urlItem[network]]: + if i.strip() != '': + _list[network].append(i) + elif item[urlItem[network]].strip() != '': + _list[network].append(item[urlItem[network]]) + else: + for item in rJson: + tmpItem = item + if urlItem is not None: + tmpItem = item[urlItem] + if tmpItem.strip() == '': + continue + elif re.search(torRegex, tmpItem): + _list['tor'].append(tmpItem) + elif re.search(i2pRegex, tmpItem): + _list['i2p'].append(tmpItem) + elif re.search(lokiRegex, tmpItem): + _list['loki'].append(tmpItem) + else: + _list['clearnet'].append(tmpItem) + + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + except Exception: + fetchCache(frontend) + logging.error(traceback.format_exc()) + + +def fetchRegexList(frontend, url, regex, mightyList): + try: + r = requests.get(url) + _list = {} + for network in networks: + _list[network] = [] + + tmp = re.findall(regex, r.text) + + for item in tmp: + if item.strip() == "": + continue + elif re.search(torRegex, item): + _list['tor'].append(item) + elif re.search(i2pRegex, item): + _list['i2p'].append(item) + elif re.search(lokiRegex, item): + _list['loki'].append(item) + else: + _list['clearnet'].append(item) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + except Exception: + fetchCache(frontend) + logging.error(traceback.format_exc()) + + +def fetchTextList(frontend, url, prepend): + try: + _list = {} + for network in networks: + _list[network] = [] + + if type(url) == dict: + for network in networks: + if url[network] is not None: + r = requests.get(url[network]) + tmp = r.text.strip().split('\n') + for item in tmp: + item = prepend[network] + item + _list[network].append(item) + else: + r = requests.get(url) + tmp = r.text.strip().split('\n') + + for item in tmp: + item = prepend + item + if re.search(torRegex, item): + _list['tor'].append(item) + elif re.search(i2pRegex, item): + _list['i2p'].append(item) + elif re.search(lokiRegex, item): + _list['loki'].append(item) + else: + _list['clearnet'].append(item) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + except Exception: + fetchCache(frontend) + logging.error(traceback.format_exc()) + + +def isValid(url): # by avanitrachhadiya2155 + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except Exception: + return False diff --git a/services/wolfram.py b/services/wolfram.py new file mode 100644 index 0000000..a6c857e --- /dev/null +++ b/services/wolfram.py @@ -0,0 +1,33 @@ +import requests +from colorama import Fore, Style +import json + +wolfreeAlpha_url_list = [ + "https://gqq.gitlab.io", + "https://jqq.gitlab.io", + "https://rqq.gitlab.io", + "https://sqq.gitlab.io", + "https://uqq.gitlab.io" +] + + +def wolfreeAlpha(mightyList): + global wolfreeAlpha_url_list_i + frontend = 'wolfreeAlpha' + for instance in wolfreeAlpha_url_list: + try: + r = requests.get(instance+"/instances.json") + if r.status_code != 200: + continue + else: + rJson = json.loads(r.text) + networks = rJson['wolfree'] + _list = {} + for i in networks.keys(): + _list[i] = networks[i] + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + break + except: + wolfreeAlpha_url_list_i += 1 + wolfreeAlpha(wolfreeAlpha_url_list_i) \ No newline at end of file diff --git a/services/youtube.py b/services/youtube.py new file mode 100644 index 0000000..0da71d7 --- /dev/null +++ b/services/youtube.py @@ -0,0 +1,114 @@ +from .utils import fetchRegexList, fetchCache, fetchJsonList +import requests +import json +from colorama import Fore, Style +import logging +import traceback +import re + + +def invidious(mightyList): + name = 'Invidious' + frontend = 'invidious' + url = 'https://api.invidious.io/instances.json' + try: + _list = {} + _list['clearnet'] = [] + _list['tor'] = [] + _list['i2p'] = [] + _list['loki'] = [] + r = requests.get(url) + rJson = json.loads(r.text) + for instance in rJson: + if instance[1]['type'] == 'https': + _list['clearnet'].append(instance[1]['uri']) + elif instance[1]['type'] == 'onion': + _list['tor'].append(instance[1]['uri']) + elif instance[1]['type'] == 'i2p': + _list['i2p'].append(instance[1]['uri']) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, mightyList) + logging.error(traceback.format_exc()) + + +def piped(mightyList): + frontend = 'piped' + try: + _list = {} + _list['clearnet'] = [] + _list['tor'] = [] + _list['i2p'] = [] + _list['loki'] = [] + r = requests.get( + 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md') + + tmp = re.findall( + r' \| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) + for item in tmp: + try: + url = requests.get(item, timeout=5).url + if url.strip("/") == item: + continue + else: + _list['clearnet'].append(url) + except Exception: + logging.error(traceback.format_exc()) + continue + mightyList[frontend] = _list + _list['clearnet'].remove("https://piped.video") + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + except Exception: + fetchCache(frontend) + logging.error(traceback.format_exc()) + + +def materialious(mightyList): + fetchRegexList( + 'materialious', + 'https://raw.githubusercontent.com/Materialious/Materialious/main/docs/INSTANCES.md', + r"- \[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)", + mightyList + ) + + +def pipedMaterial(mightyList): + fetchRegexList( + 'pipedMaterial', + 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md', + r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production", + mightyList + ) + + +def poketube(mightyList): + frontend = 'poketube' + try: + r = requests.get( + 'https://codeberg.org/Ashley/poketube/raw/branch/main/instances.json') + rJson = json.loads(r.text) + _list = { + 'clearnet': [], + 'tor': [], + 'i2p': [], + 'loki': [] + + } + for element in rJson: + _list['clearnet'].append(element[1]['uri']) + + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend) + except Exception: + fetchCache(frontend) + logging.error(traceback.format_exc()) + + +def hyperpipe(mightyList): + fetchJsonList( + 'hyperpipe', + 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', + 'url', + False, + mightyList)