Made code readable. Fixed nitter https://github.com/libredirect/browser_extension/issues/975

.github/workflows/main.yml

@@ -14,7 +14,7 @@ jobs:
       - run: |
           python -m pip install --upgrade pip
           pip install requests colorama pyyaml
-      - run: python instances.py
+      - run: python main.py
       - run: |
           git config --local user.email "action@github.com" 
           git config --local user.name "GitHub Action"

.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

blacklist.json

@@ -1,6 +1,8 @@
 {
   "cloudflare": [
     "https://invidious.perennialte.ch",
+    "https://watch.leptons.xyz",
+    "https://piped.r4fo.com",
     "https://piped.video",
     "https://piped.coldforge.xyz",
     "https://schaun.ehwurscht.at",
@@ -8,12 +10,9 @@
     "https://tiktok.wpme.pl",
     "https://send.aurorabilisim.com",
     "https://send.cyberjake.xyz",
+    "https://nitter.lucabased.xyz",
     "https://nitter.rawbit.ninja",
-    "https://n.sneed.network",
+    "https://nitter.holo-mix.com",
-    "https://nitter.freedit.eu",
-    "https://nitter.altgr.xyz",
-    "https://jote.lile.cl",
-    "https://nitter.perennialte.ch",
     "https://redlib.freedit.eu",
     "https://redlib.frontendfriendly.xyz",
     "https://teddit.domain.glass",
@@ -39,15 +38,14 @@
     "https://hp.iqbalrifai.eu.org",
     "https://hyperpipe.darkness.services",
     "https://www.openstreetmap.org",
-    "https://paste.3server.de",
     "https://bonus01.hwb0307.com",
-    "https://privatebin.io",
     "https://private.rmconsulting.com",
-    "https://bin.oldgate.org",
+    "https://paste.3server.de",
-    "https://privatebin.luckvintage.com",
-    "https://pb.blackvoid.club",
     "https://n0t.io",
+    "https://bin.oldgate.org",
+    "https://pb.blackvoid.club",
     "https://sendpass.ru",
+    "https://privatebin.io",
     "https://wikiless.rawbit.ninja",
     "https://w.sneed.network",
     "https://biblioreads.mooo.com",

cloudflare.py

@@ -1,53 +0,0 @@
-import traceback
-import logging
-import requests
-import json
-from urllib.parse import urlparse
-import re
-from colorama import Fore, Style
-import socket
-
-def get_cloudflare_ips():
-    r = requests.get('https://www.cloudflare.com/ips-v4')
-    return r.text.split('\n')
-
-
-cloudflare_ips = get_cloudflare_ips()
-
-
-
-def ip2bin(ip): return "".join(
-    map(
-        str,
-        [
-            "{0:08b}".format(int(x)) for x in ip.split(".")
-        ]
-    )
-)
-
-
-def is_cloudflare(url):
-    instance_ip = None
-    try:
-        instance_ip = socket.gethostbyname(urlparse(url).hostname)
-        if instance_ip is None:
-            return False
-    except Exception:
-        return False
-    instance_bin = ip2bin(instance_ip)
-
-    for cloudflare_ip_mask in cloudflare_ips:
-        cloudflare_ip = cloudflare_ip_mask.split('/')[0]
-        cloudflare_bin = ip2bin(cloudflare_ip)
-
-        mask = int(cloudflare_ip_mask.split('/')[1])
-        cloudflare_bin_masked = cloudflare_bin[:mask]
-        instance_bin_masked = instance_bin[:mask]
-
-        if cloudflare_bin_masked == instance_bin_masked:
-            print(url + ' is behind ' + Fore.RED +
-                  'cloudflare' + Style.RESET_ALL)
-            return True
-    return False
-
-is_cloudflare('https://pipedapi-libre.kavin.rocks')

data.json

@@ -1,6 +1,7 @@
 {
   "invidious": {
     "clearnet": [
+      "https://inv.tux.pizza",
       "https://iv.ggtyler.dev",
       "https://inv.nadeko.net",
       "https://invidious.jing.rocks",
@@ -31,8 +32,10 @@
   },
   "piped": {
     "clearnet": [
+      "https://watch.leptons.xyz",
       "https://piped.lunar.icu",
-      "http://ww25.yapi.vyper.me/?subid1=20240820-0828-04b5-b23a-516774864bee",
+      "http://ww25.yapi.vyper.me/?subid1=20240821-0732-1439-ac1f-4980fa340f80",
+      "https://piped.r4fo.com",
       "https://piped.video",
       "https://piped.projectsegfau.lt",
       "https://piped.in.projectsegfau.lt",
@@ -50,6 +53,7 @@
       "https://piped.ngn.tf",
       "https://piped.coldforge.xyz",
       "https://piped.cdsp.cz",
+      "https://piped.phoenixthrush.com",
       "https://piped.private.coffee",
       "https://schaun.ehwurscht.at",
       "https://piped.darkness.services"
@@ -137,83 +141,23 @@
   },
   "nitter": {
     "clearnet": [
-      "https://nitter:nitter@nitter.nixnet.services",
+      "https://xcancel.com",
-      "https://nitter.fdn.fr",
-      "https://nitter.1d4.us",
-      "https://nitter.kavin.rocks",
-      "https://nitter.unixfox.eu",
-      "https://nitter.moomoo.me",
-      "https://bird.trom.tf",
-      "https://nitter.it",
-      "https://nitter.weiler.rocks",
-      "https://nitter.sethforprivacy.com",
-      "https://nitter.nl",
-      "https://nitter.mint.lgbt",
-      "https://nitter.esmailelbob.xyz",
-      "https://nitter.tiekoetter.com",
-      "https://nitter.privacy.com.de",
-      "https://nitter.poast.org",
-      "https://nitter.bird.froth.zone",
-      "https://nitter.cz",
       "https://nitter.privacydev.net",
-      "https://tweet.lambda.dance",
+      "https://nitter.poast.org",
-      "https://nitter.kylrth.com",
+      "https://nitter.lucabased.xyz",
-      "https://unofficialbird.com",
+      "https://nitter.kavin.rocks",
-      "https://nitter.projectsegfau.lt",
+      "https://nitter.tiekoetter.com",
-      "https://nitter.eu.projectsegfau.lt",
-      "https://nitter.in.projectsegfau.lt",
-      "https://singapore.unofficialbird.com",
-      "https://canada.unofficialbird.com",
-      "https://india.unofficialbird.com",
-      "https://nederland.unofficialbird.com",
-      "https://uk.unofficialbird.com",
-      "https://nitter.soopy.moe",
       "https://nitter.qwik.space",
-      "https://nitter.rawbit.ninja",
-      "https://nt.vern.cc",
-      "https://n.sneed.network",
-      "https://nitter.smnz.de",
-      "https://nitter.twei.space",
-      "https://nitter.inpt.fr",
-      "https://nitter.d420.de",
-      "https://nitter.at",
       "https://bird.habedieeh.re",
-      "https://nitter.hostux.net",
-      "https://nitter.adminforge.de",
-      "https://nitter.platypush.tech",
-      "https://nitter.us.projectsegfau.lt",
       "https://t.com.sb",
-      "https://nitter.kling.gg",
       "https://nitter.lunar.icu",
-      "https://twitter.moe.ngo",
+      "https://nitter.moomoo.me",
-      "https://nitter.freedit.eu",
+      "https://nitter.bird.froth.zone",
-      "https://nitter.tux.pizza",
+      "https://nitter.kylrth.com",
-      "https://nitter.altgr.xyz",
-      "https://jote.lile.cl",
-      "https://nitter.one",
       "https://nitter.io.lol",
-      "https://nitter.no-logs.com",
+      "https://nitter.rawbit.ninja",
-      "https://tweet.whateveritworks.org",
+      "https://n.populas.no",
-      "https://nitter.fediflix.org",
+      "https://nitter.holo-mix.com"
-      "https://nitter.nohost.network",
-      "https://nitter.simpleprivacy.fr",
-      "https://nitter.onthescent.xyz",
-      "https://nitter.x86-64-unknown-linux-gnu.zip",
-      "https://nitter.private.coffee",
-      "https://nitter.oksocial.net",
-      "https://nitter.woodland.cafe",
-      "https://nitter.dafriser.be",
-      "https://nitter.catsarch.com",
-      "https://nitter.perennialte.ch",
-      "https://nitter.laserdisc.tokyo",
-      "https://nitter.datura.network",
-      "https://nitter.salastil.com",
-      "https://nt.ggtyler.dev",
-      "https://nitter.hyperreal.coffee",
-      "https://nitter.uni-sonia.com",
-      "https://n.opnxng.com",
-      "https://nitter.ktachibana.party",
-      "https://nitter.tinfoil-hat.net"
     ],
     "tor": [
       "http://nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion",
@@ -253,7 +197,9 @@
       "http://twitter.cepyxplublbyw2f4axy4pyztfbxmf63lrt2c7uwv6wl4iixz53czload.onion",
       "http://nitter.coffee2m3bjsrrqqycx6ghkxrnejl2q6nl7pjw2j4clchjj6uk5zozad.onion",
       "http://nitter.daturab6drmkhyeia4ch5gvfc2f3wgo6bhjrv3pz6n7kxmvoznlkq4yd.onion",
-      "http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion"
+      "http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion",
+      "http://6g7w5lt6qcidh6vtue64udzld2mjnvxlpgddvn34qywxafxdv6a3w5id.onion",
+      "http://nitter.catsarchywsyuss6jdxlypsw5dc7owd5u5tr6bujxb7o6xw2hipqehyd.onion"
     ],
     "i2p": [
       "http://axd6uavsstsrvstva4mzlzh4ct76rc6zdug3nxdgeitrzczhzf4q.b32.i2p",
@@ -265,9 +211,7 @@
       "http://4kzgzonccxuv4ctforjqx565jgd4bocgnzsy3dg4fmdf3ye2iciq.b32.i2p",
       "http://pjsfs4ukb6prmfx3qx3a5ef2cpcupkvcrxdh72kqn2rxc2cw4nka.b32.i2p"
     ],
-    "loki": [
+    "loki": []
-      "http://nitter.priv.loki"
-    ]
   },
   "libreddit": {
     "clearnet": [
@@ -617,7 +561,6 @@
       "https://xo.wtf"
     ],
     "tor": [
-      "http://lbmegc3rjnekmdxuisynqdc7y3m2tgyq7gj257ooddaobxqjw36bdayd.onion",
       "http://4n53nafyi77iplnbrpmxnp3x4exbswwxigujaxy3b37fvr7bvlopxeyd.onion",
       "http://whereissky27lxfdrs7ct7a5ievor3tl67qi77rv5luga7wod5jf2mad.onion",
       "http://searchpie7fejn4wctzm6fgm6oz5vnziltfzzuec6b4qmkr3wrqaoqqd.onion",
@@ -832,14 +775,6 @@
     "i2p": [],
     "loki": []
   },
-  "facil": {
-    "clearnet": [
-      "https://facilmap.org"
-    ],
-    "tor": [],
-    "i2p": [],
-    "loki": []
-  },
   "osm": {
     "clearnet": [
       "https://www.openstreetmap.org"
@@ -905,49 +840,47 @@
   },
   "privateBin": {
     "clearnet": [
-      "https://secret.hummel-web.at",
-      "https://bin.0xfc.de",
-      "https://paste.3server.de",
-      "https://paste.tuxcloud.net",
-      "https://snip.dssr.ch",
-      "https://paste.ononoki.org",
-      "https://p.vinci-concessions.com",
-      "https://extrait.facil.services",
       "https://bonus01.hwb0307.com",
+      "https://extrait.facil.services",
+      "https://paste.momobako.com",
+      "https://paste.waays.fr",
+      "https://notebin.de",
+      "https://vip.ci",
       "https://privatebin.wsmm.de",
-      "https://paste.devsite.pl",
+      "https://secret.hummel-web.at",
-      "https://onetimesecret.trans-ix.nl",
-      "https://secrets.l25.cloud",
       "https://p.darklab.sh",
-      "https://privatebin.io",
+      "https://bin.bus-hit.me",
-      "https://paste.hackliberty.org",
+      "https://paste.devsite.pl",
-      "https://0.jaegers.net",
+      "https://ots.kocsar.at",
+      "https://onetimesecret.trans-ix.nl",
       "https://private.rmconsulting.com",
       "https://zerobin.thican.net",
-      "https://bin.nixnet.services",
-      "https://paste.momobako.com",
-      "https://ots.kocsar.at",
-      "https://privatebin.net",
-      "https://vip.ci",
-      "https://bin.oldgate.org",
-      "https://paste.waays.fr",
-      "https://0.0g.gg",
-      "https://privatebin.luckvintage.com",
-      "https://paste.encrypt.co.il",
-      "https://paste.skynetcloud.site",
-      "https://paste.techbag.app",
-      "https://pb.blackvoid.club",
-      "https://privatebin.arch-linux.cz",
-      "https://paste.tildeverse.org",
-      "https://n0t.io",
-      "https://sendpass.ru",
       "https://secrets.true.nl",
-      "https://notebin.de",
+      "https://paste.tuxcloud.net",
-      "https://paste.hostux.net",
+      "https://paste.3server.de",
-      "https://bin.ngn.tf",
+      "https://n0t.io",
-      "https://encryp.ch/note",
+      "https://bin.oldgate.org",
+      "https://paste.techbag.app",
+      "https://bin.nixnet.services",
+      "https://privatebin.arch-linux.cz",
       "https://paste.ottertelecom.com",
-      "https://bin.bus-hit.me"
+      "https://paste.encrypt.co.il",
+      "https://encryp.ch/note",
+      "https://bin.ngn.tf",
+      "https://bin.0xfc.de",
+      "https://pb.blackvoid.club",
+      "https://paste.tildeverse.org",
+      "https://paste.ononoki.org",
+      "https://sendpass.ru",
+      "https://snip.dssr.ch",
+      "https://privatebin.net",
+      "https://paste.hostux.net",
+      "https://privatebin.io",
+      "https://paste.hackliberty.org",
+      "https://p.vinci-concessions.com",
+      "https://0.jaegers.net",
+      "https://secrets.l25.cloud",
+      "https://privatebin.xi.ht"
     ],
     "tor": [],
     "i2p": [],

fixed/facil.json

@@ -1,6 +0,0 @@
-{
-	"clearnet": ["https://facilmap.org"],
-	"tor": [],
-	"i2p": [],
-	"loki": []
-}

instances.py

@@ -1,745 +0,0 @@
-#!/usr/bin/python3
-
-import traceback
-import logging
-import requests
-import json
-from urllib.parse import urlparse
-import re
-from colorama import Fore, Style
-import socket
-import yaml
-
-mightyList = {}
-networks = {}
-
-startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*"
-endRegex = "(?:\/[^\s\/]+)*\/?"
-torRegex = startRegex + "onion" + endRegex
-i2pRegex = startRegex + "i2p" + endRegex
-lokiRegex = startRegex + "loki" + endRegex
-authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex
-
-with open('networks.json', 'rt') as tmp:
-    networks = json.load(tmp)
-
-
-def filterLastSlash(urlList):
-    tmp = {}
-    for frontend in urlList:
-        tmp[frontend] = {}
-        for network in urlList[frontend]:
-            tmp[frontend][network] = []
-            for url in urlList[frontend][network]:
-                if url.endswith('/'):
-                    tmp[frontend][network].append(url[:-1])
-                    print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
-                else:
-                    tmp[frontend][network].append(url)
-    return tmp
-
-
-def idnaEncode(urlList):
-    tmp = {}
-    for frontend in urlList:
-        tmp[frontend] = {}
-        for network in urlList[frontend]:
-            tmp[frontend][network] = []
-            for url in urlList[frontend][network]:
-                try:
-                    encodedUrl = url.encode("idna").decode("utf8")
-                    tmp[frontend][network].append(encodedUrl)
-                    if (encodedUrl != url):
-                        print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
-                except Exception:
-                    tmp[frontend][network].append(url)
-    return tmp
-
-
-def ip2bin(ip): return "".join(
-    map(
-        str,
-        [
-            "{0:08b}".format(int(x)) for x in ip.split(".")
-        ]
-    )
-)
-
-
-def get_cloudflare_ips():
-    r = requests.get('https://www.cloudflare.com/ips-v4')
-    return r.text.split('\n')
-
-
-cloudflare_ips = get_cloudflare_ips()
-
-
-def is_cloudflare(url):
-    instance_ip = None
-    try:
-        instance_ip = socket.gethostbyname(urlparse(url).hostname)
-        if instance_ip is None:
-            return False
-    except Exception:
-        return False
-    instance_bin = ip2bin(instance_ip)
-
-    for cloudflare_ip_mask in cloudflare_ips:
-        cloudflare_ip = cloudflare_ip_mask.split('/')[0]
-        cloudflare_bin = ip2bin(cloudflare_ip)
-
-        mask = int(cloudflare_ip_mask.split('/')[1])
-        cloudflare_bin_masked = cloudflare_bin[:mask]
-        instance_bin_masked = instance_bin[:mask]
-
-        if cloudflare_bin_masked == instance_bin_masked:
-            print(url + ' is behind ' + Fore.RED +
-                  'cloudflare' + Style.RESET_ALL)
-            return True
-    return False
-
-
-def fetchCache(frontend):
-    try:
-        with open('./data.json') as file:
-            mightyList[frontend] = json.load(file)[frontend]
-        print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + frontend)
-    except Exception:
-        print(Fore.RED + 'Failed' + Style.RESET_ALL +
-              ' to get cached ' + frontend)
-
-
-def fetchFromFile(frontend):
-    with open('./fixed/' + frontend + '.json') as file:
-        mightyList[frontend] = json.load(file)
-    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-
-
-def fetchJsonList(frontend, url, urlItem, jsonObject):
-    try:
-        r = requests.get(url)
-        rJson = json.loads(r.text)
-        if jsonObject:
-            rJson = rJson['instances']
-        _list = {}
-        for network in networks:
-            _list[network] = []
-        if type(urlItem) == dict:
-            for item in rJson:
-                for network in networks:
-                    if urlItem[network] is not None:
-                        if urlItem[network] in item and item[urlItem[network]] is not None:
-                            if type(item[urlItem[network]]) == list:
-                                for i in item[urlItem[network]]:
-                                    if i.strip() != '':
-                                        _list[network].append(i)
-                            elif item[urlItem[network]].strip() != '':
-                                _list[network].append(item[urlItem[network]])
-        else:
-            for item in rJson:
-                tmpItem = item
-                if urlItem is not None:
-                    tmpItem = item[urlItem]
-                if tmpItem.strip() == '':
-                    continue
-                elif re.search(torRegex, tmpItem):
-                    _list['tor'].append(tmpItem)
-                elif re.search(i2pRegex, tmpItem):
-                    _list['i2p'].append(tmpItem)
-                elif re.search(lokiRegex, tmpItem):
-                    _list['loki'].append(tmpItem)
-                else:
-                    _list['clearnet'].append(tmpItem)
-
-        mightyList[frontend] = _list
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-    except Exception:
-        fetchCache(frontend)
-        logging.error(traceback.format_exc())
-
-
-def fetchRegexList(frontend, url, regex):
-    try:
-        r = requests.get(url)
-        _list = {}
-        for network in networks:
-            _list[network] = []
-
-        tmp = re.findall(regex, r.text)
-
-        for item in tmp:
-            if item.strip() == "":
-                continue
-            elif re.search(torRegex, item):
-                _list['tor'].append(item)
-            elif re.search(i2pRegex, item):
-                _list['i2p'].append(item)
-            elif re.search(lokiRegex, item):
-                _list['loki'].append(item)
-            else:
-                _list['clearnet'].append(item)
-        mightyList[frontend] = _list
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-    except Exception:
-        fetchCache(frontend)
-        logging.error(traceback.format_exc())
-
-
-def fetchTextList(frontend, url, prepend):
-    try:
-        _list = {}
-        for network in networks:
-            _list[network] = []
-
-        if type(url) == dict:
-            for network in networks:
-                if url[network] is not None:
-                    r = requests.get(url[network])
-                    tmp = r.text.strip().split('\n')
-                    for item in tmp:
-                        item = prepend[network] + item
-                        _list[network].append(item)
-        else:
-            r = requests.get(url)
-            tmp = r.text.strip().split('\n')
-
-            for item in tmp:
-                item = prepend + item
-                if re.search(torRegex, item):
-                    _list['tor'].append(item)
-                elif re.search(i2pRegex, item):
-                    _list['i2p'].append(item)
-                elif re.search(lokiRegex, item):
-                    _list['loki'].append(item)
-                else:
-                    _list['clearnet'].append(item)
-        mightyList[frontend] = _list
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-    except Exception:
-        fetchCache(frontend)
-        logging.error(traceback.format_exc())
-
-
-def invidious():
-    name = 'Invidious'
-    frontend = 'invidious'
-    url = 'https://api.invidious.io/instances.json'
-    try:
-        _list = {}
-        _list['clearnet'] = []
-        _list['tor'] = []
-        _list['i2p'] = []
-        _list['loki'] = []
-        r = requests.get(url)
-        rJson = json.loads(r.text)
-        for instance in rJson:
-            if instance[1]['type'] == 'https':
-                _list['clearnet'].append(instance[1]['uri'])
-            elif instance[1]['type'] == 'onion':
-                _list['tor'].append(instance[1]['uri'])
-            elif instance[1]['type'] == 'i2p':
-                _list['i2p'].append(instance[1]['uri'])
-        mightyList[frontend] = _list
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name)
-    except Exception:
-        fetchCache(frontend)
-        logging.error(traceback.format_exc())
-
-
-def piped():
-    frontend = 'piped'
-    try:
-        _list = {}
-        _list['clearnet'] = []
-        _list['tor'] = []
-        _list['i2p'] = []
-        _list['loki'] = []
-        r = requests.get(
-            'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md')
-
-        tmp = re.findall(
-            r' \| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text)
-        for item in tmp:
-            try:
-                url = requests.get(item, timeout=5).url
-                if url.strip("/") == item:
-                    continue
-                else:
-                    _list['clearnet'].append(url)
-            except Exception:
-                logging.error(traceback.format_exc())
-                continue
-        mightyList[frontend] = _list
-        _list['clearnet'].remove("https://piped.video")
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-    except Exception:
-        fetchCache(frontend)
-        logging.error(traceback.format_exc())
-
-
-def materialious():
-    fetchRegexList('materialious', 'https://raw.githubusercontent.com/Materialious/Materialious/main/docs/INSTANCES.md',
-                    r"- \[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)")
-
-
-def pipedMaterial():
-    fetchRegexList('pipedMaterial', 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md',
-                   r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production")
-
-
-def cloudtube():
-    fetchFromFile('cloudtube')
-
-
-def proxitok():
-    fetchRegexList('proxiTok', 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md',
-                   r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))")
-
-
-def send():
-    fetchRegexList('send', 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md',
-                   r"(https.*?) \|.*?\n")
-
-
-def nitter():
-    frontend = 'nitter'
-    try:
-        r = requests.get(
-            'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md')
-        _list = {}
-
-        _list['clearnet'] = []
-        public = re.findall(r"## Public((?:\n|.*)+?)##", r.text)
-        for line in public[0].split('\n'):
-            result = re.findall(r"^\| \[.*?\]\((https.*?)\)", line)
-            if len(result) > 0:
-                _list['clearnet'].append(result[0])
-
-        _list['tor'] = []
-        public = re.findall(r"## Tor((?:\n|.*)+?)##", r.text)
-        for line in public[0].split('\n'):
-            result = re.findall(r"^\| <(http.*?)\/?>", line)
-            if len(result) > 0:
-                _list['tor'].append(result[0])
-
-        _list['i2p'] = []
-        public = re.findall(r"## I2P((?:\n|.*)+?)##", r.text)
-        for line in public[0].split('\n'):
-            result = re.findall(r"^- <(http.*?)\/?>", line)
-            if len(result) > 0:
-                _list['i2p'].append(result[0])
-
-        _list['loki'] = []
-        public = re.findall(r"## Lokinet((?:\n|.*)+?)##", r.text)
-        for line in public[0].split('\n'):
-            result = re.findall(r"^- <(http.*?)\/?>", line)
-            if len(result) > 0:
-                _list['loki'].append(result[0])
-
-        mightyList[frontend] = _list
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-
-    except Exception:
-        fetchCache(frontend)
-        logging.error(traceback.format_exc())
-
-    # fetchRegexList('nitter', '',
-    #                r"(?:(?:\| )|(?:-   ))\[(?:(?:\S+\.)+[a-zA-Z0-9]+)\/?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]+)\/?\)(?:(?: (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,4} +\|(?:(?:\n)|(?: ❌)|(?: ✅)|(?: ❓)|(?: \[)))|(?:\n))")
-
-
-def libreddit():
-    fetchJsonList('libreddit', 'https://github.com/libreddit/libreddit-instances/raw/master/instances.json',
-                  {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True)
-
-
-def redlib():
-    fetchJsonList('redlib', 'https://github.com/redlib-org/redlib-instances/raw/main/instances.json',
-                  {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True)
-
-
-def teddit():
-    fetchJsonList('teddit', 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json',
-                  {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False)
-
-
-def scribe():
-    fetchJsonList(
-        'scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', None, False)
-
-
-def quetre():
-    fetchJsonList('quetre', 'https://raw.githubusercontent.com/zyachel/quetre/main/instances.json',
-                  {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False)
-
-
-def libremdb():
-    fetchJsonList('libremdb', 'https://raw.githubusercontent.com/zyachel/libremdb/main/instances.json',
-                  {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False)
-
-
-def simplytranslate():
-    fetchFromFile('simplyTranslate')
-
-
-def linvgatranslate():
-    fetchJsonList(
-        'lingva', 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False)
-
-
-def searxng():
-    r = requests.get(
-        'https://searx.space/data/instances.json')
-    rJson = json.loads(r.text)
-    searxngList = {}
-    searxngList['clearnet'] = []
-    searxngList['tor'] = []
-    searxngList['i2p'] = []
-    searxngList['loki'] = []
-
-    for item in rJson['instances']:
-        if re.search(torRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
-            searxngList['tor'].append(item[:-1])
-        elif re.search(i2pRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
-            searxngList['i2p'].append(item[:-1])
-        elif rJson['instances'][item].get('generator') == 'searxng':
-            searxngList['clearnet'].append(item[:-1])
-
-    mightyList['searxng'] = searxngList
-    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
-
-
-def searx():
-    searxList = {}
-    searxList['clearnet'] = []
-    searxList['tor'] = []
-    searxList['i2p'] = []
-    searxList['loki'] = []
-    r = requests.get(
-        'https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml')
-    data = yaml.safe_load(r.text)
-    for key in data:
-        searxList['clearnet'].append(key)
-        if 'additional_urls' in data[key]:
-            for additional_url in data[key]['additional_urls']:
-                if data[key]['additional_urls'][additional_url] == "Hidden Service":
-                    searxList['tor'].append(additional_url)
-    mightyList['searx'] = searxList
-    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
-
-
-def whoogle():
-    fetchRegexList('whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md',
-                   r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ")
-
-
-def librex():
-    fetchJsonList('librex', 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json',
-                  {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, True)
-
-
-def rimgo():
-    try:
-        r = requests.get('https://rimgo.codeberg.page/api.json')
-        rJson = json.loads(r.text)
-        _list = {
-            'clearnet': [],
-            'tor': [],
-        }
-        for instance in rJson['clearnet']:
-            _list['clearnet'].append(instance['url'])
-
-        for instance in rJson['tor']:
-            _list['tor'].append(instance['url'])
-
-        mightyList['rimgo'] = _list
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'rimgo')
-    except Exception:
-        fetchCache('rimgo')
-        logging.error(traceback.format_exc())
-
-
-def pixivFe():
-    fetchRegexList('pixivFe', 'https://codeberg.org/VnPower/pixivfe/raw/branch/main/README.md',
-                   r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/? +\|")
-
-
-def safetwitch():
-    fetchRegexList('safetwitch', 'https://codeberg.org/dragongoose/safetwitch/raw/branch/master/README.md',
-                 re.compile(r"^\| \[.*?\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)",re.MULTILINE))
-
-
-def hyperpipe():
-    fetchJsonList(
-        'hyperpipe', 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', 'url', False)
-
-
-def facil():
-    fetchFromFile('facil')
-
-
-def osm():
-    fetchFromFile('osm')
-
-
-def libreTranslate():
-    fetchRegexList('libreTranslate', 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md',
-                   r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|")
-
-
-def breezeWiki():
-    fetchJsonList(
-        'breezeWiki', 'https://docs.breezewiki.com/files/instances.json', 'instance', False)
-
-
-def binternet():
-    fetchRegexList('binternet', 'https://raw.githubusercontent.com/Ahwxorg/Binternet/main/README.md',
-                   r"\| \[[\w\.]+!?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/?\)")
-
-
-def privateBin():
-    fetchJsonList('privateBin', 'https://privatebin.info/directory/api?top=100&https_redirect=true&min_rating=A&csp_header=true&min_uptime=100&attachments=true', 'url', False)
-
-
-def neuters():
-    fetchFromFile('neuters')
-
-
-def libMedium():
-    fetchRegexList('libMedium', 'https://raw.githubusercontent.com/realaravinth/libmedium/master/README.md',
-                   r"\| (https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/? +\|")
-
-
-def dumb():
-    fetchJsonList('dumb', 'https://raw.githubusercontent.com/rramiachraf/dumb/main/instances.json',
-                  {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, False)
-
-
-def ruralDictionary():
-    fetchJsonList('ruralDictionary',
-                  'https://codeberg.org/zortazert/rural-dictionary/raw/branch/master/instances.json',
-                  {'clearnet': 'clearnet', 'tor': 'tor',
-                      'i2p': 'i2p', 'loki': None},
-                  False
-                  )
-
-
-def anonymousOverflow():
-    fetchRegexList('anonymousOverflow', 'https://raw.githubusercontent.com/httpjamesm/AnonymousOverflow/main/README.md',
-                   r"\| \[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) +\|")
-
-
-def wikiless():
-    fetchFromFile('wikiless')
-
-
-def biblioReads():
-    fetchJsonList(
-        'biblioReads',
-        'https://raw.githubusercontent.com/nesaku/BiblioReads/main/instances.json',
-        {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None},
-        False
-    )
-
-
-def suds():
-    fetchJsonList(
-        'suds', 'https://git.vern.cc/cobra/Suds/raw/branch/main/instances.json',
-        {
-            'clearnet': 'clearnet',
-            'tor': 'tor',
-            'i2p': 'i2p',
-            'loki': None
-        },
-        False,
-    )
-
-
-def poketube():
-    frontend = 'poketube'
-    try:
-        r = requests.get(
-            'https://codeberg.org/Ashley/poketube/raw/branch/main/instances.json')
-        rJson = json.loads(r.text)
-        _list = {
-            'clearnet': [],
-            'tor': [],
-            'i2p': [],
-            'loki': []
-
-        }
-        for element in rJson:
-            _list['clearnet'].append(element[1]['uri'])
-
-        mightyList[frontend] = _list
-        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-    except Exception:
-        fetchCache(frontend)
-        logging.error(traceback.format_exc())
-
-
-def gothub():
-    fetchJsonList(
-        'gothub',
-        'https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json',
-        {
-            'clearnet': 'link',
-            'tor': None,
-            'i2p': None,
-            'loki': None
-        },
-        False
-    )
-
-
-def mikuInvidious():
-    fetchFromFile('mikuInvidious')
-
-
-def tent():
-    fetchJsonList('tent', 'https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json', 'url', False)
-
-
-def laboratory():
-    fetchRegexList('laboratory', 'https://git.vitali64.duckdns.org/utils/laboratory.git/plain/README.md',
-                   r"\| (https:\/{2}.*?) \|")
-
-
-wolfreeAlpha_url_list = [
-    "https://gqq.gitlab.io",
-    "https://jqq.gitlab.io",
-    "https://rqq.gitlab.io",
-    "https://sqq.gitlab.io",
-    "https://uqq.gitlab.io"
-]
-wolfreeAlpha_url_list_i = 0
-
-
-def wolfreeAlpha(i):
-    global wolfreeAlpha_url_list_i
-    frontend = 'wolfreeAlpha'
-    try:
-        r = requests.get(wolfreeAlpha_url_list[i]+"/instances.json")
-        if r.status_code != 200:
-            wolfreeAlpha_url_list_i += 1
-            wolfreeAlpha(wolfreeAlpha_url_list_i)
-        else:
-            rJson = json.loads(r.text)
-            networks = rJson['wolfree']
-            _list = {}
-            for i in networks.keys():
-                _list[i] = networks[i]
-            mightyList[frontend] = _list
-            print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
-    except:
-        wolfreeAlpha_url_list_i += 1
-        wolfreeAlpha(wolfreeAlpha_url_list_i)
-
-
-def jiti():
-    fetchRegexList('jitsi', "https://raw.githubusercontent.com/jitsi/handbook/master/docs/community/instances.md",
-                   r"\|(?:(?: |	)+)+((?:[a-z]+\.)+[a-z]+)(?:(?: |	)+)+\|")
-    mightyList['jitsi']['clearnet'] = list(
-        map(lambda x: "https://"+x, mightyList['jitsi']['clearnet']))
-    mightyList['jitsi']['clearnet'].insert(0, 'https://8x8.vc')
-    mightyList['jitsi']['clearnet'].insert(0, 'https://meet.jit.si')
-
-
-def proxigram():
-    fetchRegexList('proxigram', 'https://codeberg.org/ThePenguinDev/Proxigram/wiki/raw/Instances.md',
-                   r"\[(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\]")
-
-
-def twineo():
-    fetchFromFile('twineo')
-
-
-def priviblur():
-    fetchRegexList('priviblur', 'https://raw.githubusercontent.com/syeopite/priviblur/master/instances.md',
-                   r"\| ?\[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\) ?|")
-
-
-def isValid(url):  # This code is contributed by avanitrachhadiya2155
-    try:
-        result = urlparse(url)
-        return all([result.scheme, result.netloc])
-    except Exception:
-        return False
-
-
-invidious()
-materialious()
-piped()
-pipedMaterial()
-cloudtube()
-proxitok()
-send()
-nitter()
-libreddit()
-redlib()
-teddit()
-scribe()
-quetre()
-libremdb()
-simplytranslate()
-linvgatranslate()
-libreTranslate()
-searxng()
-searx()
-whoogle()
-librex()
-rimgo()
-pixivFe()
-safetwitch()
-hyperpipe()
-facil()
-osm()
-breezeWiki()
-binternet()
-privateBin()
-neuters()
-ruralDictionary()
-libMedium()
-dumb()
-anonymousOverflow()
-wikiless()
-biblioReads()
-suds()
-poketube()
-gothub()
-mikuInvidious()
-wolfreeAlpha(wolfreeAlpha_url_list_i)
-jiti()
-proxigram()
-tent()
-laboratory()
-twineo()
-priviblur()
-
-mightyList = filterLastSlash(mightyList)
-mightyList = idnaEncode(mightyList)
-
-cloudflare = []
-for k1, v1 in mightyList.items():
-    if type(mightyList[k1]) is dict:
-        for k2, v2 in mightyList[k1].items():
-            for instance in mightyList[k1][k2]:
-                if (not isValid(instance)):
-                    mightyList[k1][k2].remove(instance)
-                    print("removed " + instance)
-                else:
-                    if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance):
-                        cloudflare.append(instance)
-blacklist = {
-    'cloudflare': cloudflare
-}
-
-# Writing to file
-json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
-with open('./data.json', 'w') as outfile:
-    outfile.write(json_object)
-print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json')
-
-json_object = json.dumps(blacklist, ensure_ascii=False, indent=2)
-with open('./blacklist.json', 'w') as outfile:
-    outfile.write(json_object)
-print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json')

main.py

@@ -0,0 +1,89 @@
+import json
+from colorama import Fore, Style
+from services.from_file import *
+from services.medium import *
+from services.other import *
+from services.reddit import *
+from services.search import *
+from services.translate import *
+from services.utils import *
+from services.wolfram import *
+from services.youtube import *
+
+mightyList = {}
+
+invidious(mightyList)
+materialious(mightyList)
+piped(mightyList)
+pipedMaterial(mightyList)
+cloudtube(mightyList)
+proxitok(mightyList)
+send(mightyList)
+nitter(mightyList)
+libreddit(mightyList)
+redlib(mightyList)
+teddit(mightyList)
+scribe(mightyList)
+quetre(mightyList)
+libremdb(mightyList)
+simplytranslate(mightyList)
+linvgatranslate(mightyList)
+libreTranslate(mightyList)
+searxng(mightyList)
+searx(mightyList)
+whoogle(mightyList)
+librex(mightyList)
+rimgo(mightyList)
+pixivFe(mightyList)
+safetwitch(mightyList)
+hyperpipe(mightyList)
+osm(mightyList)
+breezeWiki(mightyList)
+binternet(mightyList)
+privateBin(mightyList)
+neuters(mightyList)
+ruralDictionary(mightyList)
+libMedium(mightyList)
+dumb(mightyList)
+anonymousOverflow(mightyList)
+wikiless(mightyList)
+biblioReads(mightyList)
+suds(mightyList)
+poketube(mightyList)
+gothub(mightyList)
+mikuInvidious(mightyList)
+wolfreeAlpha(mightyList)
+jiti(mightyList)
+proxigram(mightyList)
+tent(mightyList)
+laboratory(mightyList)
+twineo(mightyList)
+priviblur(mightyList)
+
+mightyList = filterLastSlash(mightyList)
+mightyList = idnaEncode(mightyList)
+
+cloudflare = []
+for k1, v1 in mightyList.items():
+    if type(mightyList[k1]) is dict:
+        for k2, v2 in mightyList[k1].items():
+            for instance in mightyList[k1][k2]:
+                if not isValid(instance):
+                    mightyList[k1][k2].remove(instance)
+                    print("removed " + instance)
+                else:
+                    if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance):
+                        cloudflare.append(instance)
+blacklist = {
+    'cloudflare': cloudflare
+}
+
+json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
+with open('./data.json', 'w') as outfile:
+    outfile.write(json_object)
+print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json')
+
+json_object = json.dumps(blacklist, ensure_ascii=False, indent=2)
+with open('./blacklist.json', 'w') as outfile:
+    outfile.write(json_object)
+print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json')

requirements.txt

@@ -0,0 +1,3 @@
+requests
+colorama
+pyyaml

services/from_file.py

@@ -0,0 +1,29 @@
+from .utils import fetchFromFile
+
+
+def osm(mightyList):
+    fetchFromFile('osm', mightyList)
+
+
+def neuters(mightyList):
+    fetchFromFile('neuters', mightyList)
+
+
+def wikiless(mightyList):
+    fetchFromFile('wikiless', mightyList)
+
+
+def mikuInvidious(mightyList):
+    fetchFromFile('mikuInvidious', mightyList)
+
+
+def twineo(mightyList):
+    fetchFromFile('twineo', mightyList)
+
+
+def cloudtube(mightyList):
+    fetchFromFile('cloudtube', mightyList)
+
+
+def simplytranslate(mightyList):
+    fetchFromFile('simplyTranslate', mightyList)

services/medium.py

@@ -0,0 +1,19 @@
+from .utils import fetchJsonList, fetchRegexList
+
+
+def scribe(mightyList):
+    fetchJsonList(
+        'scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json',
+        None,
+        False,
+        mightyList
+    )
+
+
+def libMedium(mightyList):
+    fetchRegexList(
+        'libMedium',
+        'https://raw.githubusercontent.com/realaravinth/libmedium/master/README.md',
+        r"\| (https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/? +\|",
+        mightyList
+    )

services/other.py

@@ -0,0 +1,310 @@
+import requests
+import json
+from .utils import fetchJsonList, fetchRegexList, fetchCache
+from colorama import Fore, Style
+import logging
+import traceback
+import re
+
+
+def tent(mightyList):
+    fetchJsonList(
+        'tent',
+        'https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json',
+        'url',
+        False,
+        mightyList
+    )
+
+
+def ruralDictionary(mightyList):
+    fetchJsonList(
+        'ruralDictionary',
+        'https://codeberg.org/zortazert/rural-dictionary/raw/branch/master/instances.json',
+        {
+            'clearnet': 'clearnet',
+            'tor': 'tor',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        False,
+        mightyList
+    )
+
+
+def laboratory(mightyList):
+    fetchRegexList(
+        'laboratory',
+        'https://git.vitali64.duckdns.org/utils/laboratory.git/plain/README.md',
+        r"\| (https:\/{2}.*?) \|",
+        mightyList
+    )
+
+
+def gothub(mightyList):
+    fetchJsonList(
+        'gothub',
+        'https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json',
+        {
+            'clearnet': 'link',
+            'tor': None,
+            'i2p': None,
+            'loki': None
+        },
+        False,
+        mightyList
+    )
+
+
+def biblioReads(mightyList):
+    fetchJsonList(
+        'biblioReads',
+        'https://raw.githubusercontent.com/nesaku/BiblioReads/main/instances.json',
+        {
+            'clearnet': 'url',
+            'tor': 'onion',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        False,
+        mightyList
+    )
+
+
+def libremdb(mightyList):
+    fetchJsonList(
+        'libremdb',
+        'https://raw.githubusercontent.com/zyachel/libremdb/main/instances.json',
+        {
+            'clearnet': 'clearnet',
+            'tor': 'tor',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        False,
+        mightyList
+    )
+
+
+def breezeWiki(mightyList):
+    fetchJsonList(
+        'breezeWiki',
+        'https://docs.breezewiki.com/files/instances.json',
+        'instance',
+        False,
+        mightyList
+    )
+
+
+def binternet(mightyList):
+    fetchRegexList(
+        'binternet',
+        'https://raw.githubusercontent.com/Ahwxorg/Binternet/main/README.md',
+        r"\| \[[\w\.]+!?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/?\)",
+        mightyList
+    )
+
+
+def dumb(mightyList):
+    fetchJsonList(
+        'dumb',
+        'https://raw.githubusercontent.com/rramiachraf/dumb/main/instances.json',
+        {
+            'clearnet': 'clearnet',
+            'tor': 'tor',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        False,
+        mightyList
+    )
+
+
+def suds(mightyList):
+    fetchJsonList(
+        'suds', 'https://git.vern.cc/cobra/Suds/raw/branch/main/instances.json',
+        {
+            'clearnet': 'clearnet',
+            'tor': 'tor',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        False,
+        mightyList
+    )
+
+
+def proxigram(mightyList):
+    fetchRegexList(
+        'proxigram',
+        'https://codeberg.org/ThePenguinDev/Proxigram/wiki/raw/Instances.md',
+        r"\[(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\]",
+        mightyList
+    )
+
+
+def rimgo(mightyList):
+    try:
+        r = requests.get('https://rimgo.codeberg.page/api.json')
+        rJson = json.loads(r.text)
+        _list = {
+            'clearnet': [],
+            'tor': [],
+        }
+        for instance in rJson['clearnet']:
+            _list['clearnet'].append(instance['url'])
+
+        for instance in rJson['tor']:
+            _list['tor'].append(instance['url'])
+
+        mightyList['rimgo'] = _list
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'rimgo')
+    except Exception:
+        fetchCache('rimgo')
+        logging.error(traceback.format_exc())
+
+
+def jiti(mightyList):
+    fetchRegexList(
+        'jitsi',
+        "https://raw.githubusercontent.com/jitsi/handbook/master/docs/community/instances.md",
+        r"\|(?:(?: |	)+)+((?:[a-z]+\.)+[a-z]+)(?:(?: |	)+)+\|",
+        mightyList
+    )
+    mightyList['jitsi']['clearnet'] = list(
+        map(
+            lambda x: "https://"+x,
+            mightyList['jitsi']['clearnet']
+        )
+    )
+    mightyList['jitsi']['clearnet'].insert(0, 'https://8x8.vc')
+    mightyList['jitsi']['clearnet'].insert(0, 'https://meet.jit.si')
+
+
+def anonymousOverflow(mightyList):
+    fetchRegexList(
+        'anonymousOverflow',
+        'https://raw.githubusercontent.com/httpjamesm/AnonymousOverflow/main/README.md',
+        r"\| \[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) +\|",
+        mightyList
+    )
+
+
+def proxitok(mightyList):
+    fetchRegexList(
+        'proxiTok',
+        'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md',
+        r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))",
+        mightyList
+    )
+
+
+def priviblur(mightyList):
+    fetchRegexList(
+        'priviblur',
+        'https://raw.githubusercontent.com/syeopite/priviblur/master/instances.md',
+        r"\| ?\[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\) ?|",
+        mightyList
+    )
+
+
+def safetwitch(mightyList):
+    fetchRegexList(
+        'safetwitch',
+        'https://codeberg.org/dragongoose/safetwitch/raw/branch/master/README.md',
+        re.compile(
+            r"^\| \[.*?\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)",
+            re.MULTILINE
+        ),
+        mightyList
+    )
+
+
+def nitter(mightyList):
+    frontend = 'nitter'
+    try:
+        r = requests.get(
+            'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md')
+        _list = {}
+
+        _list['clearnet'] = []
+        public = re.findall(r"## Public((?:\n|.*)+?)##", r.text)
+        if public:
+            for line in public[0].split('\n'):
+                result = re.findall(r"^\| \[.*?\]\((https.*?)\)", line)
+                if len(result) > 0:
+                    _list['clearnet'].append(result[0])
+
+        _list['tor'] = []
+        public = re.findall(r"## Tor((?:\n|.*)+?)##", r.text)
+        if public:
+            for line in public[0].split('\n'):
+                result = re.findall(r"^\| <(http.*?)\/?>", line)
+                if len(result) > 0:
+                    _list['tor'].append(result[0])
+
+        _list['i2p'] = []
+        public = re.findall(r"## I2P((?:\n|.*)+?)##", r.text)
+        if public:
+            for line in public[0].split('\n'):
+                result = re.findall(r"^- <(http.*?)\/?>", line)
+                if len(result) > 0:
+                    _list['i2p'].append(result[0])
+
+        _list['loki'] = []
+        public = re.findall(r"## Lokinet((?:\n|.*)+?)##", r.text)
+        if public:
+            for line in public[0].split('\n'):
+                result = re.findall(r"^- <(http.*?)\/?>", line)
+                if len(result) > 0:
+                    _list['loki'].append(result[0])
+
+        mightyList[frontend] = _list
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+
+    except Exception:
+        fetchCache(frontend, mightyList)
+        logging.error(traceback.format_exc())
+
+
+def send(mightyList):
+    fetchRegexList(
+        'send',
+        'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md',
+        r"(https.*?) \|.*?\n",
+        mightyList
+    )
+
+
+def quetre(mightyList):
+    fetchJsonList(
+        'quetre',
+        'https://raw.githubusercontent.com/zyachel/quetre/main/instances.json',
+        {
+            'clearnet': 'clearnet',
+            'tor': 'tor',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        False,
+        mightyList
+    )
+
+
+def privateBin(mightyList):
+    fetchJsonList(
+        'privateBin',
+        'https://privatebin.info/directory/api?top=100&https_redirect=true&min_rating=A&csp_header=true&min_uptime=100&attachments=true',
+        'url',
+        False,
+        mightyList
+    )
+
+
+def pixivFe(mightyList):
+    fetchRegexList(
+        'pixivFe',
+        'https://codeberg.org/VnPower/pixivfe/raw/branch/main/README.md',
+        r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*)\/? +\|",
+        mightyList
+    )

services/reddit.py

@@ -0,0 +1,46 @@
+from .utils import fetchJsonList
+
+
+def libreddit(mightyList):
+    fetchJsonList(
+        'libreddit',
+        'https://github.com/libreddit/libreddit-instances/raw/master/instances.json',
+        {
+            'clearnet': 'url',
+            'tor': 'onion',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        True,
+        mightyList
+    )
+
+
+def redlib(mightyList):
+    fetchJsonList(
+        'redlib',
+        'https://github.com/redlib-org/redlib-instances/raw/main/instances.json',
+        {
+            'clearnet': 'url',
+            'tor': 'onion',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        True,
+        mightyList
+    )
+
+
+def teddit(mightyList):
+    fetchJsonList(
+        'teddit',
+        'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json',
+        {
+            'clearnet': 'url',
+            'tor': 'onion',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        False,
+        mightyList
+    )

services/search.py

@@ -0,0 +1,71 @@
+import requests
+from colorama import Fore, Style
+import json
+import re
+from .utils import torRegex, i2pRegex, fetchRegexList, fetchJsonList
+import yaml
+
+
+def searxng(mightyList):
+    r = requests.get(
+        'https://searx.space/data/instances.json')
+    rJson = json.loads(r.text)
+    searxngList = {}
+    searxngList['clearnet'] = []
+    searxngList['tor'] = []
+    searxngList['i2p'] = []
+    searxngList['loki'] = []
+
+    for item in rJson['instances']:
+        if re.search(torRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
+            searxngList['tor'].append(item[:-1])
+        elif re.search(i2pRegex, item[:-1]) and rJson['instances'][item].get('generator') == 'searxng':
+            searxngList['i2p'].append(item[:-1])
+        elif rJson['instances'][item].get('generator') == 'searxng':
+            searxngList['clearnet'].append(item[:-1])
+
+    mightyList['searxng'] = searxngList
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
+
+
+def searx(mightyList):
+    searxList = {}
+    searxList['clearnet'] = []
+    searxList['tor'] = []
+    searxList['i2p'] = []
+    searxList['loki'] = []
+    r = requests.get(
+        'https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml')
+    data = yaml.safe_load(r.text)
+    for key in data:
+        searxList['clearnet'].append(key)
+        if 'additional_urls' in data[key]:
+            for additional_url in data[key]['additional_urls']:
+                if data[key]['additional_urls'][additional_url] == "Hidden Service":
+                    searxList['tor'].append(additional_url)
+    mightyList['searx'] = searxList
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearXNG')
+
+
+def whoogle(mightyList):
+    fetchRegexList(
+        'whoogle',
+        'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md',
+        r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ",
+        mightyList
+    )
+
+
+def librex(mightyList):
+    fetchJsonList(
+        'librex',
+        'https://raw.githubusercontent.com/hnhx/librex/main/instances.json',
+        {
+            'clearnet': 'clearnet',
+            'tor': 'tor',
+            'i2p': 'i2p',
+            'loki': None
+        },
+        True,
+        mightyList
+    )

services/translate.py

@@ -0,0 +1,20 @@
+from .utils import fetchJsonList, fetchRegexList
+
+
+def linvgatranslate(mightyList):
+    fetchJsonList(
+        'lingva',
+        'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json',
+        None,
+        False,
+        mightyList
+    )
+
+
+def libreTranslate(mightyList):
+    fetchRegexList(
+        'libreTranslate',
+        'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md',
+        r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|",
+        mightyList
+    )

services/utils.py

@@ -0,0 +1,229 @@
+import traceback
+import logging
+import requests
+import json
+from urllib.parse import urlparse
+import re
+from colorama import Fore, Style
+import socket
+
+
+startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*"
+endRegex = "(?:\\/[^\\s\\/]+)*\\/?"
+torRegex = startRegex + "onion" + endRegex
+i2pRegex = startRegex + "i2p" + endRegex
+lokiRegex = startRegex + "loki" + endRegex
+authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex
+
+
+def get_cloudflare_ips():
+    r = requests.get('https://www.cloudflare.com/ips-v4')
+    return r.text.split('\n')
+
+
+cloudflare_ips = get_cloudflare_ips()
+
+
+def ip2bin(ip): return "".join(
+    map(
+        str,
+        [
+            "{0:08b}".format(int(x)) for x in ip.split(".")
+        ]
+    )
+)
+
+
+def is_cloudflare(url):
+    instance_ip = None
+    try:
+        instance_ip = socket.gethostbyname(urlparse(url).hostname)
+        if instance_ip is None:
+            return False
+    except Exception:
+        return False
+    instance_bin = ip2bin(instance_ip)
+
+    for cloudflare_ip_mask in cloudflare_ips:
+        cloudflare_ip = cloudflare_ip_mask.split('/')[0]
+        cloudflare_bin = ip2bin(cloudflare_ip)
+
+        mask = int(cloudflare_ip_mask.split('/')[1])
+        cloudflare_bin_masked = cloudflare_bin[:mask]
+        instance_bin_masked = instance_bin[:mask]
+
+        if cloudflare_bin_masked == instance_bin_masked:
+            print(url + ' is behind ' + Fore.RED +
+                  'cloudflare' + Style.RESET_ALL)
+            return True
+    return False
+
+
+is_cloudflare('https://pipedapi-libre.kavin.rocks')
+
+
+def filterLastSlash(urlList):
+    tmp = {}
+    for frontend in urlList:
+        tmp[frontend] = {}
+        for network in urlList[frontend]:
+            tmp[frontend][network] = []
+            for url in urlList[frontend][network]:
+                if url.endswith('/'):
+                    tmp[frontend][network].append(url[:-1])
+                    print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
+                else:
+                    tmp[frontend][network].append(url)
+    return tmp
+
+
+def idnaEncode(urlList):
+    tmp = {}
+    for frontend in urlList:
+        tmp[frontend] = {}
+        for network in urlList[frontend]:
+            tmp[frontend][network] = []
+            for url in urlList[frontend][network]:
+                try:
+                    encodedUrl = url.encode("idna").decode("utf8")
+                    tmp[frontend][network].append(encodedUrl)
+                    if (encodedUrl != url):
+                        print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url)
+                except Exception:
+                    tmp[frontend][network].append(url)
+    return tmp
+
+
+def fetchCache(frontend, mightyList):
+    try:
+        with open('./data.json') as file:
+            mightyList[frontend] = json.load(file)[frontend]
+        print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + frontend)
+    except Exception:
+        print(Fore.RED + 'Failed' + Style.RESET_ALL +
+              ' to get cached ' + frontend)
+
+
+def fetchFromFile(frontend, mightyList):
+    with open('./fixed/' + frontend + '.json') as file:
+        mightyList[frontend] = json.load(file)
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+
+
+networks = {}
+
+with open('networks.json', 'rt') as tmp:
+    networks = json.load(tmp)
+
+
+def fetchJsonList(frontend, url, urlItem, jsonObject, mightyList):
+    try:
+        r = requests.get(url)
+        rJson = json.loads(r.text)
+        if jsonObject:
+            rJson = rJson['instances']
+        _list = {}
+        for network in networks:
+            _list[network] = []
+        if type(urlItem) == dict:
+            for item in rJson:
+                for network in networks:
+                    if urlItem[network] is not None:
+                        if urlItem[network] in item and item[urlItem[network]] is not None:
+                            if type(item[urlItem[network]]) == list:
+                                for i in item[urlItem[network]]:
+                                    if i.strip() != '':
+                                        _list[network].append(i)
+                            elif item[urlItem[network]].strip() != '':
+                                _list[network].append(item[urlItem[network]])
+        else:
+            for item in rJson:
+                tmpItem = item
+                if urlItem is not None:
+                    tmpItem = item[urlItem]
+                if tmpItem.strip() == '':
+                    continue
+                elif re.search(torRegex, tmpItem):
+                    _list['tor'].append(tmpItem)
+                elif re.search(i2pRegex, tmpItem):
+                    _list['i2p'].append(tmpItem)
+                elif re.search(lokiRegex, tmpItem):
+                    _list['loki'].append(tmpItem)
+                else:
+                    _list['clearnet'].append(tmpItem)
+
+        mightyList[frontend] = _list
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+    except Exception:
+        fetchCache(frontend)
+        logging.error(traceback.format_exc())
+
+
+def fetchRegexList(frontend, url, regex, mightyList):
+    try:
+        r = requests.get(url)
+        _list = {}
+        for network in networks:
+            _list[network] = []
+
+        tmp = re.findall(regex, r.text)
+
+        for item in tmp:
+            if item.strip() == "":
+                continue
+            elif re.search(torRegex, item):
+                _list['tor'].append(item)
+            elif re.search(i2pRegex, item):
+                _list['i2p'].append(item)
+            elif re.search(lokiRegex, item):
+                _list['loki'].append(item)
+            else:
+                _list['clearnet'].append(item)
+        mightyList[frontend] = _list
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+    except Exception:
+        fetchCache(frontend)
+        logging.error(traceback.format_exc())
+
+
+def fetchTextList(frontend, url, prepend):
+    try:
+        _list = {}
+        for network in networks:
+            _list[network] = []
+
+        if type(url) == dict:
+            for network in networks:
+                if url[network] is not None:
+                    r = requests.get(url[network])
+                    tmp = r.text.strip().split('\n')
+                    for item in tmp:
+                        item = prepend[network] + item
+                        _list[network].append(item)
+        else:
+            r = requests.get(url)
+            tmp = r.text.strip().split('\n')
+
+            for item in tmp:
+                item = prepend + item
+                if re.search(torRegex, item):
+                    _list['tor'].append(item)
+                elif re.search(i2pRegex, item):
+                    _list['i2p'].append(item)
+                elif re.search(lokiRegex, item):
+                    _list['loki'].append(item)
+                else:
+                    _list['clearnet'].append(item)
+        mightyList[frontend] = _list
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+    except Exception:
+        fetchCache(frontend)
+        logging.error(traceback.format_exc())
+
+
+def isValid(url):  # by avanitrachhadiya2155
+    try:
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except Exception:
+        return False

services/wolfram.py

@@ -0,0 +1,33 @@
+import requests
+from colorama import Fore, Style
+import json
+
+wolfreeAlpha_url_list = [
+    "https://gqq.gitlab.io",
+    "https://jqq.gitlab.io",
+    "https://rqq.gitlab.io",
+    "https://sqq.gitlab.io",
+    "https://uqq.gitlab.io"
+]
+
+
+def wolfreeAlpha(mightyList):
+    global wolfreeAlpha_url_list_i
+    frontend = 'wolfreeAlpha'
+    for instance in wolfreeAlpha_url_list:
+        try: 
+            r = requests.get(instance+"/instances.json")
+            if r.status_code != 200:
+                continue
+            else:
+                rJson = json.loads(r.text)
+                networks = rJson['wolfree']
+                _list = {}
+                for i in networks.keys():
+                    _list[i] = networks[i]
+                mightyList[frontend] = _list
+                print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+                break
+        except:
+            wolfreeAlpha_url_list_i += 1
+            wolfreeAlpha(wolfreeAlpha_url_list_i)

services/youtube.py

@@ -0,0 +1,114 @@
+from .utils import fetchRegexList, fetchCache, fetchJsonList
+import requests
+import json
+from colorama import Fore, Style
+import logging
+import traceback
+import re
+
+
+def invidious(mightyList):
+    name = 'Invidious'
+    frontend = 'invidious'
+    url = 'https://api.invidious.io/instances.json'
+    try:
+        _list = {}
+        _list['clearnet'] = []
+        _list['tor'] = []
+        _list['i2p'] = []
+        _list['loki'] = []
+        r = requests.get(url)
+        rJson = json.loads(r.text)
+        for instance in rJson:
+            if instance[1]['type'] == 'https':
+                _list['clearnet'].append(instance[1]['uri'])
+            elif instance[1]['type'] == 'onion':
+                _list['tor'].append(instance[1]['uri'])
+            elif instance[1]['type'] == 'i2p':
+                _list['i2p'].append(instance[1]['uri'])
+        mightyList[frontend] = _list
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name)
+    except Exception:
+        fetchCache(frontend, mightyList)
+        logging.error(traceback.format_exc())
+
+
+def piped(mightyList):
+    frontend = 'piped'
+    try:
+        _list = {}
+        _list['clearnet'] = []
+        _list['tor'] = []
+        _list['i2p'] = []
+        _list['loki'] = []
+        r = requests.get(
+            'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md')
+
+        tmp = re.findall(
+            r' \| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text)
+        for item in tmp:
+            try:
+                url = requests.get(item, timeout=5).url
+                if url.strip("/") == item:
+                    continue
+                else:
+                    _list['clearnet'].append(url)
+            except Exception:
+                logging.error(traceback.format_exc())
+                continue
+        mightyList[frontend] = _list
+        _list['clearnet'].remove("https://piped.video")
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+    except Exception:
+        fetchCache(frontend)
+        logging.error(traceback.format_exc())
+
+
+def materialious(mightyList):
+    fetchRegexList(
+        'materialious',
+        'https://raw.githubusercontent.com/Materialious/Materialious/main/docs/INSTANCES.md',
+        r"- \[.*\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)",
+        mightyList
+    )
+
+
+def pipedMaterial(mightyList):
+    fetchRegexList(
+        'pipedMaterial',
+        'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md',
+        r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production",
+        mightyList
+    )
+
+
+def poketube(mightyList):
+    frontend = 'poketube'
+    try:
+        r = requests.get(
+            'https://codeberg.org/Ashley/poketube/raw/branch/main/instances.json')
+        rJson = json.loads(r.text)
+        _list = {
+            'clearnet': [],
+            'tor': [],
+            'i2p': [],
+            'loki': []
+
+        }
+        for element in rJson:
+            _list['clearnet'].append(element[1]['uri'])
+
+        mightyList[frontend] = _list
+        print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + frontend)
+    except Exception:
+        fetchCache(frontend)
+        logging.error(traceback.format_exc())
+
+
+def hyperpipe(mightyList):
+    fetchJsonList(
+        'hyperpipe',
+        'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json',
+        'url',
+        False,
+        mightyList)