From 865db233fd271ad766b4fefef93fcdb06b891192 Mon Sep 17 00:00:00 2001 From: ManeraKai Date: Tue, 17 Jan 2023 21:20:54 +0300 Subject: [PATCH] Removed authenticate. Some modifications --- blacklist.json | 26 ++++---------- data.json | 98 +++++++++++--------------------------------------- instances.py | 51 ++++++-------------------- 3 files changed, 39 insertions(+), 136 deletions(-) diff --git a/blacklist.json b/blacklist.json index 43738c1..67bc11d 100644 --- a/blacklist.json +++ b/blacklist.json @@ -5,17 +5,18 @@ "https://yt.oelrichsgarcia.de", "https://piped.video", "https://piped.moomoo.me", - "https://piped.syncpundit.io", "https://piped.mha.fi", "https://piped.garudalinux.org", + "https://efy.piped.pages.dev", + "https://watch.leptons.xyz", + "https://piped.lunar.icu", + "https://piped.video", "https://proxitok.marcopisco.com", "https://proxitok.lunar.icu", "https://send.q1q.wtf", - "https://transferer.seinetransfert.fr", "https://nitter.domain.glass", "https://birdsite.xanny.family", "https://notabird.site", - "https://fuckthesacklers.network", "https://nitter.winscloud.net", "https://twitter.dr460nf1r3.org", "https://nitter.garudalinux.org", @@ -39,7 +40,6 @@ "https://libreddit.cachyos.org", "https://libreddit.freedit.eu", "https://libreddit.garudalinux.org", - "https://www.futureddit.gq", "https://lr.aeong.one", "https://libreddit.lunar.icu", "https://teddit.domain.glass", @@ -52,28 +52,16 @@ "https://quetre.marcopisco.com", "https://quetre.lunar.icu", "https://libremdb.lunar.icu", + "https://lingva.lunar.icu", "https://lingva.garudalinux.org", "https://translate.dr460nf1r3.org", "https://libretranslate.com", + "https://search.albony.xyz", "https://search.garudalinux.org", "https://search.dr460nf1r3.org", "https://whoogle.lunar.icu", "https://i.actionsack.com", "https://rimgo.marcopisco.com", - "https://beatbump.ml", - "https://bin.klowdee.host", - "https://privatebin.io", - "https://bonus01.hwb0307.com", - "https://0.klowdee.host", - "https://sendpass.ru", - "https://0a0bin.klowdee.host", - "https://0.0g.gg", - "https://privload.com", - "https://andoc.yateam.cc", - "https://p.darklab.sh", - "https://luckvintage.com" - ], - "authenticate": [ - "https://nitter:nitter@nitter.nixnet.services" + "https://beatbump.ml" ] } \ No newline at end of file diff --git a/data.json b/data.json index 9194ac0..7caafb0 100644 --- a/data.json +++ b/data.json @@ -8,9 +8,10 @@ "https://watch.thekitty.zone", "https://y.com.sb", "https://invidious.nerdvpn.de", - "https://yt.artemislena.eu", "https://invidious.tiekoetter.com", + "https://yt.artemislena.eu", "https://invidious.flokinet.to", + "https://inv.bp.projectsegfau.lt", "https://inv.vern.cc", "https://inv.odyssey346.dev", "https://invidious.snopyta.org", @@ -29,7 +30,6 @@ "https://invidious.esmailelbob.xyz", "https://iv.melmac.space", "https://invidious.privacydev.net", - "https://inv.bp.projectsegfau.lt", "https://invidious.lidarshield.cloud", "https://invidious.namazso.eu" ], @@ -57,10 +57,13 @@ "clearnet": [ "https://piped.video", "https://piped.moomoo.me", - "https://piped.syncpundit.io", "https://piped.mha.fi", "https://watch.whatever.social", "https://piped.garudalinux.org", + "https://efy.piped.pages.dev", + "https://watch.leptons.xyz", + "https://piped.lunar.icu", + "https://piped.video", "https://yt.jae.fi", "https://il.ax", "https://piped.esmailelbob.xyz", @@ -197,6 +200,7 @@ "https://nitter.catalyst.sx", "https://unofficialbird.com", "https://nitter.projectsegfau.lt", + "https://nitter.eu.projectsegfau.lt", "https://singapore.unofficialbird.com", "https://nitter.fprivacy.com", "https://canada.unofficialbird.com", @@ -244,7 +248,9 @@ "https://nitter.girlboss.ceo", "https://nitter.lunar.icu", "https://twitter.moe.ngo", - "https://nitter.freedit.eu" + "https://nitter.freedit.eu", + "https://ntr.frail.duckdns.org", + "https://nitter.us.projectsegfau.lt" ], "tor": [ "http://nitter7bryz3jv7e3uekphigvmoyoem4al3fynerxkj22dmoxoq553qd.onion", @@ -279,7 +285,7 @@ "http://codeine3hsqnnkb3dsu6ft4tunlomr3lmuml5hcoqmfkgiqfv2brdqqd.onion", "http://nitter.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion", "http://jdadsqkjereqronxzovgvd453snh5jsvo7mukfwt2feth353nobqotad.onion", - "http://oawb4ub2x2jehgg7k7ll4uoay34re5nbbnwnztpphdec6wkcxgarwbad.onion" + "http://nitter.pjsfkvpxlinjamtawaksbnnaqs2fc2mtvmozrzckxh7f3kis6yea25ad.onion" ], "i2p": [ "http://axd6uavsstsrvstva4mzlzh4ct76rc6zdug3nxdgeitrzczhzf4q.b32.i2p", @@ -528,6 +534,7 @@ "https://lingva.ml", "https://translate.igna.wtf", "https://translate.plausibility.cloud", + "https://lingva.lunar.icu", "https://translate.projectsegfau.lt", "https://lingva.garudalinux.org", "https://translate.dr460nf1r3.org", @@ -557,8 +564,8 @@ }, "searx": { "clearnet": [ + "https://jackgoss.xyz", "https://jsearch.pw", - "https://search.kvj.ovh", "https://search.stinpriza.org", "https://search.trom.tf", "https://searx.bissisoft.com", @@ -587,6 +594,7 @@ ], "tor": [ "http://3afisqjw2rxm6z7mmstyt5rx75qfqrgxnkzftknbp2vhipr2nrmrjdyd.onion", + "http://w5rl6wsd7mzj4bdkbuqvzidet5osdsm5jhg2f7nvfidakfq5exda5wid.onion", "http://z5vawdol25vrmorm4yydmohsd4u6rdoj2sylvoi3e3nqvxkvpqul7bqd.onion", "http://zbuc3bbzbfdqqo2x46repx2ddajbha6fpsjeeptjhhhhzji3zopxdqyd.onion", "http://f4qfqajszpx5b7itzxt6mb7kj4ktpgbdq7lq6xaiqyqx6a7de3epptad.onion", @@ -607,7 +615,6 @@ "https://etsi.me", "https://gsch.eu", "https://intelwinds.com", - "https://jackgoss.xyz", "https://northboot.xyz", "https://notsearch.uk", "https://ooglester.com", @@ -630,6 +637,7 @@ "https://search.drivet.xyz", "https://search.gcomm.ch", "https://search.kiwitalk.de", + "https://search.kvj.ovh", "https://search.leptons.xyz", "https://search.mdosch.de", "https://search.mpx.wtf", @@ -687,7 +695,6 @@ "https://xo.wtf" ], "tor": [ - "http://w5rl6wsd7mzj4bdkbuqvzidet5osdsm5jhg2f7nvfidakfq5exda5wid.onion", "http://4n53nafyi77iplnbrpmxnp3x4exbswwxigujaxy3b37fvr7bvlopxeyd.onion", "http://privateoz3u5utrimal2edr56j3r5caakektxxgixigdkycuxigvquid.onion", "http://searxyqdegtqksicnwpxa24ipnb2ylmp6iutnnwtazbhup4dit4eeyqd.onion", @@ -734,7 +741,7 @@ }, "librex": { "clearnet": [ - "https://librex.beparanoid.de", + "https://search.femboy.hu", "https://lx.vern.cc", "https://lx.sesu.cc", "https://librex.mikata.ru", @@ -749,7 +756,6 @@ "https://search.milivojevic.in.rs", "https://librex.ratakor.com", "https://search.tildevarsh.in", - "https://librex.yogeshlamichhane.com.np", "https://librex.myroware.eu" ], "tor": [ @@ -767,7 +773,6 @@ }, "rimgo": { "clearnet": [ - "https://i.bcow.xyz", "https://rimgo.pussthecat.org", "https://rimgo.totaldarkness.net", "https://rimgo.bus-hit.me", @@ -789,14 +794,15 @@ }, "librarian": { "clearnet": [ - "https://lbry.bcow.xyz", "https://odysee.076.ne.jp", "https://librarian.pussthecat.org", "https://lbry.projectsegfau.lt", "https://librarian.esmailelbob.xyz", "https://lbry.vern.cc", "https://lbry.slipfox.xyz", - "https://lbry.mywire.org" + "https://lbry.mywire.org", + "https://lbry.ooguy.com", + "https://lbn.frail.duckdns.org" ], "tor": [ "http://librarian.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", @@ -848,22 +854,6 @@ "i2p": [], "loki": [] }, - "simpleertube": { - "clearnet": [ - "https://tube.simple-web.org", - "https://simpleertube.esmailelbob.xyz", - "https://stube.tokhmi.xyz", - "https://st.vern.cc" - ], - "tor": [ - "http://simpleertube.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", - "http://st.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" - ], - "i2p": [ - "http://verntblehhel4dx6ntyxkj4mryvic3s3yptzxewzwnldq3esi4qq.b32.i2p" - ], - "loki": [] - }, "breezeWiki": { "clearnet": [ "https://breezewiki.com", @@ -871,54 +861,8 @@ "https://bw.odyssey346.dev", "https://bw.vern.cc", "https://breezewiki.esmailelbob.xyz", - "https://bw.artemislena.eu" - ], - "tor": [], - "i2p": [], - "loki": [] - }, - "privateBin": { - "clearnet": [ - "https://bin.bus-hit.me", - "https://0.jaegers.net", - "https://privatebin.net", - "https://paste.0xfc.de", - "https://bin.klowdee.host", - "https://paste.ononoki.org", - "https://paste.aryu.de", - "https://privatebin.io", - "https://cpaste.org", - "https://pastebin.hot-chilli.net", - "https://bonus01.hwb0307.com", - "https://paste.momobako.com", - "https://0.klowdee.host", - "https://sendpass.ru", - "https://0a0bin.klowdee.host", - "https://0.0g.gg", - "https://paste.hostux.net", - "https://pastebin.grey.pw", - "https://paste.ottertelecom.com", - "https://privatebin.at", - "https://snip.dssr.ch", - "https://bin.foss.wtf", - "https://paste.tildeverse.org", - "https://secrets.true.nl", - "https://secrets.cebador.fr", - "https://pastebin.linqhost.nl", - "https://paste.itefix.net", - "https://paste.tuxcloud.net", - "https://privatebin.rockhost.se", - "https://bin.0xfc.de", - "https://paste.07f.de", - "https://notebin.de", - "https://privload.com", - "https://privatebin.wsmm.de", - "https://vip.ci", - "https://andoc.yateam.cc", - "https://pb.vern.cc", - "https://paste.techbag.app", - "https://p.darklab.sh", - "https://luckvintage.com" + "https://bw.artemislena.eu", + "https://breeze.hostux.net" ], "tor": [], "i2p": [], diff --git a/instances.py b/instances.py index e7d8467..b228a89 100755 --- a/instances.py +++ b/instances.py @@ -19,9 +19,6 @@ i2pRegex = startRegex + "i2p" + endRegex lokiRegex = startRegex + "loki" + endRegex authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex -# 2.0 because Libredirect is currently on version 2.x.x -headers = {'User-Agent': 'Libredirect-instance-fetcher/2.0'} - with open('networks.json', 'rt') as tmp: networks = json.load(tmp) @@ -101,22 +98,6 @@ def is_cloudflare(url): return False -def is_authenticate(url): - try: - if re.match(authRegex, url): - print(url + ' requires ' + Fore.RED + - 'authentication' + Style.RESET_ALL) - return True - r = requests.get(url, timeout=5, headers=headers) - if 'www-authenticate' in r.headers: - print(url + ' requires ' + Fore.RED + - 'authentication' + Style.RESET_ALL) - return True - except Exception: - return False - return False - - def fetchCache(frontend, name): try: with open('./data.json') as file: @@ -134,7 +115,7 @@ def fetchFromFile(frontend, name): def fetchJsonList(frontend, name, url, urlItem, jsonObject): try: - r = requests.get(url, headers=headers) + r = requests.get(url) rJson = json.loads(r.text) if jsonObject: rJson = rJson['instances'] @@ -173,7 +154,7 @@ def fetchJsonList(frontend, name, url, urlItem, jsonObject): def fetchRegexList(frontend, name, url, regex): try: - r = requests.get(url, headers=headers) + r = requests.get(url) _list = {} for network in networks: _list[network] = [] @@ -207,13 +188,13 @@ def fetchTextList(frontend, name, url, prepend): if type(url) == dict: for network in networks: if url[network] is not None: - r = requests.get(url[network], headers=headers) + r = requests.get(url[network]) tmp = r.text.strip().split('\n') for item in tmp: item = prepend[network] + item _list[network].append(item) else: - r = requests.get(url, headers=headers) + r = requests.get(url) tmp = r.text.strip().split('\n') for item in tmp: @@ -243,7 +224,7 @@ def invidious(): _list['tor'] = [] _list['i2p'] = [] _list['loki'] = [] - r = requests.get(url, headers=headers) + r = requests.get(url) rJson = json.loads(r.text) for instance in rJson: if instance[1]['type'] == 'https': @@ -269,13 +250,13 @@ def piped(): _list['i2p'] = [] _list['loki'] = [] r = requests.get( - 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md', headers=headers) + 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md') tmp = re.findall( - r'(?:[^\s\/]+\.)+[a-zA-Z]+ (?:\(Official\) )?\| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) + r' \| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) for item in tmp: try: - url = requests.get(item, timeout=5, headers=headers).url + url = requests.get(item, timeout=5).url if url.strip("/") == item: continue else: @@ -343,11 +324,6 @@ def libremdb(): r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") -def simpleertube(): - fetchTextList('simpleertube', 'SimpleerTube', {'clearnet': 'https://simple-web.org/instances/simpleertube', 'tor': 'https://simple-web.org/instances/simpleertube_onion', - 'i2p': 'https://simple-web.org/instances/simpleertube_i2p', 'loki': None}, {'clearnet': 'https://', 'tor': 'http://', 'i2p': 'http://', 'loki': 'http://'}) - - def simplytranslate(): fetchTextList('simplyTranslate', 'SimplyTranslate', {'clearnet': 'https://simple-web.org/instances/simplytranslate', 'tor': 'https://simple-web.org/instances/simplytranslate_onion', 'i2p': 'https://simple-web.org/instances/simplytranslate_i2p', 'loki': 'https://simple-web.org/instances/simplytranslate_loki'}, {'clearnet': 'https://', 'tor': 'http://', 'i2p': 'http://', 'loki': 'http://'}) @@ -360,7 +336,7 @@ def linvgatranslate(): def searx_searxng(): r = requests.get( - 'https://searx.space/data/instances.json', headers=headers) + 'https://searx.space/data/instances.json') rJson = json.loads(r.text) searxList = {} searxList['clearnet'] = [] @@ -479,14 +455,12 @@ beatbump() hyperpipe() facil() osm() -simpleertube() breezeWiki() -privateBin() +# privateBin() mightyList = filterLastSlash(mightyList) mightyList = idnaEncode(mightyList) cloudflare = [] -authenticate = [] for k1, v1 in mightyList.items(): if type(mightyList[k1]) is dict: for k2, v2 in mightyList[k1].items(): @@ -497,11 +471,8 @@ for k1, v1 in mightyList.items(): else: if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance): cloudflare.append(instance) - if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_authenticate(instance): - authenticate.append(instance) blacklist = { - 'cloudflare': cloudflare, - 'authenticate': authenticate, + 'cloudflare': cloudflare } # Writing to file