diff --git a/main.py b/main.py index ed6e9b9..643a184 100644 --- a/main.py +++ b/main.py @@ -1,76 +1,110 @@ #!/usr/bin/python3 import requests import json -from bs4 import BeautifulSoup +import re -print("Getting HTML") - -headers = { +HEADERS = { 'User-Agent': 'Mozilla/5.0 GotHubInstanceFetcher/1.0 (+codeberg.org/gothub/gothub-instances)' } -# Get the HTML from the page -r = requests.get('https://codeberg.org/gothub/gothub', headers=headers) +README_URL = "https://codeberg.org/gothub/gothub/raw/branch/dev/README.md" # Only raw .md file!!! +README_CROP = ("## Instances", "\n## ") # Crop from instances header to next header 2. (None, None) - to fully disable crop -# Parse the HTML -soup = BeautifulSoup(r.text, 'html.parser') +SORT_KEYS = ("country", "link", "cloudflare", "host", "branch", "version", "fiberversion", "goversion") # If key's not here, then it goes upper -print("Scraping started") -# Get tables -tables = soup.find_all('table') +def from_tuple_to_instance_json(data: tuple) -> dict: # data in the same order, as in regex + res = { + 'country': data[3], + 'link': data[1], + 'cloudflare': data[2] == 'Yes', + 'host': data[4], + 'branch': data[5] + } + return res -# Get table with header 'Master Branch' -table = tables[2] -# Get all rows and columns. Skip the first row because it's the header -rows = table.find_all('tr')[1:] +def sort_instance_json(data: dict) -> dict: + res = dict() + for key in sorted(data.keys(), key=lambda x: SORT_KEYS.index(x) if x in SORT_KEYS else -1): + res[key] = data[key] + return res -theJson = [] -for row in rows: +def get_info_from_readme_iter(url: str): + """Some info about pattern - link = row.find_all('td')[0].find('a')['href'] - cloudflare = row.find_all('td')[1].text - country = row.find_all('td')[2].text - host = row.find_all('td')[3].text - branch = row.find_all('td')[4].text + TL:DR: + - Learn regex - https://github.com/ziishaned/learn-regex/blob/master/README.md + - Markdown Cheatsheet - https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#tables + - Regex example on regex101 - https://regex101.com/r/OpcjyY/1 - print("Scraping " + row.find_all('td')[0].find('a')['href'] + ' instance...') - if cloudflare == 'Yes': - isCloudflare = True - else: - isCloudflare = False + The basis of the code below is scraping raw .MD table, that has **valid** information + Table has 5 columns: | [domain or name](link) | cloudflare (usually "Yes" or "No") | country | isp | branch | + If we need match `| any information |` we need to match borders, spaces before and after info, so it would be like + `\|\s+[(?P(?:[^\|]|\\\|)+)\s+\|`. + (?:[^\|]|\\\|)+ - means any character except border of column, excluding escaped character + + Only column regex pattern's checking is [domain](url) (It's md link), specifically "url" part + URL must start with "https://" (or "http://") and can't contain ")" + + To escape matching foreign table columns table is being cropped using README_CROP constants + """ + pattern = re.compile(r"\|\s+\[(?P(?:[^\|]|\\\|)+)\]" # [example.com] or even [example.com OfFiCiAl] + r"\((?Phttps?:\/\/(?:www\.)?[^\)]+)\)\s+\|" # (url). Accepts anything like "http://www...grg", no checks + r"\s+(?P(?:[^\|]|\\\|)+)\s+\|" # All 4 columns below accepts all symbols except "|", but accepts "\|" (escaped "|" in .MD) + r"\s+(?P(?:[^\|]|\\\|)+)\s+\|" + r"\s+(?P(?:[^\|]|\\\|)+)\s+\|" + r"\s+(?P(?:[^\|]|\\\|)+)\s+\|", + flags=re.MULTILINE | re.IGNORECASE) + print("Getting MD") + r = requests.get(url, headers=HEADERS) + text = r.text + + # Cropping Instances tables only + crop_from_index = text.index(README_CROP[0])+len(README_CROP[0]) if README_CROP[0] is not None else 0 + crop_to_index = text[crop_from_index:].index(README_CROP[1]) + crop_from_index if README_CROP[1] is not None else len(text) + instances_str = text[crop_from_index:crop_to_index] + + for data in pattern.finditer(instances_str): + yield from_tuple_to_instance_json(data.groups()) + + +def get_version(instance_url: str): + instance_url = instance_url.strip("/") # Be sure there no "/" on end + versions_url = instance_url + "/api/v1/version" try: - r = requests.get(link + '/api/v1/version', headers=headers) + r = requests.get(versions_url, headers=HEADERS) if r.status_code != 200: - print("Error while fetching " + link + '/api/v1/version. We got a ' + str(r.status_code) + ' status code. Skipping...') + print("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code)) + return None + elif r.headers['Content-Type'] != 'application/json': + print("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type'])) + return None + except Exception as e: + print("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e))) + return None + return r.json() + + +def main(): + instances_json = list() + for instance_data in get_info_from_readme_iter(README_URL): + print("Scraping {} instance...".format(instance_data['link'])) + versions = get_version(instance_data['link']) + if versions is None: continue - if r.headers['Content-Type'] != 'application/json': - print("Error while fetching " + link + '/api/v1/version. We got a ' + r.headers['Content-Type'] + ' content type. Skipping...') - continue - except: - print("Error while fetching " + link + '/api/v1/version. Skipping...') - continue + instance_data.update(versions) + instance_data = sort_instance_json(instance_data) + instances_json.append(instance_data) + print("Scraping finished. Saving JSON...") - data = json.loads(r.text) - - theJson.append({ - 'country': country, - 'link': link, - 'cloudflare': isCloudflare, - 'host': host, - 'branch': branch, - 'version': data['version'], - 'fiberversion': data['fiberversion'], - 'goversion': data['goversion'], - }) + # save JSON + with open('instances.json', 'w') as outfile: + json.dump(instances_json, outfile, indent=4) + print("File saved as instances.json") -print("Scraping finished. Saving JSON...") - -# save JSON -with open('instances.json', 'w') as outfile: - json.dump(theJson, outfile, indent=4) - print("File saved as instances.json") +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 366fa2d..fd0532b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ -beautifulsoup4==4.11.2 -bs4==0.0.1 certifi==2022.12.7 charset-normalizer==3.0.1 idna==3.4 requests==2.28.2 -soupsieve==2.3.2.post1 urllib3==1.26.14