gothub-instances/main.py

#!/usr/bin/python3
import logging
import logging.config
import json
import re

import requests
from requests.adapters import HTTPAdapter
from urllib3.util import Retry

HEADERS = {
    'User-Agent': 'Mozilla/5.0 GotHubInstanceFetcher/1.0 (+codeberg.org/gothub/gothub-instances)'
}

README_URL = "https://codeberg.org/gothub/gothub/raw/branch/dev/README.md"  # Only raw .md file!!!
README_CROP = ("## Instances", "\n## ")  # Crop from instances header to next header 2. (None, None) - to fully disable crop

SORT_KEYS = ("country", "link", "cloudflare", "host", "branch", "version", "fiberversion", "goversion")  # If key's not here, then it goes upper

logging.config.fileConfig("logging.cfg")
logger = logging.getLogger("root")


def enable_log_in_module(str_module: str, level: int = logging.DEBUG):
    mod_log = logging.getLogger(str_module)
    mod_log.disabled = False
    mod_log.setLevel(level)
    mod_log.propagate = True


enable_log_in_module("urllib3.util.retry")
enable_log_in_module("urllib3.connectionpool", level=logging.WARNING)
enable_log_in_module("urllib3.connection", level=logging.WARNING)
enable_log_in_module("requests", level=logging.WARNING)

session = requests.Session()
session.headers = HEADERS
retries = Retry(total=5,
                connect=3,       # ConnectTimeoutError
                read=False,      # ReadTimeoutError or ProtocolError
                redirect=False,  # obvi, any redirections
                status=2,        # Status codes by server
                backoff_factor=1,
                backoff_max=30,  # Just to be sure that script don't go sleep for a minute
                respect_retry_after_header=False)  # TODO: add support of Retry-After + logging + limit
http_adapter = HTTPAdapter(max_retries=retries)
session.mount('https://', http_adapter)
session.mount('http://', http_adapter)  # Just to be sure


def from_tuple_to_instance_json(data: tuple) -> dict:  # data in the same order, as in regex
    res = {
        'country': data[3],
        'link': data[1].rstrip("/"),
        'cloudflare': data[2] == 'Yes',
        'host': data[4],
        'branch': data[5]
    }
    return res


def sort_instance_json(data: dict) -> dict:
    res = dict()
    for key in sorted(data.keys(), key=lambda x: SORT_KEYS.index(x) if x in SORT_KEYS else -1):
        res[key] = data[key]
    return res


def get_info_from_readme_iter(url: str):
    """Some info about pattern

    TL:DR:
     - Learn regex - https://github.com/ziishaned/learn-regex/blob/master/README.md
     - Markdown Cheatsheet - https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#tables
     - Regex example on regex101 - https://regex101.com/r/OpcjyY/1

    The basis of the code below is scraping raw .MD table, that has **valid** information
    Table has 5 columns: | [domain or name](link) | cloudflare (usually "Yes" or "No") | country | isp | branch |

    If we need match `| any information |` we need to match borders, spaces before and after info, so it would be like
    `\|\s+[(?P<info>(?:[^\|]|\\\|)+)\s+\|`.
    (?:[^\|]|\\\|)+ - means any character except border of column, excluding escaped character

    Only column regex pattern's checking is [domain](url) (It's md link), specifically "url" part
    URL must start with "https://" (or "http://") and can't contain ")"

    To escape matching foreign table columns table is being cropped using README_CROP constants
    """
    pattern = re.compile(r"\|\s+\[(?P<domain>(?:[^\|]|\\\|)+)\]"  # [example.com] or even [example.com OfFiCiAl]
                         r"\((?P<url>https?:\/\/(?:www\.)?[^\)]+)\)\s+\|"  # (url). Accepts anything like "http://www...grg", no checks
                         r"\s+(?P<cloudflare>(?:[^\|]|\\\|)+)\s+\|"  # All 4 columns below accepts all symbols except "|", but accepts "\|" (escaped "|" in .MD)
                         r"\s+(?P<country>(?:[^\|]|\\\|)+)\s+\|"
                         r"\s+(?P<isp>(?:[^\|]|\\\|)+)\s+\|"
                         r"\s+(?P<branch>(?:[^\|]|\\\|)+)\s+\|",
                         flags=re.MULTILINE | re.IGNORECASE)
    logger.info("Getting MD")
    r = session.get(url)
    text = r.text

    # Cropping Instances tables only
    crop_from_index = text.index(README_CROP[0])+len(README_CROP[0]) if README_CROP[0] is not None else 0
    crop_to_index = text[crop_from_index:].index(README_CROP[1]) + crop_from_index if README_CROP[1] is not None else len(text)
    instances_str = text[crop_from_index:crop_to_index]

    for data in pattern.finditer(instances_str):
        yield from_tuple_to_instance_json(data.groups())


def get_version(instance_url: str):
    instance_url = instance_url.strip("/")  # Be sure there no "/" on end
    versions_url = instance_url + "/api/v1/version"
    try:
        r = session.get(versions_url)
        if r.status_code != 200:
            logger.warning("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code))
            return None
        elif r.headers['Content-Type'] != 'application/json':
            logger.warning("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type']))
            return None
    except Exception as e:
        logger.warning("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e)))
        return None
    return r.json()


def main():
    instances_json = list()
    for instance_data in get_info_from_readme_iter(README_URL):
        logger.info("Scraping {} instance...".format(instance_data['link']))
        versions = get_version(instance_data['link'])
        if versions is None:
            continue
        instance_data.update(versions)
        instance_data = sort_instance_json(instance_data)
        instances_json.append(instance_data)
    logger.info("Scraping finished. Saving JSON...")

    # save JSON
    with open('instances.json', 'w') as outfile:
        json.dump(instances_json, outfile, indent=4)
    logger.info("File saved as instances.json")


if __name__ == '__main__':
    main()