diff --git a/logging.cfg b/logging.cfg new file mode 100644 index 0000000..fb265ee --- /dev/null +++ b/logging.cfg @@ -0,0 +1,21 @@ +[loggers] +keys=root + +[formatters] +keys=detail + +[handlers] +keys=console + +[formatter_detail] +format=[%(asctime)s] [%(name)s:%(lineno)d] [%(levelname)s]: %(message)s +datefmt=%m-%d %H:%M:%S + +[handler_console] +class=logging.StreamHandler +formatter=detail +level=DEBUG + +[logger_root] +level=DEBUG +handlers=console \ No newline at end of file diff --git a/main.py b/main.py index 619b740..2477ab1 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import logging +import logging.config import json import re @@ -15,14 +17,32 @@ README_CROP = ("## Instances", "\n## ") # Crop from instances header to next he SORT_KEYS = ("country", "link", "cloudflare", "host", "branch", "version", "fiberversion", "goversion") # If key's not here, then it goes upper +logging.config.fileConfig("logging.cfg") +logger = logging.getLogger("root") + + +def enable_log_in_module(str_module: str, level: int = logging.DEBUG): + mod_log = logging.getLogger(str_module) + mod_log.disabled = False + mod_log.setLevel(level) + mod_log.propagate = True + + +enable_log_in_module("urllib3.util.retry") +enable_log_in_module("urllib3.connectionpool", level=logging.WARNING) +enable_log_in_module("urllib3.connection", level=logging.WARNING) +enable_log_in_module("requests", level=logging.WARNING) + session = requests.Session() session.headers = HEADERS retries = Retry(total=5, connect=3, # ConnectTimeoutError read=False, # ReadTimeoutError or ProtocolError redirect=False, # obvi, any redirections - status=2, # Status codes by server - backoff_factor=0.5) + status=2, # Status codes by server + backoff_factor=1, + backoff_max=30, # Just to be sure that script don't go sleep for a minute + respect_retry_after_header=False) # TODO: add support of Retry-After + logging + limit http_adapter = HTTPAdapter(max_retries=retries) session.mount('https://', http_adapter) session.mount('http://', http_adapter) # Just to be sure @@ -73,7 +93,7 @@ def get_info_from_readme_iter(url: str): r"\s+(?P(?:[^\|]|\\\|)+)\s+\|" r"\s+(?P(?:[^\|]|\\\|)+)\s+\|", flags=re.MULTILINE | re.IGNORECASE) - print("Getting MD") + logger.info("Getting MD") r = session.get(url) text = r.text @@ -92,13 +112,13 @@ def get_version(instance_url: str): try: r = session.get(versions_url) if r.status_code != 200: - print("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code)) + logger.warning("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code)) return None elif r.headers['Content-Type'] != 'application/json': - print("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type'])) + logger.warning("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type'])) return None except Exception as e: - print("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e))) + logger.warning("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e))) return None return r.json() @@ -106,19 +126,19 @@ def get_version(instance_url: str): def main(): instances_json = list() for instance_data in get_info_from_readme_iter(README_URL): - print("Scraping {} instance...".format(instance_data['link'])) + logger.info("Scraping {} instance...".format(instance_data['link'])) versions = get_version(instance_data['link']) if versions is None: continue instance_data.update(versions) instance_data = sort_instance_json(instance_data) instances_json.append(instance_data) - print("Scraping finished. Saving JSON...") + logger.info("Scraping finished. Saving JSON...") # save JSON with open('instances.json', 'w') as outfile: json.dump(instances_json, outfile, indent=4) - print("File saved as instances.json") + logger.info("File saved as instances.json") if __name__ == '__main__':