Merge pull request 'Add logging' (#10) from NoPlagiarism/gothub-instances:logging into master

Reviewed-on: https://codeberg.org/gothub/gothub-instances/pulls/10

logging.cfg

@@ -0,0 +1,21 @@
+[loggers]
+keys=root
+
+[formatters]
+keys=detail
+
+[handlers]
+keys=console
+
+[formatter_detail]
+format=[%(asctime)s] [%(name)s:%(lineno)d] [%(levelname)s]: %(message)s
+datefmt=%m-%d %H:%M:%S
+
+[handler_console]
+class=logging.StreamHandler
+formatter=detail
+level=DEBUG
+
+[logger_root]
+level=DEBUG
+handlers=console

main.py

@@ -1,4 +1,6 @@
 #!/usr/bin/python3
+import logging
+import logging.config
 import json
 import re
 
@@ -15,14 +17,32 @@ README_CROP = ("## Instances", "\n## ")  # Crop from instances header to next he
 
 SORT_KEYS = ("country", "link", "cloudflare", "host", "branch", "version", "fiberversion", "goversion")  # If key's not here, then it goes upper
 
+logging.config.fileConfig("logging.cfg")
+logger = logging.getLogger("root")
+
+
+def enable_log_in_module(str_module: str, level: int = logging.DEBUG):
+    mod_log = logging.getLogger(str_module)
+    mod_log.disabled = False
+    mod_log.setLevel(level)
+    mod_log.propagate = True
+
+
+enable_log_in_module("urllib3.util.retry")
+enable_log_in_module("urllib3.connectionpool", level=logging.WARNING)
+enable_log_in_module("urllib3.connection", level=logging.WARNING)
+enable_log_in_module("requests", level=logging.WARNING)
+
 session = requests.Session()
 session.headers = HEADERS
 retries = Retry(total=5,
                 connect=3,       # ConnectTimeoutError
                 read=False,      # ReadTimeoutError or ProtocolError
                 redirect=False,  # obvi, any redirections
-                status=2,    # Status codes by server
-                backoff_factor=0.5)
+                status=2,        # Status codes by server
+                backoff_factor=1,
+                backoff_max=30,  # Just to be sure that script don't go sleep for a minute
+                respect_retry_after_header=False)  # TODO: add support of Retry-After + logging + limit
 http_adapter = HTTPAdapter(max_retries=retries)
 session.mount('https://', http_adapter)
 session.mount('http://', http_adapter)  # Just to be sure
@@ -73,7 +93,7 @@ def get_info_from_readme_iter(url: str):
                          r"\s+(?P<isp>(?:[^\|]|\\\|)+)\s+\|"
                          r"\s+(?P<branch>(?:[^\|]|\\\|)+)\s+\|",
                          flags=re.MULTILINE | re.IGNORECASE)
-    print("Getting MD")
+    logger.info("Getting MD")
     r = session.get(url)
     text = r.text
 
@@ -92,13 +112,13 @@ def get_version(instance_url: str):
     try:
         r = session.get(versions_url)
         if r.status_code != 200:
-            print("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code))
+            logger.warning("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code))
             return None
         elif r.headers['Content-Type'] != 'application/json':
-            print("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type']))
+            logger.warning("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type']))
             return None
     except Exception as e:
-        print("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e)))
+        logger.warning("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e)))
         return None
     return r.json()
 
@@ -106,19 +126,19 @@ def get_version(instance_url: str):
 def main():
     instances_json = list()
     for instance_data in get_info_from_readme_iter(README_URL):
-        print("Scraping {} instance...".format(instance_data['link']))
+        logger.info("Scraping {} instance...".format(instance_data['link']))
         versions = get_version(instance_data['link'])
         if versions is None:
             continue
         instance_data.update(versions)
         instance_data = sort_instance_json(instance_data)
         instances_json.append(instance_data)
-    print("Scraping finished. Saving JSON...")
+    logger.info("Scraping finished. Saving JSON...")
 
     # save JSON
     with open('instances.json', 'w') as outfile:
         json.dump(instances_json, outfile, indent=4)
-        print("File saved as instances.json")
+    logger.info("File saved as instances.json")
 
 
 if __name__ == '__main__':