From 37410ff714f0727898482b0ca0ccc709beaa279c Mon Sep 17 00:00:00 2001
From: NoPlagiarism <NoPlagiarism@noreply.codeberg.org>
Date: Wed, 12 Jul 2023 17:59:50 +0500
Subject: [PATCH] Add request retries support (fixes #5)

---
 main.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/main.py b/main.py
index 643a184..d4fea2e 100644
--- a/main.py
+++ b/main.py
@@ -1,8 +1,11 @@
 #!/usr/bin/python3
-import requests
 import json
 import re
 
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
 HEADERS = {
     'User-Agent': 'Mozilla/5.0 GotHubInstanceFetcher/1.0 (+codeberg.org/gothub/gothub-instances)'
 }
@@ -12,6 +15,18 @@ README_CROP = ("## Instances", "\n## ")  # Crop from instances header to next he
 
 SORT_KEYS = ("country", "link", "cloudflare", "host", "branch", "version", "fiberversion", "goversion")  # If key's not here, then it goes upper
 
+session = requests.Session()
+session.headers = HEADERS
+retries = Retry(total=5,
+                connect=3,       # ConnectTimeoutError
+                read=False,      # ReadTimeoutError or ProtocolError
+                redirect=False,  # obvi, any redirections
+                status=2,    # Status codes by server
+                backoff_factor=0.5)
+http_adapter = HTTPAdapter(max_retries=retries)
+session.mount('https://', http_adapter)
+session.mount('http://', http_adapter)  # Just to be sure
+
 
 def from_tuple_to_instance_json(data: tuple) -> dict:  # data in the same order, as in regex
     res = {
@@ -59,7 +74,7 @@ def get_info_from_readme_iter(url: str):
                          r"\s+(?P<branch>(?:[^\|]|\\\|)+)\s+\|",
                          flags=re.MULTILINE | re.IGNORECASE)
     print("Getting MD")
-    r = requests.get(url, headers=HEADERS)
+    r = session.get(url)
     text = r.text
 
     # Cropping Instances tables only
@@ -75,7 +90,7 @@ def get_version(instance_url: str):
     instance_url = instance_url.strip("/")  # Be sure there no "/" on end
     versions_url = instance_url + "/api/v1/version"
     try:
-        r = requests.get(versions_url, headers=HEADERS)
+        r = session.get(versions_url)
         if r.status_code != 200:
             print("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code))
             return None