mirror of
https://codeberg.org/gothub/gothub-instances
synced 2024-12-06 19:16:17 +01:00
NoPlagiarism
81 lines
3.1 KiB
Python
81 lines
3.1 KiB
Python
#!/usr/bin/python3
|
|
import requests
|
|
import json
|
|
import re
|
|
|
|
HEADERS = {
|
|
'User-Agent': 'Mozilla/5.0 GotHubInstanceFetcher/1.0 (+codeberg.org/gothub/gothub-instances)'
|
|
}
|
|
|
|
README_URL = "https://codeberg.org/gothub/gothub/raw/branch/dev/README.md" # Only raw .md file!!!
|
|
README_CROP = ("## Instances", "\n## ") # Crop from instances header to next header 2. (None, None) - to fully disable crop
|
|
|
|
|
|
def from_tuple_to_instance_json(data: tuple) -> dict: # data in the same order, as in regex
|
|
res = {
|
|
'country': data[3],
|
|
'link': data[1],
|
|
'cloudflare': data[2] == 'Yes',
|
|
'host': data[4],
|
|
'branch': data[5]
|
|
}
|
|
return res
|
|
|
|
|
|
def get_info_from_readme_iter(url: str):
|
|
pattern = re.compile(r"\|\s+\[(?P<domain>[\w\-\.]+)\]\((?P<url>https?:\/\/(?:www\.)?[\w\-\.]+)\/?\)\s+\|" # [domain](url)
|
|
r"\s+(?P<cloudflare>Yes|No)\s+\|" # Only "Yes" and "No" accepted
|
|
r"\s+(?P<country>[^\|]+)\s+\|" # All three columns below accepts all symbols except "|"
|
|
r"\s+(?P<isp>[^\|]+)\s+\|"
|
|
r"\s+(?P<branch>[^\|]+)\|",
|
|
flags=re.MULTILINE | re.IGNORECASE)
|
|
print("Getting MD")
|
|
r = requests.get(url, headers=HEADERS)
|
|
text = r.text
|
|
|
|
# Cropping Instances only
|
|
crop_from_index = text.index(README_CROP[0])+len(README_CROP[0]) if README_CROP[0] is not None else 0
|
|
crop_to_index = text[crop_from_index:].index(README_CROP[1]) + crop_from_index if README_CROP[1] is not None else len(text)
|
|
instances_str = text[crop_from_index:crop_to_index]
|
|
|
|
for data in pattern.finditer(instances_str):
|
|
yield from_tuple_to_instance_json(data.groups())
|
|
|
|
|
|
def get_version(instance_url: str):
|
|
instance_url = instance_url.strip("/") # Be sure there no "/" on end
|
|
versions_url = instance_url + "/api/v1/version"
|
|
try:
|
|
r = requests.get(versions_url, headers=HEADERS)
|
|
if r.status_code != 200:
|
|
print("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code))
|
|
return None
|
|
elif r.headers['Content-Type'] != 'application/json':
|
|
print("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type']))
|
|
return None
|
|
except Exception as e:
|
|
print("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e)))
|
|
return None
|
|
return r.json()
|
|
|
|
|
|
def main():
|
|
instances_json = list()
|
|
for instance_data in get_info_from_readme_iter(README_URL):
|
|
print("Scraping {} instance...".format(instance_data['link']))
|
|
versions = get_version(instance_data['link'])
|
|
if versions is None:
|
|
continue
|
|
instance_data.update(versions)
|
|
instances_json.append(instance_data)
|
|
print("Scraping finished. Saving JSON...")
|
|
|
|
# save JSON
|
|
with open('instances.json', 'w') as outfile:
|
|
json.dump(instances_json, outfile, indent=4)
|
|
print("File saved as instances.json")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|