mirror of
https://codeberg.org/gothub/gothub-instances
synced 2024-12-06 19:16:17 +01:00
NoPlagiarism
101 lines
4.3 KiB
Python
101 lines
4.3 KiB
Python
#!/usr/bin/python3
|
|
import requests
|
|
import json
|
|
import re
|
|
|
|
HEADERS = {
|
|
'User-Agent': 'Mozilla/5.0 GotHubInstanceFetcher/1.0 (+codeberg.org/gothub/gothub-instances)'
|
|
}
|
|
|
|
README_URL = "https://codeberg.org/gothub/gothub/raw/branch/dev/README.md" # Only raw .md file!!!
|
|
README_CROP = ("## Instances", "\n## ") # Crop from instances header to next header 2. (None, None) - to fully disable crop
|
|
|
|
|
|
def from_tuple_to_instance_json(data: tuple) -> dict: # data in the same order, as in regex
|
|
res = {
|
|
'country': data[3],
|
|
'link': data[1],
|
|
'cloudflare': data[2] == 'Yes',
|
|
'host': data[4],
|
|
'branch': data[5]
|
|
}
|
|
return res
|
|
|
|
|
|
def get_info_from_readme_iter(url: str):
|
|
"""Some info about pattern
|
|
|
|
TL:DR:
|
|
- Learn regex - https://github.com/ziishaned/learn-regex/blob/master/README.md
|
|
- Markdown Cheatsheet - https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#tables
|
|
- Regex example on regex101 - https://regex101.com/r/OpcjyY/1
|
|
|
|
The basis of the code below is scraping raw .MD table, that has **valid** information
|
|
Table has 5 columns: | [domain or name](link) | cloudflare (usually "Yes" or "No") | country | isp | branch |
|
|
|
|
If we need match `| any information |` we need to match borders, spaces before and after info, so it would be like
|
|
`\|\s+[(?P<info>(?:[^\|]|\\\|)+)\s+\|`.
|
|
(?:[^\|]|\\\|)+ - means any character except border of column, excluding escaped character
|
|
|
|
Only column regex pattern's checking is [domain](url) (It's md link), specifically "url" part
|
|
URL must start with "https://" (or "http://") and can't contain ")"
|
|
|
|
To escape matching foreign table columns table is being cropped using README_CROP constants
|
|
"""
|
|
pattern = re.compile(r"\|\s+\[(?P<domain>(?:[^\|]|\\\|)+)\]" # [example.com] or even [example.com OfFiCiAl]
|
|
r"\((?P<url>https?:\/\/(?:www\.)?[^\)]+)\)\s+\|" # (url). Accepts anything like "http://www...grg", no checks
|
|
r"\s+(?P<cloudflare>(?:[^\|]|\\\|)+)\s+\|" # All 4 columns below accepts all symbols except "|", but accepts "\|" (escaped "|" in .MD)
|
|
r"\s+(?P<country>(?:[^\|]|\\\|)+)\s+\|"
|
|
r"\s+(?P<isp>(?:[^\|]|\\\|)+)\s+\|"
|
|
r"\s+(?P<branch>(?:[^\|]|\\\|)+)\s+\|",
|
|
flags=re.MULTILINE | re.IGNORECASE)
|
|
print("Getting MD")
|
|
r = requests.get(url, headers=HEADERS)
|
|
text = r.text
|
|
|
|
# Cropping Instances tables only
|
|
crop_from_index = text.index(README_CROP[0])+len(README_CROP[0]) if README_CROP[0] is not None else 0
|
|
crop_to_index = text[crop_from_index:].index(README_CROP[1]) + crop_from_index if README_CROP[1] is not None else len(text)
|
|
instances_str = text[crop_from_index:crop_to_index]
|
|
|
|
for data in pattern.finditer(instances_str):
|
|
yield from_tuple_to_instance_json(data.groups())
|
|
|
|
|
|
def get_version(instance_url: str):
|
|
instance_url = instance_url.strip("/") # Be sure there no "/" on end
|
|
versions_url = instance_url + "/api/v1/version"
|
|
try:
|
|
r = requests.get(versions_url, headers=HEADERS)
|
|
if r.status_code != 200:
|
|
print("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code))
|
|
return None
|
|
elif r.headers['Content-Type'] != 'application/json':
|
|
print("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type']))
|
|
return None
|
|
except Exception as e:
|
|
print("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e)))
|
|
return None
|
|
return r.json()
|
|
|
|
|
|
def main():
|
|
instances_json = list()
|
|
for instance_data in get_info_from_readme_iter(README_URL):
|
|
print("Scraping {} instance...".format(instance_data['link']))
|
|
versions = get_version(instance_data['link'])
|
|
if versions is None:
|
|
continue
|
|
instance_data.update(versions)
|
|
instances_json.append(instance_data)
|
|
print("Scraping finished. Saving JSON...")
|
|
|
|
# save JSON
|
|
with open('instances.json', 'w') as outfile:
|
|
json.dump(instances_json, outfile, indent=4)
|
|
print("File saved as instances.json")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|