mirror of
https://codeberg.org/gothub/gothub-instances
synced 2024-12-06 19:16:17 +01:00
Merge pull request 'Recode using regex' (#4) from NoPlagiarism/gothub-instances:regex_recode into master
Reviewed-on: https://codeberg.org/gothub/gothub-instances/pulls/4
This commit is contained in:
Arya K
@@ -1,76 +1,110 @@
|
||||
#!/usr/bin/python3
|
||||
import requests
|
||||
import json
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
|
||||
print("Getting HTML")
|
||||
|
||||
headers = {
|
||||
HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 GotHubInstanceFetcher/1.0 (+codeberg.org/gothub/gothub-instances)'
|
||||
}
|
||||
|
||||
# Get the HTML from the page
|
||||
r = requests.get('https://codeberg.org/gothub/gothub', headers=headers)
|
||||
README_URL = "https://codeberg.org/gothub/gothub/raw/branch/dev/README.md" # Only raw .md file!!!
|
||||
README_CROP = ("## Instances", "\n## ") # Crop from instances header to next header 2. (None, None) - to fully disable crop
|
||||
|
||||
# Parse the HTML
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
SORT_KEYS = ("country", "link", "cloudflare", "host", "branch", "version", "fiberversion", "goversion") # If key's not here, then it goes upper
|
||||
|
||||
print("Scraping started")
|
||||
|
||||
# Get tables
|
||||
tables = soup.find_all('table')
|
||||
def from_tuple_to_instance_json(data: tuple) -> dict: # data in the same order, as in regex
|
||||
res = {
|
||||
'country': data[3],
|
||||
'link': data[1],
|
||||
'cloudflare': data[2] == 'Yes',
|
||||
'host': data[4],
|
||||
'branch': data[5]
|
||||
}
|
||||
return res
|
||||
|
||||
# Get table with header 'Master Branch'
|
||||
table = tables[2]
|
||||
|
||||
# Get all rows and columns. Skip the first row because it's the header
|
||||
rows = table.find_all('tr')[1:]
|
||||
def sort_instance_json(data: dict) -> dict:
|
||||
res = dict()
|
||||
for key in sorted(data.keys(), key=lambda x: SORT_KEYS.index(x) if x in SORT_KEYS else -1):
|
||||
res[key] = data[key]
|
||||
return res
|
||||
|
||||
theJson = []
|
||||
|
||||
for row in rows:
|
||||
def get_info_from_readme_iter(url: str):
|
||||
"""Some info about pattern
|
||||
|
||||
link = row.find_all('td')[0].find('a')['href']
|
||||
cloudflare = row.find_all('td')[1].text
|
||||
country = row.find_all('td')[2].text
|
||||
host = row.find_all('td')[3].text
|
||||
branch = row.find_all('td')[4].text
|
||||
TL:DR:
|
||||
- Learn regex - https://github.com/ziishaned/learn-regex/blob/master/README.md
|
||||
- Markdown Cheatsheet - https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#tables
|
||||
- Regex example on regex101 - https://regex101.com/r/OpcjyY/1
|
||||
|
||||
print("Scraping " + row.find_all('td')[0].find('a')['href'] + ' instance...')
|
||||
if cloudflare == 'Yes':
|
||||
isCloudflare = True
|
||||
else:
|
||||
isCloudflare = False
|
||||
The basis of the code below is scraping raw .MD table, that has **valid** information
|
||||
Table has 5 columns: | [domain or name](link) | cloudflare (usually "Yes" or "No") | country | isp | branch |
|
||||
|
||||
If we need match `| any information |` we need to match borders, spaces before and after info, so it would be like
|
||||
`\|\s+[(?P<info>(?:[^\|]|\\\|)+)\s+\|`.
|
||||
(?:[^\|]|\\\|)+ - means any character except border of column, excluding escaped character
|
||||
|
||||
Only column regex pattern's checking is [domain](url) (It's md link), specifically "url" part
|
||||
URL must start with "https://" (or "http://") and can't contain ")"
|
||||
|
||||
To escape matching foreign table columns table is being cropped using README_CROP constants
|
||||
"""
|
||||
pattern = re.compile(r"\|\s+\[(?P<domain>(?:[^\|]|\\\|)+)\]" # [example.com] or even [example.com OfFiCiAl]
|
||||
r"\((?P<url>https?:\/\/(?:www\.)?[^\)]+)\)\s+\|" # (url). Accepts anything like "http://www...grg", no checks
|
||||
r"\s+(?P<cloudflare>(?:[^\|]|\\\|)+)\s+\|" # All 4 columns below accepts all symbols except "|", but accepts "\|" (escaped "|" in .MD)
|
||||
r"\s+(?P<country>(?:[^\|]|\\\|)+)\s+\|"
|
||||
r"\s+(?P<isp>(?:[^\|]|\\\|)+)\s+\|"
|
||||
r"\s+(?P<branch>(?:[^\|]|\\\|)+)\s+\|",
|
||||
flags=re.MULTILINE | re.IGNORECASE)
|
||||
print("Getting MD")
|
||||
r = requests.get(url, headers=HEADERS)
|
||||
text = r.text
|
||||
|
||||
# Cropping Instances tables only
|
||||
crop_from_index = text.index(README_CROP[0])+len(README_CROP[0]) if README_CROP[0] is not None else 0
|
||||
crop_to_index = text[crop_from_index:].index(README_CROP[1]) + crop_from_index if README_CROP[1] is not None else len(text)
|
||||
instances_str = text[crop_from_index:crop_to_index]
|
||||
|
||||
for data in pattern.finditer(instances_str):
|
||||
yield from_tuple_to_instance_json(data.groups())
|
||||
|
||||
|
||||
def get_version(instance_url: str):
|
||||
instance_url = instance_url.strip("/") # Be sure there no "/" on end
|
||||
versions_url = instance_url + "/api/v1/version"
|
||||
try:
|
||||
r = requests.get(link + '/api/v1/version', headers=headers)
|
||||
r = requests.get(versions_url, headers=HEADERS)
|
||||
if r.status_code != 200:
|
||||
print("Error while fetching " + link + '/api/v1/version. We got a ' + str(r.status_code) + ' status code. Skipping...')
|
||||
print("Error while fetching {url}. We got {code} status code. Skipping...".format(url=versions_url, code=r.status_code))
|
||||
return None
|
||||
elif r.headers['Content-Type'] != 'application/json':
|
||||
print("Error while fetching {url}. We got {c_type} content type. Skipping...".format(url=versions_url, c_type=r.headers['Content-Type']))
|
||||
return None
|
||||
except Exception as e:
|
||||
print("Error while fetching {url}. We got {err} error. Skipping...".format(url=versions_url, err=str(e)))
|
||||
return None
|
||||
return r.json()
|
||||
|
||||
|
||||
def main():
|
||||
instances_json = list()
|
||||
for instance_data in get_info_from_readme_iter(README_URL):
|
||||
print("Scraping {} instance...".format(instance_data['link']))
|
||||
versions = get_version(instance_data['link'])
|
||||
if versions is None:
|
||||
continue
|
||||
if r.headers['Content-Type'] != 'application/json':
|
||||
print("Error while fetching " + link + '/api/v1/version. We got a ' + r.headers['Content-Type'] + ' content type. Skipping...')
|
||||
continue
|
||||
except:
|
||||
print("Error while fetching " + link + '/api/v1/version. Skipping...')
|
||||
continue
|
||||
instance_data.update(versions)
|
||||
instance_data = sort_instance_json(instance_data)
|
||||
instances_json.append(instance_data)
|
||||
print("Scraping finished. Saving JSON...")
|
||||
|
||||
data = json.loads(r.text)
|
||||
|
||||
theJson.append({
|
||||
'country': country,
|
||||
'link': link,
|
||||
'cloudflare': isCloudflare,
|
||||
'host': host,
|
||||
'branch': branch,
|
||||
'version': data['version'],
|
||||
'fiberversion': data['fiberversion'],
|
||||
'goversion': data['goversion'],
|
||||
})
|
||||
# save JSON
|
||||
with open('instances.json', 'w') as outfile:
|
||||
json.dump(instances_json, outfile, indent=4)
|
||||
print("File saved as instances.json")
|
||||
|
||||
|
||||
print("Scraping finished. Saving JSON...")
|
||||
|
||||
# save JSON
|
||||
with open('instances.json', 'w') as outfile:
|
||||
json.dump(theJson, outfile, indent=4)
|
||||
print("File saved as instances.json")
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
beautifulsoup4==4.11.2
|
||||
bs4==0.0.1
|
||||
certifi==2022.12.7
|
||||
charset-normalizer==3.0.1
|
||||
idna==3.4
|
||||
requests==2.28.2
|
||||
soupsieve==2.3.2.post1
|
||||
urllib3==1.26.14
|
||||
|
||||
Reference in New Issue
Block a user