mirror of
https://codeberg.org/gothub/gothub-instances
synced 2024-12-06 19:16:17 +01:00
Odyssey
57 lines
1.4 KiB
Python
57 lines
1.4 KiB
Python
import requests
|
|
import json
|
|
from bs4 import BeautifulSoup
|
|
|
|
print("Getting HTML")
|
|
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 GotHubInstanceFetcher/1.0 (+codeberg.org/gothub/gothub-instances)'
|
|
}
|
|
|
|
# Get the HTML from the page
|
|
r = requests.get('https://codeberg.org/gothub/gothub', headers=headers)
|
|
|
|
# Parse the HTML
|
|
soup = BeautifulSoup(r.text, 'html.parser')
|
|
|
|
print("Scraping started")
|
|
|
|
# Get tables
|
|
tables = soup.find_all('table')
|
|
|
|
# Get table with header 'Master Branch'
|
|
table = tables[1]
|
|
|
|
# Get all rows and columns. Skip the first row because it's the header
|
|
rows = table.find_all('tr')[1:]
|
|
|
|
theJson = []
|
|
|
|
for row in rows:
|
|
|
|
if row.find_all('td')[1].text == 'Yes':
|
|
cloudflare = True
|
|
else:
|
|
cloudflare = False
|
|
|
|
r = requests.get(row.find_all('td')[0].find('a')['href'] + '/api/v1/version', headers=headers)
|
|
data = json.loads(r.text)
|
|
|
|
|
|
theJson.append({
|
|
'country': row.find_all('td')[2].find('span', class_="emoji")['aria-label'][6:],
|
|
'link': row.find_all('td')[0].find('a')['href'],
|
|
'cloudflare': cloudflare,
|
|
'host': row.find_all('td')[3].text,
|
|
'branch': row.find_all('td')[4].text,
|
|
'version': data['version'],
|
|
'fiberversion': data['fiberversion'],
|
|
'goversion': data['goversion'],
|
|
})
|
|
|
|
print("Scraping finished. Saving JSON...")
|
|
|
|
# save JSON
|
|
with open('instances.json', 'w') as outfile:
|
|
json.dump(theJson, outfile)
|
|
print("File saved as instances.json") |