mirror of
https://codeberg.org/gothub/gothub-instances
synced 2024-12-06 19:16:17 +01:00
Add more comments for regex pattern
This commit is contained in:
@@ -23,18 +23,37 @@ def from_tuple_to_instance_json(data: tuple) -> dict: # data in the same order,
|
||||
|
||||
|
||||
def get_info_from_readme_iter(url: str):
|
||||
pattern = re.compile(r"\|\s+\[(?P<domain>(?:[^\|]|\\\|)+)\]" # [example.com] or even [example.com OfFiCiAl]
|
||||
"""Some info about pattern
|
||||
|
||||
TL:DR:
|
||||
- Learn regex - https://github.com/ziishaned/learn-regex/blob/master/README.md
|
||||
- Markdown Cheatsheet - https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#tables
|
||||
- Regex example on regex101 - https://regex101.com/r/OpcjyY/1
|
||||
|
||||
The basis of the code below is scraping raw .MD table, that has **valid** information
|
||||
Table has 5 columns: | [domain or name](link) | cloudflare (usually "Yes" or "No") | country | isp | branch |
|
||||
|
||||
If we need match `| any information |` we need to match borders, spaces before and after info, so it would be like
|
||||
`\|\s+[(?P<info>(?:[^\|]|\\\|)+)\s+\|`.
|
||||
(?:[^\|]|\\\|)+ - means any character except border of column, excluding escaped character
|
||||
|
||||
Only column regex pattern's checking is [domain](url) (It's md link), specifically "url" part
|
||||
URL must start with "https://" (or "http://") and can't contain ")"
|
||||
|
||||
To escape matching foreign table columns table is being cropped using README_CROP constants
|
||||
"""
|
||||
pattern = re.compile(r"\|\s+\[(?P<domain>(?:[^\|]|\\\|)+)\]" # [example.com] or even [example.com OfFiCiAl]
|
||||
r"\((?P<url>https?:\/\/(?:www\.)?[^\)]+)\)\s+\|" # (url). Accepts anything like "http://www...grg", no checks
|
||||
r"\s+(?P<cloudflare>(?:[^\|]|\\\|)+)\s+\|" # All 4 columns below accepts all symbols except "|", but accepts "\|"
|
||||
r"\s+(?P<cloudflare>(?:[^\|]|\\\|)+)\s+\|" # All 4 columns below accepts all symbols except "|", but accepts "\|" (escaped "|" in .MD)
|
||||
r"\s+(?P<country>(?:[^\|]|\\\|)+)\s+\|"
|
||||
r"\s+(?P<isp>(?:[^\|]|\\\|)+)\s+\|"
|
||||
r"\s+(?P<branch>(?:[^\|]|\\\|)+)\|",
|
||||
r"\s+(?P<branch>(?:[^\|]|\\\|)+)\s+\|",
|
||||
flags=re.MULTILINE | re.IGNORECASE)
|
||||
print("Getting MD")
|
||||
r = requests.get(url, headers=HEADERS)
|
||||
text = r.text
|
||||
|
||||
# Cropping Instances only
|
||||
# Cropping Instances tables only
|
||||
crop_from_index = text.index(README_CROP[0])+len(README_CROP[0]) if README_CROP[0] is not None else 0
|
||||
crop_to_index = text[crop_from_index:].index(README_CROP[1]) + crop_from_index if README_CROP[1] is not None else len(text)
|
||||
instances_str = text[crop_from_index:crop_to_index]
|
||||
|
||||
Reference in New Issue
Block a user