Add more comments for regex pattern

This commit is contained in:
NoPlagiarism
2023-05-17 15:07:14 +05:00
parent 29b7b0544b
commit 5a1944a06d
+23 -4
View File
@@ -23,18 +23,37 @@ def from_tuple_to_instance_json(data: tuple) -> dict: # data in the same order,
def get_info_from_readme_iter(url: str):
pattern = re.compile(r"\|\s+\[(?P<domain>(?:[^\|]|\\\|)+)\]" # [example.com] or even [example.com OfFiCiAl]
"""Some info about pattern
TL:DR:
- Learn regex - https://github.com/ziishaned/learn-regex/blob/master/README.md
- Markdown Cheatsheet - https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#tables
- Regex example on regex101 - https://regex101.com/r/OpcjyY/1
The basis of the code below is scraping raw .MD table, that has **valid** information
Table has 5 columns: | [domain or name](link) | cloudflare (usually "Yes" or "No") | country | isp | branch |
If we need match `| any information |` we need to match borders, spaces before and after info, so it would be like
`\|\s+[(?P<info>(?:[^\|]|\\\|)+)\s+\|`.
(?:[^\|]|\\\|)+ - means any character except border of column, excluding escaped character
Only column regex pattern's checking is [domain](url) (It's md link), specifically "url" part
URL must start with "https://" (or "http://") and can't contain ")"
To escape matching foreign table columns table is being cropped using README_CROP constants
"""
pattern = re.compile(r"\|\s+\[(?P<domain>(?:[^\|]|\\\|)+)\]" # [example.com] or even [example.com OfFiCiAl]
r"\((?P<url>https?:\/\/(?:www\.)?[^\)]+)\)\s+\|" # (url). Accepts anything like "http://www...grg", no checks
r"\s+(?P<cloudflare>(?:[^\|]|\\\|)+)\s+\|" # All 4 columns below accepts all symbols except "|", but accepts "\|"
r"\s+(?P<cloudflare>(?:[^\|]|\\\|)+)\s+\|" # All 4 columns below accepts all symbols except "|", but accepts "\|" (escaped "|" in .MD)
r"\s+(?P<country>(?:[^\|]|\\\|)+)\s+\|"
r"\s+(?P<isp>(?:[^\|]|\\\|)+)\s+\|"
r"\s+(?P<branch>(?:[^\|]|\\\|)+)\|",
r"\s+(?P<branch>(?:[^\|]|\\\|)+)\s+\|",
flags=re.MULTILINE | re.IGNORECASE)
print("Getting MD")
r = requests.get(url, headers=HEADERS)
text = r.text
# Cropping Instances only
# Cropping Instances tables only
crop_from_index = text.index(README_CROP[0])+len(README_CROP[0]) if README_CROP[0] is not None else 0
crop_to_index = text[crop_from_index:].index(README_CROP[1]) + crop_from_index if README_CROP[1] is not None else len(text)
instances_str = text[crop_from_index:crop_to_index]