mirror of
https://github.com/Viren070/MediaFusion.git
synced 2025-12-01 23:21:11 +01:00
Migrate mhdtvplay scraping with scrapy & refactorings
This commit is contained in:
mhdzumair
+2
-1
@@ -39,4 +39,5 @@ resources/poster_cache
|
||||
deployment
|
||||
prowlarr-config/
|
||||
config/
|
||||
mediafusion.local*
|
||||
mediafusion.local*
|
||||
.scrapy
|
||||
|
||||
@@ -38,6 +38,7 @@ gunicorn = "*"
|
||||
scrapy = "*"
|
||||
aioqbt = {git = "git+https://github.com/mhdzumair/aioqbt.git"}
|
||||
aiowebdav = "*"
|
||||
scrapy-playwright = "*"
|
||||
|
||||
[dev-packages]
|
||||
pysocks = "*"
|
||||
|
||||
Generated
+60
-51
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "c4f0cccc167e5c86cc664635228cca7445feea11d2c58ebefd1d1a3cbbadb7b9"
|
||||
"sha256": "3a0f97b805a09c2667742db064f8c798e0e4353f1ce3207c9628116ffad1b501"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@@ -1871,6 +1871,15 @@
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==2.11.1"
|
||||
},
|
||||
"scrapy-playwright": {
|
||||
"hashes": [
|
||||
"sha256:26947ee32a9f837d325da4fe0bcf8384e2c097e12de430283aeb550754d69fca",
|
||||
"sha256:b7e171ec517460d4ec7a8552f7561a124ec945ac9ac77eb8a6639278dc7615d7"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==0.0.34"
|
||||
},
|
||||
"seedrcc": {
|
||||
"hashes": [
|
||||
"sha256:6c41d35b49118b5e3abb5ea643e848afad68125ccd328ebcc5e8c9a11420dd5d",
|
||||
@@ -1922,58 +1931,58 @@
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:03f448ffb731b48323bda68bcc93152f751436ad6037f18a42b7e16af9e91c07",
|
||||
"sha256:0de1263aac858f288a80b2071990f02082c51d88335a1db0d589237a3435fe71",
|
||||
"sha256:0fb3bffc0ced37e5aa4ac2416f56d6d858f46d4da70c09bb731a246e70bff4d5",
|
||||
"sha256:120af1e49d614d2525ac247f6123841589b029c318b9afbfc9e2b70e22e1827d",
|
||||
"sha256:1306102f6d9e625cebaca3d4c9c8f10588735ef877f0360b5cdb4fdfd3fd7131",
|
||||
"sha256:15e19a84b84528f52a68143439d0c7a3a69befcd4f50b8ef9b7b69d2628ae7c4",
|
||||
"sha256:1ab4e0448018d01b142c916cc7119ca573803a4745cfe341b8f95657812700ac",
|
||||
"sha256:1fc19ae2e07a067663dd24fca55f8ed06a288384f0e6e3910420bf4b1270cc51",
|
||||
"sha256:2f5c9dfb0b9ab5e3a8a00249534bdd838d943ec4cfb9abe176a6c33408430230",
|
||||
"sha256:30d81cc1192dc693d49d5671cd40cdec596b885b0ce3b72f323888ab1c3863d5",
|
||||
"sha256:33e8bde8fff203de50399b9039c4e14e42d4d227759155c21f8da4a47fc8053c",
|
||||
"sha256:4535c49d961fe9a77392e3a630a626af5baa967172d42732b7a43496c8b28876",
|
||||
"sha256:48217be1de7d29a5600b5c513f3f7664b21d32e596d69582be0a94e36b8309cb",
|
||||
"sha256:5ada0438f5b74c3952d916c199367c29ee4d6858edff18eab783b3978d0db16d",
|
||||
"sha256:5b78aa9f4f68212248aaf8943d84c0ff0f74efc65a661c2fc68b82d498311fd5",
|
||||
"sha256:5cd20f58c29bbf2680039ff9f569fa6d21453fbd2fa84dbdb4092f006424c2e6",
|
||||
"sha256:611068511b5531304137bcd7fe8117c985d1b828eb86043bd944cebb7fae3910",
|
||||
"sha256:680b9a36029b30cf063698755d277885d4a0eab70a2c7c6e71aab601323cba45",
|
||||
"sha256:6c5bad7c60a392850d2f0fee8f355953abaec878c483dd7c3836e0089f046bf6",
|
||||
"sha256:6c7a596d0be71b7baa037f4ac10d5e057d276f65a9a611c46970f012752ebf2d",
|
||||
"sha256:7f470327d06400a0aa7926b375b8e8c3c31d335e0884f509fe272b3c700a7254",
|
||||
"sha256:86a6ed69a71fe6b88bf9331594fa390a2adda4a49b5c06f98e47bf0d392534f8",
|
||||
"sha256:8dfc936870507da96aebb43e664ae3a71a7b96278382bcfe84d277b88e379b18",
|
||||
"sha256:954d9735ee9c3fa74874c830d089a815b7b48df6f6b6e357a74130e478dbd951",
|
||||
"sha256:9e56afce6431450442f3ab5973156289bd5ec33dd618941283847c9fd5ff06bf",
|
||||
"sha256:a3012ab65ea42de1be81fff5fb28d6db893ef978950afc8130ba707179b4284a",
|
||||
"sha256:ad862295ad3f644e3c2c0d8b10a988e1600d3123ecb48702d2c0f26771f1c396",
|
||||
"sha256:b1d9d1bfd96eef3c3faedb73f486c89e44e64e40e5bfec304ee163de01cf996f",
|
||||
"sha256:b86abba762ecfeea359112b2bb4490802b340850bbee1948f785141a5e020de8",
|
||||
"sha256:b90053be91973a6fb6020a6e44382c97739736a5a9d74e08cc29b196639eb979",
|
||||
"sha256:c4fbe6a766301f2e8a4519f4500fe74ef0a8509a59e07a4085458f26228cd7cc",
|
||||
"sha256:ca891af9f3289d24a490a5fde664ea04fe2f4984cd97e26de7442a4251bd4b7c",
|
||||
"sha256:cb0845e934647232b6ff5150df37ceffd0b67b754b9fdbb095233deebcddbd4a",
|
||||
"sha256:ce850db091bf7d2a1f2fdb615220b968aeff3849007b1204bf6e3e50a57b3d32",
|
||||
"sha256:d04e579e911562f1055d26dab1868d3e0bb905db3bccf664ee8ad109f035618a",
|
||||
"sha256:d07ee7793f2aeb9b80ec8ceb96bc8cc08a2aec8a1b152da1955d64e4825fcbac",
|
||||
"sha256:d177b7e82f6dd5e1aebd24d9c3297c70ce09cd1d5d37b43e53f39514379c029c",
|
||||
"sha256:d7b5a3e2120982b8b6bd1d5d99e3025339f7fb8b8267551c679afb39e9c7c7f1",
|
||||
"sha256:d873c21b356bfaf1589b89090a4011e6532582b3a8ea568a00e0c3aab09399dd",
|
||||
"sha256:d997c5938a08b5e172c30583ba6b8aad657ed9901fc24caf3a7152eeccb2f1b4",
|
||||
"sha256:dbcd77c4d94b23e0753c5ed8deba8c69f331d4fd83f68bfc9db58bc8983f49cd",
|
||||
"sha256:e36aa62b765cf9f43a003233a8c2d7ffdeb55bc62eaa0a0380475b228663a38f",
|
||||
"sha256:e97cf143d74a7a5a0f143aa34039b4fecf11343eed66538610debc438685db4a",
|
||||
"sha256:eb15ef40b833f5b2f19eeae65d65e191f039e71790dd565c2af2a3783f72262f",
|
||||
"sha256:ec1f5a328464daf7a1e4e385e4f5652dd9b1d12405075ccba1df842f7774b4fc",
|
||||
"sha256:f9374e270e2553653d710ece397df67db9d19c60d2647bcd35bfc616f1622dcd",
|
||||
"sha256:fa67d821c1fd268a5a87922ef4940442513b4e6c377553506b9db3b83beebbd8",
|
||||
"sha256:fd8aafda7cdff03b905d4426b714601c0978725a19efc39f5f207b86d188ba01",
|
||||
"sha256:ff2f1b7c963961d41403b650842dc2039175b906ab2093635d8319bef0b7d620"
|
||||
"sha256:0315d9125a38026227f559488fe7f7cee1bd2fbc19f9fd637739dc50bb6380b2",
|
||||
"sha256:0d3dd67b5d69794cfe82862c002512683b3db038b99002171f624712fa71aeaa",
|
||||
"sha256:124202b4e0edea7f08a4db8c81cc7859012f90a0d14ba2bf07c099aff6e96462",
|
||||
"sha256:1ee8bd6d68578e517943f5ebff3afbd93fc65f7ef8f23becab9fa8fb315afb1d",
|
||||
"sha256:243feb6882b06a2af68ecf4bec8813d99452a1b62ba2be917ce6283852cf701b",
|
||||
"sha256:2858bbab1681ee5406650202950dc8f00e83b06a198741b7c656e63818633526",
|
||||
"sha256:2f60843068e432311c886c5f03c4664acaef507cf716f6c60d5fde7265be9d7b",
|
||||
"sha256:328529f7c7f90adcd65aed06a161851f83f475c2f664a898af574893f55d9e53",
|
||||
"sha256:33157920b233bc542ce497a81a2e1452e685a11834c5763933b440fedd1d8e2d",
|
||||
"sha256:3eba73ef2c30695cb7eabcdb33bb3d0b878595737479e152468f3ba97a9c22a4",
|
||||
"sha256:426f2fa71331a64f5132369ede5171c52fd1df1bd9727ce621f38b5b24f48750",
|
||||
"sha256:45c7b78dfc7278329f27be02c44abc0d69fe235495bb8e16ec7ef1b1a17952db",
|
||||
"sha256:46a3d4e7a472bfff2d28db838669fc437964e8af8df8ee1e4548e92710929adc",
|
||||
"sha256:4a5adf383c73f2d49ad15ff363a8748319ff84c371eed59ffd0127355d6ea1da",
|
||||
"sha256:4b6303bfd78fb3221847723104d152e5972c22367ff66edf09120fcde5ddc2e2",
|
||||
"sha256:56856b871146bfead25fbcaed098269d90b744eea5cb32a952df00d542cdd368",
|
||||
"sha256:5da98815f82dce0cb31fd1e873a0cb30934971d15b74e0d78cf21f9e1b05953f",
|
||||
"sha256:5df5d1dafb8eee89384fb7a1f79128118bc0ba50ce0db27a40750f6f91aa99d5",
|
||||
"sha256:68722e6a550f5de2e3cfe9da6afb9a7dd15ef7032afa5651b0f0c6b3adb8815d",
|
||||
"sha256:78bb7e8da0183a8301352d569900d9d3594c48ac21dc1c2ec6b3121ed8b6c986",
|
||||
"sha256:81ba314a08c7ab701e621b7ad079c0c933c58cdef88593c59b90b996e8b58fa5",
|
||||
"sha256:843a882cadebecc655a68bd9a5b8aa39b3c52f4a9a5572a3036fb1bb2ccdc197",
|
||||
"sha256:87724e7ed2a936fdda2c05dbd99d395c91ea3c96f029a033a4a20e008dd876bf",
|
||||
"sha256:8c7f10720fc34d14abad5b647bc8202202f4948498927d9f1b4df0fb1cf391b7",
|
||||
"sha256:8e91b5e341f8c7f1e5020db8e5602f3ed045a29f8e27f7f565e0bdee3338f2c7",
|
||||
"sha256:943aa74a11f5806ab68278284a4ddd282d3fb348a0e96db9b42cb81bf731acdc",
|
||||
"sha256:9461802f2e965de5cff80c5a13bc945abea7edaa1d29360b485c3d2b56cdb075",
|
||||
"sha256:9b66fcd38659cab5d29e8de5409cdf91e9986817703e1078b2fdaad731ea66f5",
|
||||
"sha256:a6bec1c010a6d65b3ed88c863d56b9ea5eeefdf62b5e39cafd08c65f5ce5198b",
|
||||
"sha256:a921002be69ac3ab2cf0c3017c4e6a3377f800f1fca7f254c13b5f1a2f10022c",
|
||||
"sha256:aca7b6d99a4541b2ebab4494f6c8c2f947e0df4ac859ced575238e1d6ca5716b",
|
||||
"sha256:ad7acbe95bac70e4e687a4dc9ae3f7a2f467aa6597049eeb6d4a662ecd990bb6",
|
||||
"sha256:af8ce2d31679006e7b747d30a89cd3ac1ec304c3d4c20973f0f4ad58e2d1c4c9",
|
||||
"sha256:b4a2cf92995635b64876dc141af0ef089c6eea7e05898d8d8865e71a326c0385",
|
||||
"sha256:bbda76961eb8f27e6ad3c84d1dc56d5bc61ba8f02bd20fcf3450bd421c2fcc9c",
|
||||
"sha256:bd7e4baf9161d076b9a7e432fce06217b9bd90cfb8f1d543d6e8c4595627edb9",
|
||||
"sha256:bea30da1e76cb1acc5b72e204a920a3a7678d9d52f688f087dc08e54e2754c67",
|
||||
"sha256:c61e2e41656a673b777e2f0cbbe545323dbe0d32312f590b1bc09da1de6c2a02",
|
||||
"sha256:c6c4da4843e0dabde41b8f2e8147438330924114f541949e6318358a56d1875a",
|
||||
"sha256:d3499008ddec83127ab286c6f6ec82a34f39c9817f020f75eca96155f9765097",
|
||||
"sha256:dbb990612c36163c6072723523d2be7c3eb1517bbdd63fe50449f56afafd1133",
|
||||
"sha256:dd53b6c4e6d960600fd6532b79ee28e2da489322fcf6648738134587faf767b6",
|
||||
"sha256:df40c16a7e8be7413b885c9bf900d402918cc848be08a59b022478804ea076b8",
|
||||
"sha256:e0a5354cb4de9b64bccb6ea33162cb83e03dbefa0d892db88a672f5aad638a75",
|
||||
"sha256:e0b148ab0438f72ad21cb004ce3bdaafd28465c4276af66df3b9ecd2037bf252",
|
||||
"sha256:e23b88c69497a6322b5796c0781400692eca1ae5532821b39ce81a48c395aae9",
|
||||
"sha256:fc4974d3684f28b61b9a90fcb4c41fb340fd4b6a50c04365704a4da5a9603b05",
|
||||
"sha256:feea693c452d85ea0015ebe3bb9cd15b6f49acc1a31c28b3c50f4db0f8fb1e71",
|
||||
"sha256:fffcc8edc508801ed2e6a4e7b0d150a62196fd28b4e16ab9f65192e8186102b6"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==2.0.27"
|
||||
"version": "==2.0.28"
|
||||
},
|
||||
"starlette": {
|
||||
"hashes": [
|
||||
|
||||
+70
-57
@@ -5,7 +5,7 @@ from typing import Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from beanie import WriteRules
|
||||
from beanie.operators import In
|
||||
from beanie.operators import In, Set
|
||||
from pymongo.errors import DuplicateKeyError
|
||||
from redis.asyncio import Redis
|
||||
|
||||
@@ -18,6 +18,7 @@ from db.models import (
|
||||
Season,
|
||||
Episode,
|
||||
MediaFusionTVMetaData,
|
||||
TVStreams,
|
||||
)
|
||||
from db.schemas import Stream, MetaIdProjection, TorrentStreamsList
|
||||
from scrapers import tamilmv
|
||||
@@ -77,9 +78,7 @@ async def get_meta_list(
|
||||
async def get_tv_meta_list(
|
||||
genre: Optional[str] = None, skip: int = 0, limit: int = 25
|
||||
) -> list[schemas.Meta]:
|
||||
query = MediaFusionTVMetaData.find(
|
||||
MediaFusionTVMetaData.is_approved == True, fetch_links=True
|
||||
)
|
||||
query = MediaFusionTVMetaData.find(fetch_links=True)
|
||||
if genre:
|
||||
query = query.find(In(MediaFusionTVMetaData.genres, [genre]))
|
||||
|
||||
@@ -561,66 +560,80 @@ async def get_stream_by_info_hash(info_hash: str) -> TorrentStreams | None:
|
||||
return stream
|
||||
|
||||
|
||||
async def save_tv_channel_metadata(tv_metadata: schemas.TVMetaData) -> tuple[str, bool]:
|
||||
# Try to get the existing TV channel
|
||||
async def save_tv_channel_metadata(tv_metadata: schemas.TVMetaData) -> str:
|
||||
channel_id = "mf" + hashlib.sha256(tv_metadata.title.encode()).hexdigest()[:10]
|
||||
logging.info(f"Processing TV channel id {channel_id}")
|
||||
existing_channel = await models.MediaFusionTVMetaData.get(channel_id)
|
||||
|
||||
# Map the TVStreams schema to the TVStreams model
|
||||
streams_models = []
|
||||
# Prepare the genres list
|
||||
genres = list(
|
||||
set(tv_metadata.genres + [tv_metadata.country, tv_metadata.tv_language])
|
||||
)
|
||||
|
||||
# Ensure the channel document is upserted
|
||||
try:
|
||||
await MediaFusionTVMetaData.find_one(
|
||||
MediaFusionTVMetaData.id == channel_id
|
||||
).upsert(
|
||||
Set({}), # Update operation is a no-op for now
|
||||
on_insert=MediaFusionTVMetaData(
|
||||
id=channel_id,
|
||||
title=tv_metadata.title,
|
||||
poster=tv_metadata.poster,
|
||||
background=tv_metadata.background,
|
||||
country=tv_metadata.country,
|
||||
tv_language=tv_metadata.tv_language,
|
||||
logo=tv_metadata.logo,
|
||||
genres=genres,
|
||||
type="tv",
|
||||
streams=[],
|
||||
),
|
||||
)
|
||||
except DuplicateKeyError:
|
||||
pass
|
||||
|
||||
# Stream processing
|
||||
stream_ids = []
|
||||
for stream in tv_metadata.streams:
|
||||
streams_models.append(
|
||||
models.TVStreams(
|
||||
url=stream.url,
|
||||
name=stream.name,
|
||||
behaviorHints=stream.behaviorHints.model_dump(exclude_none=True),
|
||||
ytId=stream.ytId,
|
||||
source=stream.source,
|
||||
)
|
||||
# Define stream document with meta_id
|
||||
stream_doc = TVStreams(
|
||||
url=stream.url,
|
||||
name=stream.name,
|
||||
behaviorHints=stream.behaviorHints.model_dump(exclude_none=True),
|
||||
ytId=stream.ytId,
|
||||
source=stream.source,
|
||||
country=stream.country,
|
||||
meta_id=channel_id,
|
||||
)
|
||||
|
||||
if existing_channel:
|
||||
# Check for existing streams and update if necessary
|
||||
await existing_channel.fetch_all_links()
|
||||
for new_stream in streams_models:
|
||||
matching_stream = next(
|
||||
(
|
||||
s
|
||||
for s in existing_channel.streams
|
||||
if (s.url and s.url == new_stream.url)
|
||||
or (s.ytId and s.ytId == new_stream.ytId)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not matching_stream:
|
||||
existing_channel.streams.append(new_stream)
|
||||
await existing_channel.save(link_rule=WriteRules.WRITE)
|
||||
logging.info(f"Updated TV channel {existing_channel.title}")
|
||||
is_new = False
|
||||
else:
|
||||
tv_metadata.genres.extend([tv_metadata.country, tv_metadata.tv_language])
|
||||
genres = list(set(tv_metadata.genres))
|
||||
|
||||
# If the channel doesn't exist, create a new one
|
||||
tv_channel_data = models.MediaFusionTVMetaData(
|
||||
id=channel_id,
|
||||
title=tv_metadata.title,
|
||||
poster=tv_metadata.poster,
|
||||
background=tv_metadata.background,
|
||||
streams=streams_models,
|
||||
type="tv",
|
||||
country=tv_metadata.country,
|
||||
tv_language=tv_metadata.tv_language,
|
||||
logo=tv_metadata.logo,
|
||||
genres=genres,
|
||||
is_approved=False,
|
||||
# Check if the stream exists (by URL or ytId) and upsert accordingly
|
||||
existing_stream = await TVStreams.find_one(
|
||||
TVStreams.url == stream.url,
|
||||
TVStreams.ytId == stream.ytId,
|
||||
)
|
||||
await tv_channel_data.insert(link_rule=WriteRules.WRITE)
|
||||
logging.info(f"Added TV channel {tv_channel_data.title}")
|
||||
is_new = True
|
||||
if existing_stream:
|
||||
stream_ids.append(existing_stream.id)
|
||||
else:
|
||||
inserted_stream = await stream_doc.insert()
|
||||
stream_ids.append(inserted_stream.id)
|
||||
|
||||
return channel_id, is_new
|
||||
# Update the TV channel with new stream links, if there are any new streams
|
||||
if stream_ids:
|
||||
await MediaFusionTVMetaData.find_one(
|
||||
MediaFusionTVMetaData.id == channel_id
|
||||
).update(
|
||||
{
|
||||
"$addToSet": {
|
||||
"streams": {
|
||||
"$each": [
|
||||
TVStreams.link_from_id(stream_id)
|
||||
for stream_id in stream_ids
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
logging.info(f"Processed TV channel {tv_metadata.title}")
|
||||
return channel_id
|
||||
|
||||
|
||||
async def delete_search_history():
|
||||
|
||||
+1
-1
@@ -62,6 +62,7 @@ class TVStreams(Document):
|
||||
behaviorHints: dict[str, Any] | None = None
|
||||
created_at: datetime = Field(default_factory=datetime.now)
|
||||
meta_id: Optional[str] = None
|
||||
country: str | None = None
|
||||
|
||||
|
||||
class MediaFusionMetaData(Document):
|
||||
@@ -96,7 +97,6 @@ class MediaFusionTVMetaData(MediaFusionMetaData):
|
||||
tv_language: str
|
||||
logo: Optional[str] = None
|
||||
genres: Optional[list[str]] = None
|
||||
is_approved: bool = False
|
||||
streams: list[Link[TVStreams]]
|
||||
|
||||
|
||||
|
||||
+3
-2
@@ -172,6 +172,7 @@ class TVStreams(BaseModel):
|
||||
url: str | None = None
|
||||
ytId: str | None = None
|
||||
source: str
|
||||
country: str | None = None
|
||||
behaviorHints: TVStreamsBehaviorHints | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
@@ -185,8 +186,8 @@ class TVMetaData(BaseModel):
|
||||
title: str
|
||||
poster: str | None = None
|
||||
background: Optional[str] = None
|
||||
country: str
|
||||
tv_language: str
|
||||
country: str | None = None
|
||||
tv_language: str | None = None
|
||||
logo: Optional[str] = None
|
||||
genres: list[str] = []
|
||||
streams: list[TVStreams]
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
# See documentation in:
|
||||
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
from scrapy import signals
|
||||
|
||||
# useful for handling different item types with a single interface
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
import aiohttp
|
||||
from scrapy import signals
|
||||
|
||||
from db import database
|
||||
|
||||
|
||||
@@ -7,7 +7,14 @@ from beanie import WriteRules
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
from db.models import TorrentStreams, Season, MediaFusionSeriesMetaData, Episode
|
||||
from db import crud
|
||||
from db.models import (
|
||||
TorrentStreams,
|
||||
Season,
|
||||
MediaFusionSeriesMetaData,
|
||||
Episode,
|
||||
)
|
||||
from db.schemas import TVMetaData
|
||||
from utils.parser import convert_size_to_bytes
|
||||
|
||||
|
||||
@@ -257,3 +264,13 @@ class FormulaStorePipeline:
|
||||
logging.info("Updated series %s", series.title)
|
||||
|
||||
return item
|
||||
|
||||
|
||||
class TVStorePipeline:
|
||||
async def process_item(self, item, spider):
|
||||
if "title" not in item:
|
||||
logging.warning(f"title not found in item: {item}")
|
||||
raise DropItem(f"title not found in item: {item}")
|
||||
|
||||
tv_metadata = TVMetaData.model_validate(item)
|
||||
await crud.save_tv_channel_metadata(tv_metadata)
|
||||
|
||||
@@ -14,7 +14,7 @@ NEWSPIDER_MODULE = "mediafusion_scrapy.spiders"
|
||||
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
# USER_AGENT = "mediafusion_scrapy (+http://www.yourdomain.com)"
|
||||
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = False
|
||||
@@ -34,7 +34,7 @@ ROBOTSTXT_OBEY = False
|
||||
# COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
# TELNETCONSOLE_ENABLED = False
|
||||
TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
# DEFAULT_REQUEST_HEADERS = {
|
||||
@@ -81,13 +81,14 @@ AUTOTHROTTLE_ENABLED = True
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
# HTTPCACHE_ENABLED = True
|
||||
# HTTPCACHE_EXPIRATION_SECS = 0
|
||||
# HTTPCACHE_DIR = "httpcache"
|
||||
# HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
# HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
HTTPCACHE_ENABLED = True
|
||||
HTTPCACHE_EXPIRATION_SECS = 0
|
||||
HTTPCACHE_DIR = "httpcache"
|
||||
HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
|
||||
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||||
FEED_EXPORT_ENCODING = "utf-8"
|
||||
LOG_LEVEL = "INFO"
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
import re
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import scrapy
|
||||
|
||||
from scrapers.helpers import get_country_name
|
||||
|
||||
|
||||
class MhdtvworldSpider(scrapy.Spider):
|
||||
name = "mhdtvworld"
|
||||
start_urls = ["https://mhdtvmax.net/", "https://mhdtvsports.net/"]
|
||||
|
||||
direct_pattern = re.compile(r"(?:source|url): ['\"](.*?\.m3u8.*?)['\"]")
|
||||
fallback_pattern = re.compile(
|
||||
r"source: ['\"](.*?)['\"],\s*[\s\S]*?mimeType: ['\"]application/x-mpegURL['\"]"
|
||||
)
|
||||
|
||||
m3u8_valid_content_types = [
|
||||
"application/vnd.apple.mpegurl",
|
||||
"application/x-mpegurl",
|
||||
]
|
||||
|
||||
# this site sometimes returns html instead of image
|
||||
exclude_validation_urls = [
|
||||
"https://imgur.com",
|
||||
]
|
||||
|
||||
category_substrings = [
|
||||
"/channel/",
|
||||
"/live/",
|
||||
"/channels/",
|
||||
]
|
||||
|
||||
custom_settings = {
|
||||
"ITEM_PIPELINES": {
|
||||
"mediafusion_scrapy.pipelines.TVStorePipeline": 300,
|
||||
},
|
||||
}
|
||||
|
||||
def parse(self, response, **kwargs):
|
||||
category_urls = [
|
||||
urljoin(response.url, link.get())
|
||||
for link in response.css("#header a::attr(href)")
|
||||
if any(substring in link.get() for substring in self.category_substrings)
|
||||
]
|
||||
self.logger.info(f"Found {len(category_urls)} categories")
|
||||
|
||||
# Iterate over each category URL to scrape channels
|
||||
for category_url in category_urls:
|
||||
yield scrapy.Request(
|
||||
category_url,
|
||||
callback=self.parse_categories,
|
||||
)
|
||||
|
||||
def parse_categories(self, response):
|
||||
# Process the current page immediately
|
||||
yield from self.parse_page(response)
|
||||
|
||||
# Extract the total number of pages from the pagination text
|
||||
pagination_text = response.css("div.pagination span::text").get()
|
||||
total_pages = int(pagination_text.split(" ")[-1]) if pagination_text else 1
|
||||
self.logger.info(f"Found {total_pages} pages in category {response.url}")
|
||||
|
||||
base_url = response.url
|
||||
if "/page/" in response.url:
|
||||
base_url = response.url.split("/page/")[0]
|
||||
else:
|
||||
base_url = base_url.rstrip("/")
|
||||
|
||||
# Since we've already processed the first page, start from the second page
|
||||
page_urls = [f"{base_url}/page/{page}/" for page in range(2, total_pages + 1)]
|
||||
|
||||
# Iterate over each subsequent page URL to scrape channels
|
||||
for page_url in page_urls:
|
||||
yield scrapy.Request(page_url, callback=self.parse_page)
|
||||
|
||||
def parse_page(self, response):
|
||||
channel_elements = response.css("article.item.movies")
|
||||
source_name = response.css(".logo a img::attr(alt)").get(default="MHDTVWORLD")
|
||||
|
||||
for channel_element in channel_elements:
|
||||
# Extract title, poster, and stream page URL using Scrapy's CSS selectors
|
||||
title = channel_element.css("h3 > a::text").get()
|
||||
channel_page_url = channel_element.css(".poster > a::attr(href)").get()
|
||||
|
||||
channel_data = {
|
||||
"title": title,
|
||||
"channel_page_url": channel_page_url,
|
||||
"source": source_name,
|
||||
}
|
||||
|
||||
# Enqueue a request to the channel page URL to scrape M3U8 URLs
|
||||
yield scrapy.Request(
|
||||
channel_page_url,
|
||||
callback=self.parse_channel_page,
|
||||
meta={
|
||||
"channel_data": channel_data,
|
||||
},
|
||||
)
|
||||
|
||||
def parse_channel_page(self, response):
|
||||
channel_data = self.extract_channel_data(response)
|
||||
player_api_base = self.extract_player_api_base(response)
|
||||
if not player_api_base:
|
||||
self.logger.error(f"Player API base URL not found for {response.url}")
|
||||
return
|
||||
|
||||
poster = channel_data.get("poster")
|
||||
if poster:
|
||||
# The validation and subsequent actions happen in the callback.
|
||||
yield scrapy.Request(
|
||||
poster,
|
||||
callback=self.on_validate_poster,
|
||||
meta={
|
||||
"channel_data": channel_data,
|
||||
"player_api_base": player_api_base,
|
||||
"response": response, # Pass the original response to access player options later
|
||||
},
|
||||
dont_filter=True,
|
||||
)
|
||||
|
||||
def on_validate_poster(self, response):
|
||||
meta = response.meta
|
||||
original_response = meta["response"]
|
||||
channel_data = meta["channel_data"]
|
||||
player_api_base = meta["player_api_base"]
|
||||
content_type = response.headers.get("Content-Type", b"").decode().lower()
|
||||
is_image = "image" in content_type
|
||||
|
||||
is_allowed_url = any(
|
||||
url in response.url for url in self.exclude_validation_urls
|
||||
)
|
||||
|
||||
if is_image or is_allowed_url:
|
||||
yield from self.process_player_options(
|
||||
original_response, channel_data, player_api_base
|
||||
)
|
||||
else:
|
||||
self.logger.error(f"Invalid poster URL: {response.url}")
|
||||
|
||||
def extract_channel_data(self, response):
|
||||
"""Extracts channel data such as genres and poster."""
|
||||
channel_data = response.meta.get("channel_data").copy()
|
||||
poster = response.css(".poster > img::attr(src)").get()
|
||||
genres = response.css(".sgeneros a[rel='tag']::text").getall()
|
||||
channel_data.update(
|
||||
{
|
||||
"genres": genres,
|
||||
"poster": poster,
|
||||
"background": poster,
|
||||
"tv_language": genres[0] if genres else "English",
|
||||
}
|
||||
)
|
||||
return channel_data
|
||||
|
||||
def extract_player_api_base(self, response):
|
||||
"""Extracts the player API base URL."""
|
||||
return (
|
||||
response.xpath("//script[contains(text(), 'player_api')]/text()")
|
||||
.re_first(r'"player_api":"([^"]+)"', default="")
|
||||
.replace("\\/", "/")
|
||||
)
|
||||
|
||||
def extract_stream_details(self, element):
|
||||
"""Extracts stream title and country name from an element."""
|
||||
stream_title = element.css("span.title::text").get().strip()
|
||||
country_flag_url = element.css("span.flag > img::attr(src)").get()
|
||||
country_name = "India"
|
||||
if country_flag_url:
|
||||
country_code = country_flag_url.split("/")[-1].split(".")[0]
|
||||
country_name = get_country_name(country_code)
|
||||
return stream_title, country_name
|
||||
|
||||
def process_player_options(self, response, channel_data, player_api_base):
|
||||
for element in response.css("#playeroptionsul > li.dooplay_player_option"):
|
||||
yield from self.process_player_option(
|
||||
element, channel_data, player_api_base
|
||||
)
|
||||
|
||||
def process_player_option(self, element, channel_data, player_api_base):
|
||||
"""Processes each player option element to yield API request."""
|
||||
stream_title, country_name = self.extract_stream_details(element)
|
||||
data_post, data_nume, data_type = (
|
||||
element.attrib.get("data-post"),
|
||||
element.attrib.get("data-nume"),
|
||||
element.attrib.get("data-type"),
|
||||
)
|
||||
|
||||
if all([data_post, data_nume, data_type]):
|
||||
api_url = f"{player_api_base}{data_post}/{data_type}/{data_nume}"
|
||||
yield scrapy.Request(
|
||||
url=api_url,
|
||||
callback=self.parse_api_response,
|
||||
meta={
|
||||
"channel_data": channel_data,
|
||||
"stream_title": stream_title,
|
||||
"country_name": country_name,
|
||||
},
|
||||
)
|
||||
|
||||
def parse_api_response(self, response):
|
||||
channel_data = response.meta.get("channel_data")
|
||||
stream_title = response.meta.get("stream_title")
|
||||
country_name = response.meta.get("country_name")
|
||||
|
||||
# Deserialize JSON response
|
||||
api_data = response.json()
|
||||
iframe_url = urljoin(
|
||||
response.url, api_data.get("embed_url", "").replace("\\/", "/")
|
||||
)
|
||||
|
||||
if iframe_url:
|
||||
yield scrapy.Request(
|
||||
url=iframe_url,
|
||||
callback=self.request_and_extract_video_url,
|
||||
meta={
|
||||
"channel_data": channel_data,
|
||||
"stream_title": stream_title,
|
||||
"country_name": country_name,
|
||||
},
|
||||
)
|
||||
|
||||
def get_behavior_hints(self, response):
|
||||
"""Generates behavior hints for requests."""
|
||||
user_agent = response.request.headers.get("User-Agent").decode()
|
||||
parsed_url = urlparse(response.url)
|
||||
referer = f"{parsed_url.scheme}://{parsed_url.netloc}"
|
||||
|
||||
return {
|
||||
"notWebReady": True,
|
||||
"proxyHeaders": {
|
||||
"request": {
|
||||
"User-Agent": user_agent,
|
||||
"Referer": referer,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def extract_m3u8_urls(self, response):
|
||||
"""Extracts M3U8 URLs using direct and fallback regex patterns."""
|
||||
m3u8_urls = self.direct_pattern.findall(response.text)
|
||||
if not m3u8_urls:
|
||||
m3u8_urls = self.fallback_pattern.findall(response.text)
|
||||
return m3u8_urls
|
||||
|
||||
def request_and_extract_video_url(self, response):
|
||||
channel_data = response.meta.get("channel_data")
|
||||
stream_title = response.meta.get("stream_title")
|
||||
country_name = response.meta.get("country_name")
|
||||
behavior_hints = self.get_behavior_hints(response)
|
||||
|
||||
# Extract M3U8 URLs
|
||||
m3u8_urls = self.extract_m3u8_urls(response)
|
||||
if not m3u8_urls:
|
||||
self.logger.error(
|
||||
"No M3U8 URLs found for channel url: %s, stream title: %s",
|
||||
channel_data["channel_page_url"],
|
||||
stream_title,
|
||||
)
|
||||
return
|
||||
|
||||
for index, url in enumerate(m3u8_urls, 1):
|
||||
full_url = urljoin(response.url, url)
|
||||
# Instead of appending to streams_info, initiate validation request
|
||||
yield scrapy.Request(
|
||||
url=full_url,
|
||||
headers=behavior_hints["proxyHeaders"]["request"],
|
||||
callback=self.validate_m3u8_url,
|
||||
errback=self.handle_m3u8_failure,
|
||||
meta={
|
||||
"index": index if len(m3u8_urls) > 1 else None,
|
||||
"stream_title": stream_title,
|
||||
"full_url": full_url,
|
||||
"country_name": country_name,
|
||||
"channel_data": channel_data,
|
||||
"behavior_hints": behavior_hints,
|
||||
},
|
||||
dont_filter=True,
|
||||
)
|
||||
|
||||
def validate_m3u8_url(self, response):
|
||||
meta = response.meta
|
||||
content_type = response.headers.get("Content-Type", b"").decode().lower()
|
||||
|
||||
if response.status == 200 and content_type in self.m3u8_valid_content_types:
|
||||
# Content type is valid, proceed with adding the stream
|
||||
stream_info = {
|
||||
"name": f"{meta['stream_title']} - {meta['index']}"
|
||||
if meta["index"]
|
||||
else meta["stream_title"],
|
||||
"url": meta["full_url"],
|
||||
"country": meta["country_name"],
|
||||
"behaviorHints": meta["behavior_hints"],
|
||||
"source": meta["channel_data"]["source"],
|
||||
}
|
||||
# Update channel data with this stream and yield
|
||||
updated_channel_data = self.create_channel_data_with_stream(
|
||||
meta["channel_data"], stream_info, meta["country_name"]
|
||||
)
|
||||
yield updated_channel_data
|
||||
else:
|
||||
self.logger.error(
|
||||
f"Invalid M3U8 URL: {meta['full_url']} with Content-Type: {content_type}"
|
||||
)
|
||||
|
||||
def handle_m3u8_failure(self, failure):
|
||||
self.logger.error(
|
||||
"Failed to get m3u8 URL from channel page: %s stream title: %s",
|
||||
failure.request.meta["channel_data"]["channel_page_url"],
|
||||
failure.request.meta["stream_title"],
|
||||
)
|
||||
|
||||
def create_channel_data_with_stream(self, channel_data, stream_info, country_name):
|
||||
"""Create channel data with a single stream info"""
|
||||
# Copy the original channel data to avoid mutating the original meta
|
||||
channel_data_copy = channel_data.copy()
|
||||
|
||||
# Directly set the streams list to only include the current stream
|
||||
channel_data_copy["streams"] = [stream_info]
|
||||
channel_data_copy["country"] = country_name
|
||||
|
||||
return channel_data_copy
|
||||
+8
-3
@@ -10,14 +10,13 @@ from urllib3.util.retry import Retry
|
||||
|
||||
from db.config import settings
|
||||
from db.models import TorrentStreams, Episode, Season
|
||||
from utils.const import UA_HEADER
|
||||
from utils.torrent import extract_torrent_metadata, info_hashes_to_torrent_metadata
|
||||
|
||||
# set httpx logging level
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
|
||||
UA_HEADER = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
PROXIES = (
|
||||
{
|
||||
"http": settings.scraper_proxy_url,
|
||||
@@ -190,3 +189,9 @@ async def update_torrent_series_streams_metadata(info_hashes: list[str]):
|
||||
|
||||
await torrent_stream.save()
|
||||
logging.info(f"Updated {torrent_stream.id} metadata")
|
||||
|
||||
|
||||
def get_country_name(country_code):
|
||||
with open("resources/json/countries.json") as file:
|
||||
countries = json.load(file)
|
||||
return countries.get(country_code.upper(), "India")
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urlparse, urljoin, parse_qs
|
||||
|
||||
import aiohttp
|
||||
import requests
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from scrapers.helpers import get_scraper_config
|
||||
|
||||
logging.basicConfig(
|
||||
format="%(levelname)s::%(asctime)s - %(message)s", level=logging.INFO
|
||||
)
|
||||
BASE_URL = get_scraper_config("mhdtvplay", "homepage")
|
||||
MEDIAFUSION_URL = "http://127.0.0.1:8000"
|
||||
|
||||
|
||||
def get_country_name(country_code):
|
||||
with open("resources/json/countries.json") as file:
|
||||
countries = json.load(file)
|
||||
return countries.get(country_code.upper(), "India")
|
||||
|
||||
|
||||
async def extract_player_source_url(iframe_src):
|
||||
"""
|
||||
Asynchronously extracts the player source URL from the HTML response.
|
||||
|
||||
:param iframe_src: The iframe source URL.
|
||||
:return: The final player source URL or None if not found.
|
||||
"""
|
||||
headers = {
|
||||
"Referer": BASE_URL,
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
|
||||
}
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(iframe_src, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
html_response = await response.text()
|
||||
player_source_regex = re.compile(r'source: [\'"]([^\'"]+)[\'"]')
|
||||
match = player_source_regex.search(html_response)
|
||||
|
||||
if match:
|
||||
source_url = match.group(1)
|
||||
# Check if the URL is a DASH stream (MPD)
|
||||
if source_url.endswith(".mpd"):
|
||||
return None
|
||||
# Check if the URL is complete or relative
|
||||
if source_url.startswith(("http:", "https:")):
|
||||
return source_url
|
||||
else:
|
||||
# Construct the full URL from the base URL and the relative path
|
||||
parsed_iframe_src = urlparse(iframe_src)
|
||||
base_url = (
|
||||
f"{parsed_iframe_src.scheme}://{parsed_iframe_src.netloc}"
|
||||
)
|
||||
return urljoin(base_url, source_url)
|
||||
return None
|
||||
|
||||
|
||||
async def scrape_tv_channels(page):
|
||||
# Scrape channel metadata
|
||||
channels_data = []
|
||||
channel_elements = await page.query_selector_all("article.item.movies")
|
||||
|
||||
# First, store all channel information in a list
|
||||
channel_info_list = []
|
||||
for channel_element in channel_elements:
|
||||
title_element = await channel_element.query_selector("h3 > a")
|
||||
title = await title_element.text_content() if title_element else "No Title"
|
||||
poster_element = await channel_element.query_selector(".poster > img")
|
||||
poster_url = (
|
||||
await poster_element.get_attribute("src")
|
||||
if poster_element
|
||||
else "No Poster URL"
|
||||
)
|
||||
stream_page_link_element = await channel_element.query_selector(".poster > a")
|
||||
stream_page_url = (
|
||||
await stream_page_link_element.get_attribute("href")
|
||||
if stream_page_link_element
|
||||
else "No Stream Page URL"
|
||||
)
|
||||
|
||||
channel_info_list.append((title, poster_url, stream_page_url))
|
||||
|
||||
# Then, navigate to each channel's stream page and capture the M3U8 URLs
|
||||
for title, poster_url, stream_page_url in channel_info_list:
|
||||
# Navigate to the stream page
|
||||
await page.goto(stream_page_url)
|
||||
|
||||
m3u8_url_data = []
|
||||
|
||||
# Scrape genre tags
|
||||
genre_elements = await page.query_selector_all(".sgeneros a[rel='tag']")
|
||||
genres = [await genre.text_content() for genre in genre_elements]
|
||||
genres = [genre.title() for genre in genres]
|
||||
|
||||
# Query for player option elements and click to load M3U8 URL
|
||||
player_option_elements = await page.query_selector_all(
|
||||
"#playeroptionsul > li.dooplay_player_option"
|
||||
)
|
||||
country_name = "India"
|
||||
for player_option_element in player_option_elements:
|
||||
# Click the player option element
|
||||
await player_option_element.click()
|
||||
stream_title_element = await player_option_element.query_selector(
|
||||
"span.title"
|
||||
)
|
||||
stream_title = (
|
||||
await stream_title_element.text_content()
|
||||
if stream_title_element
|
||||
else "No Stream Title"
|
||||
)
|
||||
country_flag_element = await player_option_element.query_selector(
|
||||
"span.flag > img"
|
||||
)
|
||||
country_flag_url = (
|
||||
await country_flag_element.get_attribute("src")
|
||||
if country_flag_element
|
||||
else "No Country Flag URL"
|
||||
)
|
||||
country_name = get_country_name(
|
||||
country_flag_url.split("/")[-1].split(".")[0]
|
||||
)
|
||||
# Wait for the iframe to load and get its 'src' attribute
|
||||
try:
|
||||
iframe_element = await page.wait_for_selector("iframe.metaframe.rptss")
|
||||
except Exception:
|
||||
continue
|
||||
iframe_src = await iframe_element.get_attribute("src")
|
||||
iframe_src = iframe_src.strip()
|
||||
# Check if iframe_src is a valid URL
|
||||
parsed_src = urlparse(iframe_src)
|
||||
behavior_hints = {}
|
||||
# if "source" in parsed_src.query:
|
||||
# m3u8_url = parse_qs(parsed_src.query)["source"][0]
|
||||
if "youtube.com" in iframe_src:
|
||||
m3u8_url = iframe_src
|
||||
elif "yuppstream.net.in" in iframe_src:
|
||||
m3u8_url = iframe_src
|
||||
else:
|
||||
try:
|
||||
m3u8_url = await extract_player_source_url(iframe_src)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if m3u8_url is None:
|
||||
if "https://mhdtvplay.com" in iframe_src:
|
||||
channel_id = parse_qs(parsed_src.query)["watch"][0]
|
||||
m3u8_url = f"https://mhdtvplay.com/crichd/stream.php?id={channel_id}&e=.m3u8"
|
||||
else:
|
||||
continue
|
||||
behavior_hints = {
|
||||
"notWebReady": True,
|
||||
"proxyHeaders": {
|
||||
"request": {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
|
||||
"Referer": iframe_src,
|
||||
}
|
||||
},
|
||||
}
|
||||
m3u8_url_data.append((stream_title, m3u8_url, behavior_hints))
|
||||
|
||||
if not m3u8_url_data:
|
||||
continue
|
||||
|
||||
channels_data.append(
|
||||
{
|
||||
"title": title.replace("Hd", "").strip(),
|
||||
"poster": poster_url,
|
||||
"genres": genres,
|
||||
"country": country_name, # Set default country
|
||||
"tv_language": genres[0],
|
||||
"streams": [
|
||||
{
|
||||
"name": f"{title} - {index} | {stream_title}",
|
||||
"url": m3u8_url,
|
||||
"source": "MHDTVWorld",
|
||||
"behaviorHints": behavior_hints,
|
||||
}
|
||||
for index, (stream_title, m3u8_url, behavior_hints) in enumerate(
|
||||
m3u8_url_data, 1
|
||||
)
|
||||
],
|
||||
}
|
||||
)
|
||||
logging.info("Scraped %s", title)
|
||||
|
||||
return channels_data
|
||||
|
||||
|
||||
async def scrape_category(category_url, page):
|
||||
# Navigate to the category page
|
||||
await page.goto(category_url)
|
||||
|
||||
# Collect all page URLs
|
||||
page_urls = [category_url] + [
|
||||
await link.get_attribute("href")
|
||||
for link in await page.query_selector_all("div.pagination a.inactive")
|
||||
if await link.get_attribute("href")
|
||||
]
|
||||
|
||||
logging.info("found %d pages", len(page_urls))
|
||||
|
||||
# Scrape channels from each page
|
||||
channels_data = []
|
||||
for page_url in page_urls:
|
||||
await page.goto(page_url)
|
||||
try:
|
||||
channels_data.extend(await scrape_tv_channels(page))
|
||||
except Exception as e:
|
||||
logging.error("Failed to scrape %s. error: %s", page_url, e, exc_info=True)
|
||||
|
||||
return channels_data
|
||||
|
||||
|
||||
async def scrape_all_categories():
|
||||
async with async_playwright() as p:
|
||||
browser = await p.firefox.launch(headless=True)
|
||||
page = await browser.new_page()
|
||||
|
||||
# Extract category URLs
|
||||
await page.goto(BASE_URL)
|
||||
category_elements = await page.query_selector_all("#header a")
|
||||
category_urls = [
|
||||
urljoin(BASE_URL, await element.get_attribute("href"))
|
||||
for element in category_elements
|
||||
]
|
||||
# remove duplicates and BASE_URL
|
||||
category_urls = list(set(category_urls))
|
||||
category_urls.remove(BASE_URL)
|
||||
# category_urls.remove("https://mhdtv.org/")
|
||||
|
||||
# Scrape channels from each category
|
||||
all_channels_data = []
|
||||
for category_url in category_urls:
|
||||
logging.info("Scraping %s", category_url)
|
||||
try:
|
||||
all_channels_data.extend(await scrape_category(category_url, page))
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Failed to scrape %s. error: %s", category_url, e, exc_info=True
|
||||
)
|
||||
|
||||
await browser.close()
|
||||
|
||||
# remove duplicates
|
||||
unique_channels = {channel["title"]: channel for channel in all_channels_data}
|
||||
unique_channels_data = list(unique_channels.values())
|
||||
|
||||
logging.info("found %d channels", len(unique_channels_data))
|
||||
|
||||
with open("tamilultra.json", "w") as file:
|
||||
json.dump({"channels": unique_channels_data}, file, indent=4)
|
||||
|
||||
logging.info(
|
||||
"Done scraping TamilUltra. Manually verify the data & add it via /tv-metadata endpoint"
|
||||
)
|
||||
|
||||
|
||||
def main(is_scraping: bool = True):
|
||||
if is_scraping:
|
||||
asyncio.run(scrape_all_categories())
|
||||
return
|
||||
|
||||
with open("scrapers/temp.json") as file:
|
||||
channels = json.load(file)["channels"]
|
||||
|
||||
for channel in channels:
|
||||
logging.info("Adding %s", channel["title"])
|
||||
response = requests.post(f"{MEDIAFUSION_URL}/tv-metadata", json=channel)
|
||||
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except requests.HTTPError as err:
|
||||
logging.info("Response data: %s", response.text)
|
||||
continue
|
||||
|
||||
logging.info("Response data: %s", response.json())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Scrape TamilUltra Live TV")
|
||||
parser.add_argument(
|
||||
"--no-scrape",
|
||||
action="store_true",
|
||||
help="Don't scrape TamilUltra. Use this option to add the data to MediaFusion",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(not args.no_scrape)
|
||||
@@ -16,8 +16,8 @@ from scrapers import therarbg, torrent_downloads
|
||||
from scrapers.helpers import (
|
||||
update_torrent_series_streams_metadata,
|
||||
update_torrent_movie_streams_metadata,
|
||||
UA_HEADER,
|
||||
)
|
||||
from utils.const import UA_HEADER
|
||||
from utils.network import CircuitBreaker, batch_process_with_circuit_breaker
|
||||
from utils.parser import is_contain_18_plus_keywords
|
||||
from utils.torrent import extract_torrent_metadata
|
||||
|
||||
@@ -10,10 +10,10 @@ from redis.asyncio import Redis
|
||||
from db.config import settings
|
||||
from db.models import TorrentStreams, Season, Episode
|
||||
from scrapers.helpers import (
|
||||
UA_HEADER,
|
||||
update_torrent_series_streams_metadata,
|
||||
update_torrent_movie_streams_metadata,
|
||||
)
|
||||
from utils.const import UA_HEADER
|
||||
from utils.parser import convert_size_to_bytes, is_contain_18_plus_keywords
|
||||
from utils.validation_helper import is_video_file
|
||||
|
||||
|
||||
@@ -112,3 +112,7 @@ STREAMING_SERVICE_REQUIREMENTS = {
|
||||
"qbittorrent": ["qbittorrent_config"],
|
||||
"default": ["token"],
|
||||
}
|
||||
|
||||
UA_HEADER = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user