mirror of
https://github.com/Viren070/MediaFusion.git
synced 2025-12-01 23:21:11 +01:00
Add tamilultra scrapy & add support for trigger scrapers & consolidate apscheduler, background worker & bugfixes
This commit is contained in:
mhdzumair
Generated
+119
-127
@@ -153,14 +153,6 @@
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==3.10.4"
|
||||
},
|
||||
"async-timeout": {
|
||||
"hashes": [
|
||||
"sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f",
|
||||
"sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==4.0.3"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
|
||||
@@ -1219,11 +1211,11 @@
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
|
||||
"sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
|
||||
"sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
|
||||
"sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==23.2"
|
||||
"version": "==24.0"
|
||||
},
|
||||
"parse-torrent-title": {
|
||||
"git": "git+https://github.com/mhdzumair/parse-torrent-title",
|
||||
@@ -1231,11 +1223,11 @@
|
||||
},
|
||||
"parsel": {
|
||||
"hashes": [
|
||||
"sha256:2708fc74daeeb4ce471e2c2e9089b650ec940c7a218053e57421e69b5b00f82c",
|
||||
"sha256:aff28e68c9b3f1a901db2a4e3f158d8480a38724d7328ee751c1a4e1c1801e39"
|
||||
"sha256:1272baeda331cbc7f3a9a91df4c15c39276dc94b4ff2e8d5cffdf8bb6c382ea1",
|
||||
"sha256:a5a6bcb0c5fc741540ba3075636ff5cb968852a78cc16ae82250e5d486fa7d48"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.8.1"
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.9.0"
|
||||
},
|
||||
"pikpakapi": {
|
||||
"git": "git+https://github.com/mhdzumair/PikPakAPI.git",
|
||||
@@ -1420,12 +1412,12 @@
|
||||
},
|
||||
"pydantic": {
|
||||
"hashes": [
|
||||
"sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a",
|
||||
"sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f"
|
||||
"sha256:b1704e0847db01817624a6b86766967f552dd9dbf3afba4004409f908dcc84e6",
|
||||
"sha256:cc46fce86607580867bdc3361ad462bab9c222ef042d3da86f2fb333e1d916c5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==2.6.3"
|
||||
"version": "==2.6.4"
|
||||
},
|
||||
"pydantic-core": {
|
||||
"hashes": [
|
||||
@@ -1629,19 +1621,19 @@
|
||||
},
|
||||
"pyopenssl": {
|
||||
"hashes": [
|
||||
"sha256:6aa33039a93fffa4563e655b61d11364d01264be8ccb49906101e02a334530bf",
|
||||
"sha256:ba07553fb6fd6a7a2259adb9b84e12302a9a8a75c44046e8bb5d3e5ee887e3c3"
|
||||
"sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad",
|
||||
"sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==24.0.0"
|
||||
"version": "==24.1.0"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb",
|
||||
"sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"
|
||||
"sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad",
|
||||
"sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"
|
||||
],
|
||||
"markers": "python_full_version >= '3.6.8'",
|
||||
"version": "==3.1.1"
|
||||
"version": "==3.1.2"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
@@ -1733,110 +1725,110 @@
|
||||
},
|
||||
"rapidfuzz": {
|
||||
"hashes": [
|
||||
"sha256:01835d02acd5d95c1071e1da1bb27fe213c84a013b899aba96380ca9962364bc",
|
||||
"sha256:01eb03cd880a294d1bf1a583fdd00b87169b9cc9c9f52587411506658c864d73",
|
||||
"sha256:03f73b381bdeccb331a12c3c60f1e41943931461cdb52987f2ecf46bfc22f50d",
|
||||
"sha256:0402f1629e91a4b2e4aee68043a30191e5e1b7cd2aa8dacf50b1a1bcf6b7d3ab",
|
||||
"sha256:060bd7277dc794279fa95522af355034a29c90b42adcb7aa1da358fc839cdb11",
|
||||
"sha256:064c1d66c40b3a0f488db1f319a6e75616b2e5fe5430a59f93a9a5e40a656d15",
|
||||
"sha256:06e98ff000e2619e7cfe552d086815671ed09b6899408c2c1b5103658261f6f3",
|
||||
"sha256:08b6fb47dd889c69fbc0b915d782aaed43e025df6979b6b7f92084ba55edd526",
|
||||
"sha256:0a9fc714b8c290261669f22808913aad49553b686115ad0ee999d1cb3df0cd66",
|
||||
"sha256:0bbfae35ce4de4c574b386c43c78a0be176eeddfdae148cb2136f4605bebab89",
|
||||
"sha256:12ff8eaf4a9399eb2bebd838f16e2d1ded0955230283b07376d68947bbc2d33d",
|
||||
"sha256:1936d134b6c513fbe934aeb668b0fee1ffd4729a3c9d8d373f3e404fbb0ce8a0",
|
||||
"sha256:1c47d592e447738744905c18dda47ed155620204714e6df20eb1941bb1ba315e",
|
||||
"sha256:1dfc557c0454ad22382373ec1b7df530b4bbd974335efe97a04caec936f2956a",
|
||||
"sha256:1e12319c6b304cd4c32d5db00b7a1e36bdc66179c44c5707f6faa5a889a317c0",
|
||||
"sha256:23de71e7f05518b0bbeef55d67b5dbce3bcd3e2c81e7e533051a2e9401354eb0",
|
||||
"sha256:266dd630f12696ea7119f31d8b8e4959ef45ee2cbedae54417d71ae6f47b9848",
|
||||
"sha256:2963f4a3f763870a16ee076796be31a4a0958fbae133dbc43fc55c3968564cf5",
|
||||
"sha256:2a791168e119cfddf4b5a40470620c872812042f0621e6a293983a2d52372db0",
|
||||
"sha256:2b155e67fff215c09f130555002e42f7517d0ea72cbd58050abb83cb7c880cec",
|
||||
"sha256:2b19795b26b979c845dba407fe79d66975d520947b74a8ab6cee1d22686f7967",
|
||||
"sha256:2e03038bfa66d2d7cffa05d81c2f18fd6acbb25e7e3c068d52bb7469e07ff382",
|
||||
"sha256:3028ee8ecc48250607fa8a0adce37b56275ec3b1acaccd84aee1f68487c8557b",
|
||||
"sha256:35660bee3ce1204872574fa041c7ad7ec5175b3053a4cb6e181463fc07013de7",
|
||||
"sha256:3c772d04fb0ebeece3109d91f6122b1503023086a9591a0b63d6ee7326bd73d9",
|
||||
"sha256:3c84294f4470fcabd7830795d754d808133329e0a81d62fcc2e65886164be83b",
|
||||
"sha256:40cced1a8852652813f30fb5d4b8f9b237112a0bbaeebb0f4cc3611502556764",
|
||||
"sha256:4243a9c35667a349788461aae6471efde8d8800175b7db5148a6ab929628047f",
|
||||
"sha256:42f211e366e026de110a4246801d43a907cd1a10948082f47e8a4e6da76fef52",
|
||||
"sha256:4381023fa1ff32fd5076f5d8321249a9aa62128eb3f21d7ee6a55373e672b261",
|
||||
"sha256:484759b5dbc5559e76fefaa9170147d1254468f555fd9649aea3bad46162a88b",
|
||||
"sha256:49b9ed2472394d306d5dc967a7de48b0aab599016aa4477127b20c2ed982dbf9",
|
||||
"sha256:53251e256017e2b87f7000aee0353ba42392c442ae0bafd0f6b948593d3f68c6",
|
||||
"sha256:588c4b20fa2fae79d60a4e438cf7133d6773915df3cc0a7f1351da19eb90f720",
|
||||
"sha256:5a2f3e9df346145c2be94e4d9eeffb82fab0cbfee85bd4a06810e834fe7c03fa",
|
||||
"sha256:5d82b9651e3d34b23e4e8e201ecd3477c2baa17b638979deeabbb585bcb8ba74",
|
||||
"sha256:5dd95b6b7bfb1584f806db89e1e0c8dbb9d25a30a4683880c195cc7f197eaf0c",
|
||||
"sha256:692c9a50bea7a8537442834f9bc6b7d29d8729a5b6379df17c31b6ab4df948c2",
|
||||
"sha256:6b0ccc2ec1781c7e5370d96aef0573dd1f97335343e4982bdb3a44c133e27786",
|
||||
"sha256:6dede83a6b903e3ebcd7e8137e7ff46907ce9316e9d7e7f917d7e7cdc570ee05",
|
||||
"sha256:7142ee354e9c06e29a2636b9bbcb592bb00600a88f02aa5e70e4f230347b373e",
|
||||
"sha256:7183157edf0c982c0b8592686535c8b3e107f13904b36d85219c77be5cefd0d8",
|
||||
"sha256:7420e801b00dee4a344ae2ee10e837d603461eb180e41d063699fb7efe08faf0",
|
||||
"sha256:757dfd7392ec6346bd004f8826afb3bf01d18a723c97cbe9958c733ab1a51791",
|
||||
"sha256:76c23ceaea27e790ddd35ef88b84cf9d721806ca366199a76fd47cfc0457a81b",
|
||||
"sha256:7fec74c234d3097612ea80f2a80c60720eec34947066d33d34dc07a3092e8105",
|
||||
"sha256:82300e5f8945d601c2daaaac139d5524d7c1fdf719aa799a9439927739917460",
|
||||
"sha256:841eafba6913c4dfd53045835545ba01a41e9644e60920c65b89c8f7e60c00a9",
|
||||
"sha256:8d7a072f10ee57c8413c8ab9593086d42aaff6ee65df4aa6663eecdb7c398dca",
|
||||
"sha256:8e4da90e4c2b444d0a171d7444ea10152e07e95972bb40b834a13bdd6de1110c",
|
||||
"sha256:96cd19934f76a1264e8ecfed9d9f5291fde04ecb667faef5f33bdbfd95fe2d1f",
|
||||
"sha256:a03863714fa6936f90caa7b4b50ea59ea32bb498cc91f74dc25485b3f8fccfe9",
|
||||
"sha256:a1788ebb5f5b655a15777e654ea433d198f593230277e74d51a2a1e29a986283",
|
||||
"sha256:a3ee4f8f076aa92184e80308fc1a079ac356b99c39408fa422bbd00145be9854",
|
||||
"sha256:a490cd645ef9d8524090551016f05f052e416c8adb2d8b85d35c9baa9d0428ab",
|
||||
"sha256:a553cc1a80d97459d587529cc43a4c7c5ecf835f572b671107692fe9eddf3e24",
|
||||
"sha256:a59472b43879012b90989603aa5a6937a869a72723b1bf2ff1a0d1edee2cc8e6",
|
||||
"sha256:ac434fc71edda30d45db4a92ba5e7a42c7405e1a54cb4ec01d03cc668c6dcd40",
|
||||
"sha256:ad9d74ef7c619b5b0577e909582a1928d93e07d271af18ba43e428dc3512c2a1",
|
||||
"sha256:ae598a172e3a95df3383634589660d6b170cc1336fe7578115c584a99e0ba64d",
|
||||
"sha256:b2ef4c0fd3256e357b70591ffb9e8ed1d439fb1f481ba03016e751a55261d7c1",
|
||||
"sha256:b3e5af946f419c30f5cb98b69d40997fe8580efe78fc83c2f0f25b60d0e56efb",
|
||||
"sha256:b53137d81e770c82189e07a8f32722d9e4260f13a0aec9914029206ead38cac3",
|
||||
"sha256:b7e3375e4f2bfec77f907680328e4cd16cc64e137c84b1886d547ab340ba6928",
|
||||
"sha256:bcc957c0a8bde8007f1a8a413a632a1a409890f31f73fe764ef4eac55f59ca87",
|
||||
"sha256:be156f51f3a4f369e758505ed4ae64ea88900dcb2f89d5aabb5752676d3f3d7e",
|
||||
"sha256:be368573255f8fbb0125a78330a1a40c65e9ba3c5ad129a426ff4289099bfb41",
|
||||
"sha256:c1a23eee225dfb21c07f25c9fcf23eb055d0056b48e740fe241cbb4b22284379",
|
||||
"sha256:c65f92881753aa1098c77818e2b04a95048f30edbe9c3094dc3707d67df4598b",
|
||||
"sha256:ca3dfcf74f2b6962f411c33dd95b0adf3901266e770da6281bc96bb5a8b20de9",
|
||||
"sha256:cd4ba4c18b149da11e7f1b3584813159f189dc20833709de5f3df8b1342a9759",
|
||||
"sha256:d056e342989248d2bdd67f1955bb7c3b0ecfa239d8f67a8dfe6477b30872c607",
|
||||
"sha256:d2f0274595cc5b2b929c80d4e71b35041104b577e118cf789b3fe0a77b37a4c5",
|
||||
"sha256:d73dcfe789d37c6c8b108bf1e203e027714a239e50ad55572ced3c004424ed3b",
|
||||
"sha256:d79aec8aeee02ab55d0ddb33cea3ecd7b69813a48e423c966a26d7aab025cdfe",
|
||||
"sha256:da3e8c9f7e64bb17faefda085ff6862ecb3ad8b79b0f618a6cf4452028aa2222",
|
||||
"sha256:dad55a514868dae4543ca48c4e1fc0fac704ead038dafedf8f1fc0cc263746c1",
|
||||
"sha256:dec307b57ec2d5054d77d03ee4f654afcd2c18aee00c48014cb70bfed79597d6",
|
||||
"sha256:e06c4242a1354cf9d48ee01f6f4e6e19c511d50bb1e8d7d20bcadbb83a2aea90",
|
||||
"sha256:e19d519386e9db4a5335a4b29f25b8183a1c3f78cecb4c9c3112e7f86470e37f",
|
||||
"sha256:e49b9575d16c56c696bc7b06a06bf0c3d4ef01e89137b3ddd4e2ce709af9fe06",
|
||||
"sha256:ebcfb5bfd0a733514352cfc94224faad8791e576a80ffe2fd40b2177bf0e7198",
|
||||
"sha256:ed0f712e0bb5fea327e92aec8a937afd07ba8de4c529735d82e4c4124c10d5a0",
|
||||
"sha256:edf97c321fd641fea2793abce0e48fa4f91f3c202092672f8b5b4e781960b891",
|
||||
"sha256:eef8b346ab331bec12bbc83ac75641249e6167fab3d84d8f5ca37fd8e6c7a08c",
|
||||
"sha256:f056ba42fd2f32e06b2c2ba2443594873cfccc0c90c8b6327904fc2ddf6d5799",
|
||||
"sha256:f382f7ffe384ce34345e1c0b2065451267d3453cadde78946fbd99a59f0cc23c",
|
||||
"sha256:f59d19078cc332dbdf3b7b210852ba1f5db8c0a2cd8cc4c0ed84cc00c76e6802",
|
||||
"sha256:fbc07e2e4ac696497c5f66ec35c21ddab3fc7a406640bffed64c26ab2f7ce6d6",
|
||||
"sha256:fde9b14302a31af7bdafbf5cfbb100201ba21519be2b9dedcf4f1048e4fbe65d"
|
||||
"sha256:012221629d54d3bee954148247f711eb86d4d390b589ebfe03172ea0b37a7531",
|
||||
"sha256:080cb71b50cb6aff11d1c6aeb157f273e2da0b2bdb3f9d7b01257e49e69a8576",
|
||||
"sha256:153d065e353371cc0aeff32b99999a5758266a64e958d1364189367c1c9f6814",
|
||||
"sha256:16270b5529de83b7bae7457e952e4d9cf3fbf029a837dd32d415bb9e0eb8e599",
|
||||
"sha256:1b86b93d93020c2b3edc1665d75c8855784845fc0a739b312c26c3a4bf0c80d5",
|
||||
"sha256:1c54d3c85e522d3ac9ee39415f183c8fa184c4f87e7e5a37938f15a6d50e853a",
|
||||
"sha256:1df2faf80201952e252413b6fac6f3e146080dcebb87bb1bb722508e67558ed8",
|
||||
"sha256:1dfe4c24957474ce0ac75d886387e30e292b4be39228a6d71f76de414dc187db",
|
||||
"sha256:259364199cbfeca33b1af369fc7951f71717aa285184a3fa5a7b1772da1b89db",
|
||||
"sha256:28243086ed0e50808bb56632e5442c457241646aeafafd501ac87901f40a3237",
|
||||
"sha256:28c011fb31f2c3f82f503aedd6097d3d3854e574e327a119a3b7eb2cf90b79ca",
|
||||
"sha256:2a46220f86a5f9cb016af31525e0d0865cad437d02239aa0d8aed2ab8bff1f1c",
|
||||
"sha256:2acd2514defce81e6ff4bbff50252d5e7df8e85a731442c4b83e44c86cf1c916",
|
||||
"sha256:2d580d937146e803c8e5e1b87916cab8d6f84013b6392713e201efcda335c7d8",
|
||||
"sha256:3168ff565d4b8c239cf11fb604dd2507d30e9bcaac76a4077c0ac23cf2c866ed",
|
||||
"sha256:337e357f693130c4c6be740652542b260e36f622c59e01fa33d58f1d2750c930",
|
||||
"sha256:3549123fca5bb817341025f98e8e49ca99f84596c7c4f92b658f8e5836040d4a",
|
||||
"sha256:354ad5fe655beb7b279390cb58334903931c5452ecbad1b1666ffb06786498e2",
|
||||
"sha256:35aeac852bca06023d6bbd50c1fc504ca5a9a3613d5e75a140f0be7601fa34ef",
|
||||
"sha256:35bd4bc9c40e6994c5d6edea4b9319388b4d9711c13c66d543bb4c37624b4184",
|
||||
"sha256:36ce7b68a7b90b787cdd73480a68d2f1ca63c31a3a9d5a79a8736f978e1e9344",
|
||||
"sha256:380586664f2f63807050ddb95e7702888b4f0b425abf17655940c411f39287ad",
|
||||
"sha256:3ae7c86914cb6673e97e187ba431b9c4cf4177d9ae77f8a1e5b2ba9a5628839e",
|
||||
"sha256:3c1dc078ef371fce09f9f3eec2ca4eaa2a8cd412ec53941015b4f39f14d34407",
|
||||
"sha256:4209816626d8d6ff8ae7dc248061c6059e618b70c6e6f6e4d7444ae3740b2b85",
|
||||
"sha256:423c7c588b09d618601097b7a0017dfcb91132a2076bef29023c5f3cd2dc3de1",
|
||||
"sha256:48e1eaea8fcd522fca7f04f0480663f0f0cfb77957092cce60a93f4462864996",
|
||||
"sha256:495c0d8e14e6f12520eb7fc71b9ba9fcaafb47fc23a654e6e89b6c7985ec0020",
|
||||
"sha256:4c279928651ce0e9e5220dcb25a00cc53b65e592a0861336a38299bcdca3a596",
|
||||
"sha256:4edcceebb85ebfa49a3ddcde20ad891d36c08dc0fd592efdab0e7d313a4e36af",
|
||||
"sha256:4fcfa23b5553b27f4016df77c53172ea743454cf12c28cfa7c35a309a2be93b3",
|
||||
"sha256:53597fd72a9340bcdd80d3620f4957c2b92f9b569313b969a3abdaffd193aae6",
|
||||
"sha256:5378c04102c7f084cde30a100154fa6d7e2baf0d51a6bdd2f912545559c1fb35",
|
||||
"sha256:617949a70150e6fffdaed19253dd49f7a53528411dc8bf7663d499ba21e0f61e",
|
||||
"sha256:62df2136068e2515ed8beb01756381ff62c29384d785e3bf46e3111d4ea3ba1e",
|
||||
"sha256:6440ed0b3007c1c9286b0b88fe2ab2d9e83edd60cd62293b3dfabb732b4e8a30",
|
||||
"sha256:6468f8bc8c3c50604f43631550ef9cfec873515dba5023ca34d461be94669fc8",
|
||||
"sha256:6478f7803efebf5f644d0b758439c5b25728550fdfbb19783d150004c46a75a9",
|
||||
"sha256:66b008bf2972740cd2dda5d382eb8bdb87265cd88198e71c7797bdc0d1f79d20",
|
||||
"sha256:673ba2c343644805acdae1cb949c6a4de71aa2f62a998978551ebea59603af3f",
|
||||
"sha256:6d67b649bf3e1b1722d04eca44d37919aef88305ce7ad05564502d013cf550fd",
|
||||
"sha256:6ee98d88ae9ccc77ff61992ed33b2496478def5dc0da55c9a9aa06fcb725a352",
|
||||
"sha256:7382c90170f60c846c81a07ddd80bb2e8c43c8383754486fa37f67391a571897",
|
||||
"sha256:74c6773b11445b5e5cf93ca383171cd0ac0cdeafea11a7b2a5688f8bf8d813e6",
|
||||
"sha256:7d830bc7a9b586a374147ec60b08b1f9ae5996b43f75cc514f37faef3866b519",
|
||||
"sha256:7f18397c8d6a65fc0b288d2fc29bc7baeea6ba91eeb95163a3cd98f23cd3bc85",
|
||||
"sha256:7f9fa14136a5b0cba1ec42531f7c3e0b0d3edb7fd6bc5e5ae7b498541f3855ab",
|
||||
"sha256:80e51b22a7da83f9c87a97e92df07ed0612c74c35496590255f4b5d5b4212dfe",
|
||||
"sha256:81a630ed2fc3ec5fc7400eb66bab1f87e282b4d47f0abe3e48c6634dfa13b5e4",
|
||||
"sha256:824cc381cf81cbf8d158f6935664ec2a69e6ac3b1d39fa201988bf81a257f775",
|
||||
"sha256:84c1032ae42628465b7a5cc35249906061e18a8193c9c27cbd2db54e9823a9a6",
|
||||
"sha256:85a5b6e026393fe39fb61146b9c17c5af66fffbe1410e992c4bb06d9ec327bd3",
|
||||
"sha256:87ac3a87f2251ae2e95fc9478ca5c759de6d141d04c84d3fec9f9cdcfc167b33",
|
||||
"sha256:95a49c6b8bf1229743ae585dd5b7d57f0d15a7eb6e826866d5c9965ba958503c",
|
||||
"sha256:9a07dffac439223b4f1025dbfc68f4445a3460a859309c9858c2a3fa29617cdc",
|
||||
"sha256:9a74102fc5a2534fe91f7507838623e1f3a149d8e05648389c42bb42e14b1c3f",
|
||||
"sha256:9bcc91ebd8fc69a6bd3b5711c8250f5f4e70606b4da75ef415f57ad209978205",
|
||||
"sha256:9d457c89bac1471442002e70551e8268e639b3870b4a4521eae363c07253be87",
|
||||
"sha256:a5637e6bf11b15b5aff6ee818c76bdec99ad208511b78985e6209ba648a6e3ee",
|
||||
"sha256:a7895e04a22d6515bc91a850e0831f2405547605aa311d1ffec51e4818abc3c1",
|
||||
"sha256:a945567c2b0b6e069454c9782d5234b0b6795718adf7a9f868bd3144afa6a023",
|
||||
"sha256:ab269adfc64480f209e99f253391a10735edd5c09046e04899adab5fb132f20e",
|
||||
"sha256:af7c19ec86e11488539380d3db1755be5d561a3c0e7b04ff9d07abd7f9a8e9d8",
|
||||
"sha256:b593cc51aed887e93b78c2f94dfae9008be2b23d17afd3b1f1d3eb3913b58f26",
|
||||
"sha256:bd82f9838519136b7083dd1e3149ee80344521f3dc37f744f227505ff0883efb",
|
||||
"sha256:be69f7fd46b5c6467fe5e2fd4cff3816b0c03048eed8a4becb9a73e6000960e7",
|
||||
"sha256:bec353f022011e6e5cd28ccb8700fbd2a33918197af0d4e0abb3c3f4845cc864",
|
||||
"sha256:c17c5efee347a40a6f4c1eec59e3d7d1e22f7613a97f8b8a07733ef723483a04",
|
||||
"sha256:cbd5894f23fdf5697499cf759523639838ac822bd1600e343fdce7313baa02ae",
|
||||
"sha256:cda81d0e0ce0c13abfa46b24e10c1e85f9c6acb628f0a9a948f5779f9c2076a2",
|
||||
"sha256:cf911e792ab0c431694c9bf2648afabfd92099103f2e31492893e078ddca5e1a",
|
||||
"sha256:d07899506a5a8760448d9df036d528b55a554bf571714173635c79eef4a86e58",
|
||||
"sha256:d41dd59a70decfce6595315367a2fea2af660d92a9d144acc6479030501014d7",
|
||||
"sha256:d4f6de745fe6ce46a422d353ee10599013631d7d714a36d025f164b2d4e8c000",
|
||||
"sha256:d57b98013b802621bbc8b12a46bfc9d36ac552ab51ca207f7ce167ad46adabeb",
|
||||
"sha256:d8e5a437b9089df6242a718d9c31ab1742989e9400a0977af012ef483b63b4c2",
|
||||
"sha256:dbee7f5ff11872b76505cbd87c814abc823e8757f11c69062eb3b25130a283da",
|
||||
"sha256:ddc380ffaa90f204cc9ddcb779114b9ab6f015246d549de9d47871a97ef9f18a",
|
||||
"sha256:de8adc12161bf282c60f12dc9233bb31632f71d446a010fe7469a69b8153427f",
|
||||
"sha256:e06f6d270112f5db001f1cba5a97e1a48aee3d3dbdcbea3ec027c230462dbf9b",
|
||||
"sha256:e1507fc5769aa109dda4de3a15f822a0f6a03e18d627bd0ba3ddbb253cf70e07",
|
||||
"sha256:e48dde8ca83d11daa00900cf6a5d281a1297aef9b7bfa73801af6e8822be5019",
|
||||
"sha256:e706f302c6a3ae0d74edd0d6ace46aee1ae07c563b436ccf5ff04db2b3571e60",
|
||||
"sha256:eb2e51d01b9c6d6954a3e055c57a80d4685b4fc82719db5519fc153566bcd6bb",
|
||||
"sha256:ed52461ae5a9ea4c400d38e2649c74a413f1a6d8fb8308b66f1fbd122514732f",
|
||||
"sha256:ef3925daaa93eed20401012e219f569ff0c039ed5bf4ce2d3737b4f75d441622",
|
||||
"sha256:f31314fd2e2f3dc3e519e6f93669462ce7953df2def1c344aa8f5345976d0eb2",
|
||||
"sha256:f3a70f341c4c111bad910d2df69c78577a98af140319a996af24c9385939335d",
|
||||
"sha256:f8b77779174b1b40aa70827692571ab457061897846255ad7d5d559e2edb1932",
|
||||
"sha256:fe2a68be734e8e88af23385c68d6467e15818b6b1df1cbfebf7bff577226c957"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==3.6.1"
|
||||
"version": "==3.6.2"
|
||||
},
|
||||
"redis": {
|
||||
"extras": [
|
||||
"hiredis"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:3f82cc80d350e93042c8e6e7a5d0596e4dd68715babffba79492733e1f367037",
|
||||
"sha256:4caa8e1fcb6f3c0ef28dba99535101d80934b7d4cd541bbb47f4a3826ee472d1"
|
||||
"sha256:4973bae7444c0fbed64a06b87446f79361cb7e4ec1538c022d696ed7a5015580",
|
||||
"sha256:5da9b8fe9e1254293756c16c008e8620b3d15fcc6dde6babde9541850e72a32d"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==5.0.2"
|
||||
"version": "==5.0.3"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
@@ -1899,11 +1891,11 @@
|
||||
},
|
||||
"setuptools": {
|
||||
"hashes": [
|
||||
"sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56",
|
||||
"sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"
|
||||
"sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e",
|
||||
"sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==69.1.1"
|
||||
"version": "==69.2.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
@@ -2019,11 +2011,11 @@
|
||||
},
|
||||
"torf": {
|
||||
"hashes": [
|
||||
"sha256:e84ec73cd81cb2d56e36b9f4604806f16ee0c70aeafadfe2f31ca3266ca03a87",
|
||||
"sha256:f846ef58b6b00bc3b6a5c25bf225c59149ee871738d87a20a8953f620b77671c"
|
||||
"sha256:261e861683f4d15cd8ae43c9db4828ef9da08f8efdf3b2878f82c0385d1e3707",
|
||||
"sha256:9892b68c414c48a6dc0d5127cc6c76088fef3f9ffa91851cb0b4334d72e9565f"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==4.2.4"
|
||||
"version": "==4.2.5"
|
||||
},
|
||||
"twisted": {
|
||||
"hashes": [
|
||||
@@ -2062,11 +2054,11 @@
|
||||
"standard"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:3d9a267296243532db80c83a959a3400502165ade2c1338dea4e67915fd4745a",
|
||||
"sha256:5c89da2f3895767472a35556e539fd59f7edbe9b1e9c0e1c99eebeadc61838e4"
|
||||
"sha256:6623abbbe6176204a4226e67607b4d52cc60ff62cda0ff177613645cefa2ece1",
|
||||
"sha256:cab4473b5d1eaeb5a0f6375ac4bc85007ffc75c3cc1768816d9e5d589857b067"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==0.27.1"
|
||||
"version": "==0.28.0"
|
||||
},
|
||||
"uvloop": {
|
||||
"hashes": [
|
||||
|
||||
+41
-2
@@ -18,8 +18,12 @@ from fastapi.responses import FileResponse, StreamingResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
from api import middleware
|
||||
from api.scheduler import setup_scheduler
|
||||
from db import database, crud, schemas
|
||||
from db.config import settings
|
||||
from mediafusion_scrapy.task import run_spider
|
||||
from scrapers.tamilmv import run_tamilmv_scraper
|
||||
from scrapers.tamil_blasters import run_tamil_blasters_scraper
|
||||
from streaming_providers.routes import router as streaming_provider_router
|
||||
from utils import crypto, torrent, poster, const, wrappers
|
||||
from utils.parser import generate_manifest
|
||||
@@ -64,6 +68,8 @@ async def start_scheduler():
|
||||
scheduler.add_job(
|
||||
crud.delete_search_history, CronTrigger(day="*/1"), name="delete_search_history"
|
||||
)
|
||||
setup_scheduler(scheduler)
|
||||
scheduler.start()
|
||||
app.state.scheduler = scheduler
|
||||
|
||||
|
||||
@@ -96,7 +102,7 @@ async def get_home(request: Request):
|
||||
|
||||
|
||||
@app.get("/health", tags=["health"])
|
||||
@wrappers.exclude
|
||||
@wrappers.exclude_rate_limit
|
||||
async def health(request: Request):
|
||||
return {"status": "healthy"}
|
||||
|
||||
@@ -378,6 +384,7 @@ async def get_streams(
|
||||
user_data, secret_str, request.app.state.redis, video_id, season, episode
|
||||
)
|
||||
else:
|
||||
response.headers.update(const.NO_CACHE_HEADERS)
|
||||
fetched_streams = await crud.get_tv_streams(video_id)
|
||||
|
||||
return {"streams": fetched_streams}
|
||||
@@ -391,7 +398,7 @@ async def encrypt_user_data(user_data: schemas.UserData):
|
||||
|
||||
|
||||
@app.get("/poster/{catalog_type}/{mediafusion_id}.jpg", tags=["poster"])
|
||||
@wrappers.exclude
|
||||
@wrappers.exclude_rate_limit
|
||||
async def get_poster(
|
||||
catalog_type: Literal["movie", "series", "tv"],
|
||||
mediafusion_id: str,
|
||||
@@ -437,6 +444,38 @@ async def get_poster(
|
||||
raise HTTPException(status_code=404, detail="Failed to create poster.")
|
||||
|
||||
|
||||
@app.get("/scraper", tags=["scraper"])
|
||||
async def get_scraper(request: Request):
|
||||
return TEMPLATES.TemplateResponse(
|
||||
"html/scraper.html",
|
||||
{
|
||||
"request": request,
|
||||
"authentication_required": settings.api_password is not None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/scraper/run", tags=["scraper"])
|
||||
async def run_scraper_task(task: schemas.ScraperTask):
|
||||
if settings.api_password and task.api_password != settings.api_password:
|
||||
raise HTTPException(status_code=401, detail="Invalid API password.")
|
||||
|
||||
if task.scraper_type == "tamilmv":
|
||||
run_tamilmv_scraper.send(task.pages, task.start_page)
|
||||
elif task.scraper_type == "tamilblasters":
|
||||
run_tamil_blasters_scraper.send(task.pages, task.start_page)
|
||||
elif task.scraper_type == "scrapy":
|
||||
if not task.spider_name:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Spider name is required for scrapy tasks."
|
||||
)
|
||||
run_spider.send(task.spider_name)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Invalid scraper type.")
|
||||
|
||||
return {"status": f"Scraping {task.scraper_type} task has been scheduled."}
|
||||
|
||||
|
||||
app.include_router(
|
||||
streaming_provider_router, prefix="/streaming_provider", tags=["streaming_provider"]
|
||||
)
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
if not endpoint:
|
||||
return await call_next(request)
|
||||
|
||||
is_exclude = getattr(endpoint, "exclude", False)
|
||||
is_exclude = getattr(endpoint, "exclude_rate_limit", False)
|
||||
if is_exclude:
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
+50
-57
@@ -1,73 +1,66 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from db.config import settings
|
||||
from scrapers import tamil_blasters, tamilmv
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
from mediafusion_scrapy.task import run_spider
|
||||
from utils.validation_helper import validate_tv_streams_in_db
|
||||
|
||||
|
||||
def run_formula_tgx_spider(spider_name: str):
|
||||
def setup_scheduler(scheduler: AsyncIOScheduler):
|
||||
"""
|
||||
Function to start the formula_tgx spider.
|
||||
Set up the scheduler with the required jobs.
|
||||
"""
|
||||
process = CrawlerProcess(get_project_settings())
|
||||
process.crawl(spider_name)
|
||||
process.start()
|
||||
|
||||
# Setup tamil blasters scraper
|
||||
scheduler.add_job(
|
||||
tamil_blasters.run_tamil_blasters_scraper.send,
|
||||
CronTrigger.from_crontab(settings.tamil_blasters_scheduler_crontab),
|
||||
name="tamil_blasters",
|
||||
)
|
||||
|
||||
scheduler = AsyncIOScheduler()
|
||||
# Setup tamilmv scraper
|
||||
scheduler.add_job(
|
||||
tamilmv.run_tamilmv_scraper.send,
|
||||
CronTrigger.from_crontab(settings.tamilmv_scheduler_crontab),
|
||||
name="tamilmv",
|
||||
)
|
||||
|
||||
# Setup tamil blasters scraper
|
||||
scheduler.add_job(
|
||||
tamil_blasters.run_schedule_scrape,
|
||||
CronTrigger.from_crontab(settings.tamil_blasters_scheduler_crontab),
|
||||
name="tamil_blasters",
|
||||
)
|
||||
# Setup formula_tgx scraper
|
||||
scheduler.add_job(
|
||||
run_spider.send,
|
||||
CronTrigger.from_crontab(settings.formula_tgx_scheduler_crontab),
|
||||
name="formula_tgx",
|
||||
kwargs={"spider_name": "formula_tgx"},
|
||||
)
|
||||
|
||||
# Setup tamilmv scraper
|
||||
scheduler.add_job(
|
||||
tamilmv.run_schedule_scrape,
|
||||
CronTrigger.from_crontab(settings.tamilmv_scheduler_crontab),
|
||||
name="tamilmv",
|
||||
)
|
||||
# Setup mhdtvworld scraper
|
||||
scheduler.add_job(
|
||||
run_spider.send,
|
||||
CronTrigger.from_crontab(settings.mhdtvworld_scheduler_crontab),
|
||||
name="mhdtvworld",
|
||||
kwargs={"spider_name": "mhdtvworld"},
|
||||
)
|
||||
|
||||
# Setup formula_tgx scraper
|
||||
scheduler.add_job(
|
||||
run_formula_tgx_spider,
|
||||
CronTrigger.from_crontab(settings.formula_tgx_scheduler_crontab),
|
||||
name="formula_tgx",
|
||||
kwargs={"spider_name": "formula_tgx"},
|
||||
)
|
||||
# Setup mhdtvsports scraper
|
||||
scheduler.add_job(
|
||||
run_spider.send,
|
||||
CronTrigger.from_crontab(settings.mhdtvsports_scheduler_crontab),
|
||||
name="mhdtvsports",
|
||||
kwargs={"spider_name": "mhdtvsports"},
|
||||
)
|
||||
|
||||
# Setup mhdtvworld scraper
|
||||
scheduler.add_job(
|
||||
run_formula_tgx_spider,
|
||||
CronTrigger.from_crontab(settings.mhdtvworld_scheduler_crontab),
|
||||
name="mhdtvworld",
|
||||
kwargs={"spider_name": "mhdtvworld"},
|
||||
)
|
||||
# Setup tamilultra scraper
|
||||
scheduler.add_job(
|
||||
run_spider.send,
|
||||
CronTrigger.from_crontab(settings.tamilultra_scheduler_crontab),
|
||||
name="tamilultra",
|
||||
kwargs={"spider_name": "tamilultra"},
|
||||
)
|
||||
|
||||
# Setup mhdtvsports scraper
|
||||
scheduler.add_job(
|
||||
run_formula_tgx_spider,
|
||||
CronTrigger.from_crontab(settings.mhdtvsports_scheduler_crontab),
|
||||
name="mhdtvsports",
|
||||
kwargs={"spider_name": "mhdtvsports"},
|
||||
)
|
||||
|
||||
# Start the scheduler
|
||||
scheduler.start()
|
||||
|
||||
try:
|
||||
asyncio.get_event_loop().run_forever()
|
||||
except Exception as e:
|
||||
logging.error(f"Error occurred: {e}")
|
||||
# Schedule validate_tv_streams_in_db
|
||||
scheduler.add_job(
|
||||
validate_tv_streams_in_db.send,
|
||||
CronTrigger.from_crontab(settings.validate_tv_streams_in_db_crontab),
|
||||
name="validate_tv_streams_in_db",
|
||||
)
|
||||
|
||||
+7
-3
@@ -1,10 +1,14 @@
|
||||
# import background actors
|
||||
import asyncio
|
||||
|
||||
from db import database
|
||||
from scrapers.helpers import update_torrent_movie_streams_metadata # noqa: F401
|
||||
from scrapers.prowlarr import parse_and_store_movie_stream_data # noqa: F401
|
||||
|
||||
# import background actors
|
||||
from scrapers import helpers # noqa: F401
|
||||
from scrapers import prowlarr # noqa: F401
|
||||
from mediafusion_scrapy import task # noqa: F401
|
||||
from utils import torrent
|
||||
from utils import validation_helper # noqa: F401
|
||||
from scrapers import tamil_blasters, tamilmv # noqa: F401
|
||||
|
||||
|
||||
async def async_setup():
|
||||
|
||||
@@ -29,6 +29,8 @@ class Settings(BaseSettings):
|
||||
formula_tgx_scheduler_crontab: str = "0 */12 * * *"
|
||||
mhdtvworld_scheduler_crontab: str = "0 0 * * 5"
|
||||
mhdtvsports_scheduler_crontab: str = "0 10 * * *"
|
||||
tamilultra_scheduler_crontab: str = "0 8 * * *"
|
||||
validate_tv_streams_in_db_crontab: str = "0 */2 * * *"
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
|
||||
+17
-4
@@ -6,6 +6,7 @@ from uuid import uuid4
|
||||
|
||||
from beanie import WriteRules
|
||||
from beanie.operators import In, Set
|
||||
from pydantic import ValidationError
|
||||
from pymongo.errors import DuplicateKeyError
|
||||
from redis.asyncio import Redis
|
||||
|
||||
@@ -78,7 +79,9 @@ async def get_meta_list(
|
||||
async def get_tv_meta_list(
|
||||
genre: Optional[str] = None, skip: int = 0, limit: int = 25
|
||||
) -> list[schemas.Meta]:
|
||||
query = MediaFusionTVMetaData.find(fetch_links=True)
|
||||
query = MediaFusionTVMetaData.find(
|
||||
MediaFusionMovieMetaData.streams.is_working == True, fetch_links=True
|
||||
)
|
||||
if genre:
|
||||
query = query.find(In(MediaFusionTVMetaData.genres, [genre]))
|
||||
|
||||
@@ -238,11 +241,21 @@ async def get_series_streams(
|
||||
|
||||
|
||||
async def get_tv_streams(video_id: str) -> list[Stream]:
|
||||
tv_data = await get_tv_data_by_id(video_id, True)
|
||||
if not tv_data:
|
||||
tv_streams = await TVStreams.find(
|
||||
{"meta_id": video_id, "is_working": True}
|
||||
).to_list()
|
||||
if not tv_streams:
|
||||
return []
|
||||
|
||||
return parse_tv_stream_data(tv_data)
|
||||
return await parse_tv_stream_data(tv_streams)
|
||||
|
||||
|
||||
async def get_tv_stream_by_id(stream_id: str) -> TVStreams | None:
|
||||
try:
|
||||
stream = await TVStreams.get(stream_id)
|
||||
except ValidationError:
|
||||
return None
|
||||
return stream
|
||||
|
||||
|
||||
async def get_movie_meta(meta_id: str):
|
||||
|
||||
@@ -63,6 +63,7 @@ class TVStreams(Document):
|
||||
created_at: datetime = Field(default_factory=datetime.now)
|
||||
meta_id: Optional[str] = None
|
||||
country: str | None = None
|
||||
is_working: Optional[bool] = True
|
||||
|
||||
|
||||
class MediaFusionMetaData(Document):
|
||||
|
||||
@@ -195,3 +195,11 @@ class TVMetaData(BaseModel):
|
||||
|
||||
class TorrentStreamsList(BaseModel):
|
||||
streams: list[TorrentStreams]
|
||||
|
||||
|
||||
class ScraperTask(BaseModel):
|
||||
scraper_type: str
|
||||
pages: int = 1
|
||||
start_page: int = 1
|
||||
spider_name: str = None
|
||||
api_password: str = None
|
||||
|
||||
@@ -59,14 +59,6 @@ services:
|
||||
volumes:
|
||||
- prowlar-config:/config
|
||||
|
||||
scraper:
|
||||
image: mhdzumair/mediafusion:v3.7.4
|
||||
command: ["pipenv", "run", "python", "-m", "api.scheduler"]
|
||||
env_file:
|
||||
- .env
|
||||
depends_on:
|
||||
- mongodb
|
||||
|
||||
volumes:
|
||||
mongo-data:
|
||||
prowlar-config:
|
||||
|
||||
@@ -129,56 +129,6 @@ spec:
|
||||
memory: "1Gi"
|
||||
cpu: "200m"
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: scraper-deployment
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: scraper
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: scraper
|
||||
spec:
|
||||
containers:
|
||||
- name: scraper
|
||||
image: mhdzumair/mediafusion:v3.7.4
|
||||
command: ["pipenv", "run", "python", "-m", "api.scheduler"]
|
||||
env:
|
||||
- name: MONGO_URI
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: MONGO_URI
|
||||
- name: SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: SECRET_KEY
|
||||
- name: PROWLARR_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: PROWLARR_API_KEY
|
||||
- name: REDIS_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: REDIS_URL
|
||||
- name: db_max_connections
|
||||
value: "30"
|
||||
resources:
|
||||
requests:
|
||||
memory: "500Mi"
|
||||
cpu: "200m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "200m"
|
||||
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
|
||||
@@ -147,57 +147,6 @@ spec:
|
||||
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: scraper-deployment
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: scraper
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: scraper
|
||||
spec:
|
||||
containers:
|
||||
- name: scraper
|
||||
image: mhdzumair/mediafusion:beta
|
||||
command: ["pipenv", "run", "python", "-m", "api.scheduler"]
|
||||
env:
|
||||
- name: MONGO_URI
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: MONGO_URI
|
||||
- name: SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: SECRET_KEY
|
||||
- name: PROWLARR_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: PROWLARR_API_KEY
|
||||
- name: REDIS_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mediafusion-secrets
|
||||
key: REDIS_URL
|
||||
- name: db_max_connections
|
||||
value: "30"
|
||||
resources:
|
||||
requests:
|
||||
memory: "500Mi"
|
||||
cpu: "200m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "200m"
|
||||
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
|
||||
@@ -50,9 +50,6 @@ class LiveTVSpider(scrapy.Spider):
|
||||
)
|
||||
|
||||
def parse_categories(self, response):
|
||||
# Process the current page immediately
|
||||
yield from self.parse_page(response)
|
||||
|
||||
# Extract the total number of pages from the pagination text
|
||||
pagination_text = response.css("div.pagination span::text").get()
|
||||
total_pages = int(pagination_text.split(" ")[-1]) if pagination_text else 1
|
||||
@@ -64,13 +61,16 @@ class LiveTVSpider(scrapy.Spider):
|
||||
else:
|
||||
base_url = base_url.rstrip("/")
|
||||
|
||||
# Since we've already processed the first page, start from the second page
|
||||
page_urls = [f"{base_url}/page/{page}/" for page in range(2, total_pages + 1)]
|
||||
# Generate URLs for all pages in reverse order, excluding the first page since it's already processed
|
||||
page_urls = [f"{base_url}/page/{page}/" for page in range(total_pages, 1, -1)]
|
||||
|
||||
# Iterate over each subsequent page URL to scrape channels
|
||||
# Iterate over each subsequent page URL in reverse order to scrape channels
|
||||
for page_url in page_urls:
|
||||
yield scrapy.Request(page_url, callback=self.parse_page)
|
||||
|
||||
# Process the first page last
|
||||
yield from self.parse_page(response)
|
||||
|
||||
def parse_page(self, response):
|
||||
channel_elements = response.css("article.item.movies")
|
||||
source_name = response.css(".logo a img::attr(alt)").get(default="MHDTVWORLD")
|
||||
@@ -217,13 +217,17 @@ class LiveTVSpider(scrapy.Spider):
|
||||
},
|
||||
)
|
||||
|
||||
def get_behavior_hints(self, response):
|
||||
"""Generates behavior hints for requests."""
|
||||
def extract_m3u8_urls(self, response):
|
||||
"""Extracts M3U8 URLs using direct and fallback regex patterns."""
|
||||
m3u8_urls = self.direct_pattern.findall(response.text)
|
||||
if not m3u8_urls:
|
||||
m3u8_urls = self.fallback_pattern.findall(response.text)
|
||||
|
||||
user_agent = response.request.headers.get("User-Agent").decode()
|
||||
parsed_url = urlparse(response.url)
|
||||
referer = f"{parsed_url.scheme}://{parsed_url.netloc}"
|
||||
|
||||
return {
|
||||
behavior_hints = {
|
||||
"notWebReady": True,
|
||||
"proxyHeaders": {
|
||||
"request": {
|
||||
@@ -233,21 +237,15 @@ class LiveTVSpider(scrapy.Spider):
|
||||
},
|
||||
}
|
||||
|
||||
def extract_m3u8_urls(self, response):
|
||||
"""Extracts M3U8 URLs using direct and fallback regex patterns."""
|
||||
m3u8_urls = self.direct_pattern.findall(response.text)
|
||||
if not m3u8_urls:
|
||||
m3u8_urls = self.fallback_pattern.findall(response.text)
|
||||
return m3u8_urls
|
||||
return m3u8_urls, behavior_hints
|
||||
|
||||
def request_and_extract_video_url(self, response):
|
||||
channel_data = response.meta.get("channel_data")
|
||||
stream_title = response.meta.get("stream_title")
|
||||
country_name = response.meta.get("country_name")
|
||||
behavior_hints = self.get_behavior_hints(response)
|
||||
|
||||
# Extract M3U8 URLs
|
||||
m3u8_urls = self.extract_m3u8_urls(response)
|
||||
m3u8_urls, behavior_hints = self.extract_m3u8_urls(response)
|
||||
if not m3u8_urls:
|
||||
self.logger.error(
|
||||
"No M3U8 URLs found for channel url: %s, stream title: %s",
|
||||
@@ -281,6 +279,9 @@ class LiveTVSpider(scrapy.Spider):
|
||||
|
||||
if response.status == 200 and content_type in self.m3u8_valid_content_types:
|
||||
# Content type is valid, proceed with adding the stream
|
||||
if response.meta.get("redirect_times", 0) > 0:
|
||||
meta["behavior_hints"]["is_redirect"] = True
|
||||
|
||||
stream_info = {
|
||||
"name": f"{meta['stream_title']} - {meta['index']}"
|
||||
if meta["index"]
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import scrapy
|
||||
|
||||
from mediafusion_scrapy.spiders.live_tv import LiveTVSpider
|
||||
|
||||
|
||||
class TamilUltraSpider(LiveTVSpider):
|
||||
name = "tamilultra"
|
||||
start_urls = ["https://tamilultra.team/"]
|
||||
|
||||
def extract_player_api_base(self, response):
|
||||
"""Extracts the admin-ajax URL for POST requests."""
|
||||
# Directly extract the URL used for admin-ajax POST requests
|
||||
admin_ajax_url = response.xpath(
|
||||
"//script[contains(text(), 'player_api')]/text()"
|
||||
).re_first(r'"url":"([^"]+)"')
|
||||
if admin_ajax_url:
|
||||
# Correctly format and return the full URL
|
||||
admin_ajax_full_url = urljoin(
|
||||
response.url, admin_ajax_url.replace("\\/", "/")
|
||||
)
|
||||
return admin_ajax_full_url
|
||||
else:
|
||||
self.logger.error("Admin AJAX URL not found for TamilUltra.")
|
||||
return None
|
||||
|
||||
def process_player_option(self, element, channel_data, player_api_post_url):
|
||||
"""Processes each player option element to send a POST request."""
|
||||
stream_title, country_name = self.extract_stream_details(element)
|
||||
data_post, data_nume, data_type = (
|
||||
element.attrib.get("data-post"),
|
||||
element.attrib.get("data-nume"),
|
||||
element.attrib.get("data-type"),
|
||||
)
|
||||
|
||||
if all([data_post, data_nume, data_type]):
|
||||
form_data = {
|
||||
"action": "doo_player_ajax",
|
||||
"post": data_post,
|
||||
"nume": data_nume,
|
||||
"type": data_type,
|
||||
}
|
||||
yield scrapy.FormRequest(
|
||||
url=player_api_post_url,
|
||||
formdata=form_data,
|
||||
callback=self.parse_api_response,
|
||||
meta={
|
||||
"channel_data": channel_data,
|
||||
"stream_title": stream_title,
|
||||
"country_name": country_name,
|
||||
},
|
||||
)
|
||||
|
||||
def extract_m3u8_urls(self, response):
|
||||
"""Extracts M3U8 URLs using direct and fallback regex patterns."""
|
||||
query_string = urlparse(response.url).query
|
||||
m3u8_urls = [urljoin(response.url, query_string)]
|
||||
|
||||
user_agent = response.request.headers.get("User-Agent").decode()
|
||||
parsed_url = urlparse(response.url)
|
||||
referer = f"{parsed_url.scheme}://{parsed_url.netloc}"
|
||||
|
||||
behavior_hints = {
|
||||
"notWebReady": True,
|
||||
"proxyHeaders": {
|
||||
"request": {
|
||||
"User-Agent": user_agent,
|
||||
"Referer": referer,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return m3u8_urls, behavior_hints
|
||||
@@ -0,0 +1,24 @@
|
||||
from multiprocessing import Process
|
||||
|
||||
import dramatiq
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
|
||||
def run_spider_in_process(spider_name, *args, **kwargs):
|
||||
"""
|
||||
Function to start a scrapy spider in a new process.
|
||||
"""
|
||||
process = CrawlerProcess(get_project_settings())
|
||||
process.crawl(spider_name, *args, **kwargs)
|
||||
process.start()
|
||||
|
||||
|
||||
@dramatiq.actor(priority=5, time_limit=60 * 60 * 1000)
|
||||
def run_spider(spider_name: str, *args, **kwargs):
|
||||
"""
|
||||
Wrapper function to run the spider in a separate process.
|
||||
"""
|
||||
p = Process(target=run_spider_in_process, args=(spider_name, *args), kwargs=kwargs)
|
||||
p.start()
|
||||
p.join()
|
||||
Binary file not shown.
@@ -0,0 +1,72 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Scraper Controls - MediaFusion Stremio Addon</title>
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap-icons/font/bootstrap-icons.css" rel="stylesheet">
|
||||
<link href="/static/css/styles.css" rel="stylesheet">
|
||||
<link href="https://cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/css/toastr.min.css" rel="stylesheet"/>
|
||||
<link rel="shortcut icon" href="/static/images/mediafusion_logo.png" type="image/x-icon">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="row justify-content-center">
|
||||
<div class="col-10 col-lg-10 col-md-8 col-sm-10 config-container">
|
||||
<img class="logo" src="/static/images/mediafusion_logo.png">
|
||||
<h3 class="text-center mb-5"><b>Scraper Controls</b> - Manage Scraping Tasks</h3>
|
||||
|
||||
<!-- Scraper Selection -->
|
||||
<div class="section-container">
|
||||
<h4 class="section-header">Scraper Selection</h4>
|
||||
<hr class="section-divider">
|
||||
<div class="mb-3">
|
||||
<label for="scraperSelect" class="form-label">Select Scraper</label>
|
||||
<select class="form-select" id="scraperSelect" onchange="updateFormFields()">
|
||||
<option value="tamilmv">TamilMV</option>
|
||||
<option value="tamilblasters">TamilBlasters</option>
|
||||
<option value="scrapy">Scrapy Spider</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<!-- Dynamic Scraper Parameters -->
|
||||
<div id="scraperParameters"></div>
|
||||
</div>
|
||||
|
||||
<!-- API Password Configuration -->
|
||||
{% if authentication_required %}
|
||||
<div class="section-container">
|
||||
<h4 class="section-header">API Security Configuration</h4>
|
||||
<hr class="section-divider">
|
||||
<div class="mb-3">
|
||||
<label for="api_password">API Password:</label>
|
||||
<div class="input-group">
|
||||
<input class="form-control" type="password" id="api_password" name="api_password" placeholder="Enter API Password">
|
||||
<button class="btn btn-outline-secondary" type="button" id="toggleApiPassword">
|
||||
<span id="toggleApiPasswordIcon" class="bi bi-eye"></span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="button-container mt-5">
|
||||
<button type="button" class="btn btn-primary" onclick="submitScraperForm()">Run Scraper</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- JS for Bootstrap -->
|
||||
<script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.8/dist/umd/popper.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.min.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/js/toastr.min.js"></script>
|
||||
<script src="/static/js/scraperControl.js"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -0,0 +1,95 @@
|
||||
// ---- Helper Functions ----
|
||||
|
||||
function showNotification(message, type = 'info') {
|
||||
toastr.options = {
|
||||
closeButton: true,
|
||||
newestOnTop: true,
|
||||
progressBar: true,
|
||||
positionClass: "toast-top-center",
|
||||
preventDuplicates: true,
|
||||
onclick: null,
|
||||
showDuration: "300",
|
||||
hideDuration: "1000",
|
||||
timeOut: "5000",
|
||||
extendedTimeOut: "1000",
|
||||
showEasing: "swing",
|
||||
hideEasing: "linear",
|
||||
showMethod: "fadeIn",
|
||||
hideMethod: "fadeOut",
|
||||
};
|
||||
|
||||
toastr[type](message);
|
||||
}
|
||||
|
||||
|
||||
// Function to update form fields based on scraper selection
|
||||
function updateFormFields() {
|
||||
const scraperType = document.getElementById('scraperSelect').value;
|
||||
let parametersHtml = '';
|
||||
const scraperParametersDiv = document.getElementById('scraperParameters');
|
||||
|
||||
if (scraperType === 'tamilmv' || scraperType === 'tamilblasters') {
|
||||
parametersHtml += `
|
||||
<div class="mb-3">
|
||||
<label for="pages" class="form-label">Number of Pages</label>
|
||||
<input type="number" class="form-control" id="pages" name="pages" value="1">
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="startPage" class="form-label">Start Page</label>
|
||||
<input type="number" class="form-control" id="startPage" name="start_page" value="1">
|
||||
</div>
|
||||
`;
|
||||
} else if (scraperType === 'scrapy') {
|
||||
parametersHtml += `
|
||||
<div class="mb-3">
|
||||
<label for="spiderName" class="form-label">Spider Name</label>
|
||||
<select class="form-select" id="spiderName" name="spider_name">
|
||||
<option value="formula_tgx">Formula TGX</option>
|
||||
<option value="mhdtvworld">MHDTV World</option>
|
||||
<option value="mhdtvsports">MHDTV Sports</option>
|
||||
<option value="tamilultra">Tamil Ultra</option>
|
||||
</select>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
// Update the form fields based on the selection
|
||||
scraperParametersDiv.innerHTML = parametersHtml;
|
||||
}
|
||||
|
||||
// Function to submit the form and display the response using Toastr
|
||||
async function submitScraperForm() {
|
||||
const apiPassword = document.getElementById('api_password') ? document.getElementById('api_password').value : '';
|
||||
const scraperType = document.getElementById('scraperSelect').value;
|
||||
let payload = {
|
||||
scraper_type: scraperType,
|
||||
api_password: apiPassword,
|
||||
pages: document.getElementById('pages') ? document.getElementById('pages').value : 1,
|
||||
start_page: document.getElementById('startPage') ? document.getElementById('startPage').value : 1,
|
||||
spider_name: document.getElementById('spiderName') ? document.getElementById('spiderName').value : ''
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch("/scraper/run", {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (response.ok) {
|
||||
showNotification(data.status, 'success');
|
||||
} else {
|
||||
showNotification(data.detail, 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error submitting scraper form:', error);
|
||||
showNotification('Error submitting scraper form. Please check the console for more details.', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Initial update for form fields on page load
|
||||
document.addEventListener('DOMContentLoaded', updateFormFields);
|
||||
+2
-2
@@ -129,7 +129,7 @@ def get_scraper_config(site_name: str, get_key: str) -> dict:
|
||||
return config.get(site_name, {}).get(get_key, {})
|
||||
|
||||
|
||||
@dramatiq.actor(time_limit=30 * 60 * 1000)
|
||||
@dramatiq.actor(time_limit=30 * 60 * 1000, priority=10)
|
||||
async def update_torrent_movie_streams_metadata(info_hashes: list[str]):
|
||||
"""Update torrent streams metadata."""
|
||||
if not info_hashes:
|
||||
@@ -153,7 +153,7 @@ async def update_torrent_movie_streams_metadata(info_hashes: list[str]):
|
||||
logging.info(f"Updated {torrent_stream.id} metadata")
|
||||
|
||||
|
||||
@dramatiq.actor(time_limit=30 * 60 * 1000)
|
||||
@dramatiq.actor(time_limit=30 * 60 * 1000, priority=10)
|
||||
async def update_torrent_series_streams_metadata(info_hashes: list[str]):
|
||||
"""Update torrent streams metadata."""
|
||||
if not info_hashes:
|
||||
|
||||
@@ -141,6 +141,7 @@ async def scrap_movies_streams_from_prowlarr(
|
||||
min_backoff=2 * 60 * 1000, # 2 minutes
|
||||
max_backoff=60 * 60 * 1000, # 60 minutes
|
||||
retry_when=should_retry_prowlarr_scrap,
|
||||
priority=100,
|
||||
)
|
||||
async def background_movie_title_search(video_id: str, title: str, year: str):
|
||||
url = f"{settings.prowlarr_url}/api/v1/search"
|
||||
@@ -189,6 +190,7 @@ async def scrap_series_streams_from_prowlarr(
|
||||
min_backoff=2 * 60 * 1000, # 2 minutes
|
||||
max_backoff=60 * 60 * 1000, # 60 minutes
|
||||
retry_when=should_retry_prowlarr_scrap,
|
||||
priority=100,
|
||||
)
|
||||
async def background_series_title_search(
|
||||
video_id: str, title: str, season: int, episode: int
|
||||
|
||||
+28
-13
@@ -6,7 +6,9 @@ import logging
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
from multiprocessing import Process
|
||||
|
||||
import dramatiq
|
||||
from bs4 import BeautifulSoup
|
||||
from dateutil.parser import parse as dateparser
|
||||
from playwright.async_api import async_playwright
|
||||
@@ -221,7 +223,6 @@ async def run_scraper(
|
||||
search_keyword: str = None,
|
||||
scrap_with_playwright: bool = None,
|
||||
):
|
||||
await database.init()
|
||||
if search_keyword:
|
||||
await scrap_search_keyword(search_keyword)
|
||||
return
|
||||
@@ -242,19 +243,33 @@ async def run_scraper(
|
||||
|
||||
|
||||
async def run_schedule_scrape(
|
||||
pages: int = 1,
|
||||
start_page: int = 1,
|
||||
scrap_with_playwright: bool = None,
|
||||
pages: int = 1, start_page: int = 1, scrap_with_playwright: bool = False
|
||||
):
|
||||
for language in TAMIL_BLASTER_CATALOGS:
|
||||
for video_type in TAMIL_BLASTER_CATALOGS[language]:
|
||||
await run_scraper(
|
||||
language,
|
||||
video_type,
|
||||
pages=pages,
|
||||
start_page=start_page,
|
||||
scrap_with_playwright=scrap_with_playwright,
|
||||
)
|
||||
await database.init()
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
for language in TAMIL_BLASTER_CATALOGS:
|
||||
for video_type in TAMIL_BLASTER_CATALOGS[language]:
|
||||
tg.create_task(
|
||||
run_scraper(
|
||||
language,
|
||||
video_type,
|
||||
pages=pages,
|
||||
start_page=start_page,
|
||||
scrap_with_playwright=scrap_with_playwright,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run_schedule_scrape_sync(pages, start_page, scrap_with_playwright):
|
||||
asyncio.run(run_schedule_scrape(pages, start_page, scrap_with_playwright))
|
||||
|
||||
|
||||
@dramatiq.actor(priority=5, time_limit=60 * 60 * 1000)
|
||||
def run_tamil_blasters_scraper(pages: int = 1, start_page: int = 1):
|
||||
# Use a separate process to run the scraper
|
||||
process = Process(target=run_schedule_scrape_sync, args=(pages, start_page, False))
|
||||
process.start()
|
||||
process.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+27
-10
@@ -6,7 +6,9 @@ import logging
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
from multiprocessing import Process
|
||||
|
||||
import dramatiq
|
||||
from bs4 import BeautifulSoup
|
||||
from dateutil.parser import parse as dateparser
|
||||
from playwright.async_api import async_playwright
|
||||
@@ -196,7 +198,6 @@ async def run_scraper(
|
||||
search_keyword: str = None,
|
||||
scrap_with_playwright: bool = None,
|
||||
):
|
||||
await database.init()
|
||||
if search_keyword:
|
||||
await scrap_search_keyword(search_keyword)
|
||||
return
|
||||
@@ -228,15 +229,31 @@ async def run_schedule_scrape(
|
||||
start_page: int = 1,
|
||||
scrap_with_playwright: bool = None,
|
||||
):
|
||||
for language in TAMIL_MV_CATALOGS:
|
||||
for video_type in TAMIL_MV_CATALOGS[language]:
|
||||
await run_scraper(
|
||||
language,
|
||||
video_type,
|
||||
pages=pages,
|
||||
start_page=start_page,
|
||||
scrap_with_playwright=scrap_with_playwright,
|
||||
)
|
||||
await database.init()
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
for language in TAMIL_MV_CATALOGS:
|
||||
for video_type in TAMIL_MV_CATALOGS[language]:
|
||||
tg.create_task(
|
||||
run_scraper(
|
||||
language,
|
||||
video_type,
|
||||
pages=pages,
|
||||
start_page=start_page,
|
||||
scrap_with_playwright=scrap_with_playwright,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run_schedule_scrape_sync(pages, start_page, scrap_with_playwright):
|
||||
asyncio.run(run_schedule_scrape(pages, start_page, scrap_with_playwright))
|
||||
|
||||
|
||||
@dramatiq.actor(priority=5, time_limit=60 * 60 * 1000)
|
||||
def run_tamilmv_scraper(pages: int = 1, start_page: int = 1):
|
||||
# Use a separate process to run the scraper
|
||||
process = Process(target=run_schedule_scrape_sync, args=(pages, start_page, False))
|
||||
process.start()
|
||||
process.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -177,10 +177,10 @@ async def retrieve_or_download_file(
|
||||
pikpak, info_hash, stream.torrent_name, max_retries, retry_interval
|
||||
)
|
||||
selected_file = await find_file_in_folder_tree(pikpak, folder_id, filename)
|
||||
if selected_file is None:
|
||||
raise ProviderException(
|
||||
"No matching file available for this torrent", "no_matching_file.mp4"
|
||||
)
|
||||
if selected_file is None:
|
||||
raise ProviderException(
|
||||
"No matching file available for this torrent", "no_matching_file.mp4"
|
||||
)
|
||||
return selected_file
|
||||
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/{secret_str}/stream", tags=["streaming_provider"])
|
||||
@wrappers.exclude
|
||||
@wrappers.exclude_rate_limit
|
||||
@wrappers.auth_required
|
||||
async def streaming_provider_endpoint(
|
||||
secret_str: str,
|
||||
|
||||
@@ -110,3 +110,15 @@ async def batch_process_with_circuit_breaker(
|
||||
await asyncio.sleep(rate_limit_delay) # Always respect the rate limit
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def get_redirector_url(url: str, headers: dict) -> str | None:
|
||||
"""
|
||||
Get the final URL after following all redirects.
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.head(url, headers=headers, follow_redirects=True)
|
||||
return str(response.url)
|
||||
except httpx.HTTPError as e:
|
||||
return
|
||||
|
||||
+20
-9
@@ -6,7 +6,7 @@ import requests
|
||||
from imdb import Cinemagoer
|
||||
|
||||
from db.config import settings
|
||||
from db.models import TorrentStreams, MediaFusionTVMetaData
|
||||
from db.models import TorrentStreams, TVStreams
|
||||
from db.schemas import Stream, UserData
|
||||
from streaming_providers.alldebrid.utils import (
|
||||
update_ad_cache_status,
|
||||
@@ -45,6 +45,7 @@ from streaming_providers.torbox.utils import (
|
||||
fetch_downloaded_info_hashes_from_torbox,
|
||||
)
|
||||
from utils import const
|
||||
from utils.network import get_redirector_url
|
||||
|
||||
ia = Cinemagoer()
|
||||
ADULT_CONTENT_KEYWORDS = re.compile(
|
||||
@@ -311,27 +312,37 @@ def get_imdb_data(video_id: str) -> tuple[str, str]:
|
||||
return movie.get("title"), movie.get("year")
|
||||
|
||||
|
||||
def parse_tv_stream_data(tv_data: MediaFusionTVMetaData) -> list[Stream]:
|
||||
async def parse_tv_stream_data(tv_streams: list[TVStreams]) -> list[Stream]:
|
||||
stream_list = []
|
||||
for stream in tv_data.streams:
|
||||
for stream in tv_streams:
|
||||
if stream.behaviorHints.get("is_redirect", False):
|
||||
response = requests.get(
|
||||
stream_link = await get_redirector_url(
|
||||
stream.url,
|
||||
headers=stream.behaviorHints["proxyHeaders"]["request"],
|
||||
allow_redirects=False,
|
||||
stream.behaviorHints.get("proxyHeaders", {}).get("request", {}),
|
||||
)
|
||||
if response.status_code == 302:
|
||||
stream.url = response.headers["Location"]
|
||||
if stream_link is None:
|
||||
continue
|
||||
stream.url = stream_link
|
||||
stream_list.append(
|
||||
Stream(
|
||||
name="MediaFusion",
|
||||
description=f"{stream.name}, {tv_data.tv_language}, {stream.source}",
|
||||
description=f"📺 {stream.name}\n🌐 {stream.country}\n🔗 {stream.source}",
|
||||
url=stream.url,
|
||||
ytId=stream.ytId,
|
||||
behaviorHints=stream.behaviorHints,
|
||||
)
|
||||
)
|
||||
|
||||
if not stream_list:
|
||||
stream_list.append(
|
||||
Stream(
|
||||
name="MediaFusion",
|
||||
description="No streams live at the moment.",
|
||||
url=f"{settings.host_url}/static/exceptions/source_not_live.mp4",
|
||||
behaviorHints={"notWebReady": True},
|
||||
)
|
||||
)
|
||||
|
||||
return stream_list
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import aiohttp
|
||||
import dramatiq
|
||||
from aiohttp import ClientError
|
||||
|
||||
from db import schemas
|
||||
@@ -38,7 +39,7 @@ async def validate_m3u8_url(url: str, behaviour_hint: dict) -> bool:
|
||||
headers = behaviour_hint.get("proxyHeaders", {}).get("request", {})
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
async with session.get(
|
||||
async with session.head(
|
||||
url,
|
||||
allow_redirects=True,
|
||||
headers=headers,
|
||||
@@ -152,3 +153,22 @@ def is_video_file(filename: str) -> bool:
|
||||
".yuv",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dramatiq.actor(time_limit=30 * 60 * 1000, priority=5) # time limit is 30 minutes
|
||||
async def validate_tv_streams_in_db():
|
||||
"""Validate TV streams in the database."""
|
||||
from db.models import TVStreams
|
||||
|
||||
async def validate_and_update_tv_stream(stream):
|
||||
is_valid = await validate_m3u8_url(stream.url, stream.behaviorHints)
|
||||
stream.is_working = is_valid
|
||||
await stream.save()
|
||||
logging.info(f"Stream: {stream.name}, Status: {is_valid}")
|
||||
|
||||
tv_streams = await TVStreams.all().to_list()
|
||||
tasks = [
|
||||
asyncio.create_task(validate_and_update_tv_stream(stream))
|
||||
for stream in tv_streams
|
||||
]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
+2
-2
@@ -23,12 +23,12 @@ def rate_limit(limit: int, window: int, scope: str = None):
|
||||
return decorator
|
||||
|
||||
|
||||
def exclude(func):
|
||||
def exclude_rate_limit(func):
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
wrapper.exclude = True
|
||||
wrapper.exclude_rate_limit = True
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user