diff --git a/Pipfile.lock b/Pipfile.lock index 65f536f..79171ff 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -153,14 +153,6 @@ "markers": "python_version >= '3.6'", "version": "==3.10.4" }, - "async-timeout": { - "hashes": [ - "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f", - "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028" - ], - "markers": "python_version >= '3.7'", - "version": "==4.0.3" - }, "attrs": { "hashes": [ "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", @@ -1219,11 +1211,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "parse-torrent-title": { "git": "git+https://github.com/mhdzumair/parse-torrent-title", @@ -1231,11 +1223,11 @@ }, "parsel": { "hashes": [ - "sha256:2708fc74daeeb4ce471e2c2e9089b650ec940c7a218053e57421e69b5b00f82c", - "sha256:aff28e68c9b3f1a901db2a4e3f158d8480a38724d7328ee751c1a4e1c1801e39" + "sha256:1272baeda331cbc7f3a9a91df4c15c39276dc94b4ff2e8d5cffdf8bb6c382ea1", + "sha256:a5a6bcb0c5fc741540ba3075636ff5cb968852a78cc16ae82250e5d486fa7d48" ], - "markers": "python_version >= '3.7'", - "version": "==1.8.1" + "markers": "python_version >= '3.8'", + "version": "==1.9.0" }, "pikpakapi": { "git": "git+https://github.com/mhdzumair/PikPakAPI.git", @@ -1420,12 +1412,12 @@ }, "pydantic": { "hashes": [ - "sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a", - "sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f" + "sha256:b1704e0847db01817624a6b86766967f552dd9dbf3afba4004409f908dcc84e6", + "sha256:cc46fce86607580867bdc3361ad462bab9c222ef042d3da86f2fb333e1d916c5" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==2.6.3" + "version": "==2.6.4" }, "pydantic-core": { "hashes": [ @@ -1629,19 +1621,19 @@ }, "pyopenssl": { "hashes": [ - "sha256:6aa33039a93fffa4563e655b61d11364d01264be8ccb49906101e02a334530bf", - "sha256:ba07553fb6fd6a7a2259adb9b84e12302a9a8a75c44046e8bb5d3e5ee887e3c3" + "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad", + "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f" ], "markers": "python_version >= '3.7'", - "version": "==24.0.0" + "version": "==24.1.0" }, "pyparsing": { "hashes": [ - "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb", - "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db" + "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad", + "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742" ], "markers": "python_full_version >= '3.6.8'", - "version": "==3.1.1" + "version": "==3.1.2" }, "python-dateutil": { "hashes": [ @@ -1733,110 +1725,110 @@ }, "rapidfuzz": { "hashes": [ - "sha256:01835d02acd5d95c1071e1da1bb27fe213c84a013b899aba96380ca9962364bc", - "sha256:01eb03cd880a294d1bf1a583fdd00b87169b9cc9c9f52587411506658c864d73", - "sha256:03f73b381bdeccb331a12c3c60f1e41943931461cdb52987f2ecf46bfc22f50d", - "sha256:0402f1629e91a4b2e4aee68043a30191e5e1b7cd2aa8dacf50b1a1bcf6b7d3ab", - "sha256:060bd7277dc794279fa95522af355034a29c90b42adcb7aa1da358fc839cdb11", - "sha256:064c1d66c40b3a0f488db1f319a6e75616b2e5fe5430a59f93a9a5e40a656d15", - "sha256:06e98ff000e2619e7cfe552d086815671ed09b6899408c2c1b5103658261f6f3", - "sha256:08b6fb47dd889c69fbc0b915d782aaed43e025df6979b6b7f92084ba55edd526", - "sha256:0a9fc714b8c290261669f22808913aad49553b686115ad0ee999d1cb3df0cd66", - "sha256:0bbfae35ce4de4c574b386c43c78a0be176eeddfdae148cb2136f4605bebab89", - "sha256:12ff8eaf4a9399eb2bebd838f16e2d1ded0955230283b07376d68947bbc2d33d", - "sha256:1936d134b6c513fbe934aeb668b0fee1ffd4729a3c9d8d373f3e404fbb0ce8a0", - "sha256:1c47d592e447738744905c18dda47ed155620204714e6df20eb1941bb1ba315e", - "sha256:1dfc557c0454ad22382373ec1b7df530b4bbd974335efe97a04caec936f2956a", - "sha256:1e12319c6b304cd4c32d5db00b7a1e36bdc66179c44c5707f6faa5a889a317c0", - "sha256:23de71e7f05518b0bbeef55d67b5dbce3bcd3e2c81e7e533051a2e9401354eb0", - "sha256:266dd630f12696ea7119f31d8b8e4959ef45ee2cbedae54417d71ae6f47b9848", - "sha256:2963f4a3f763870a16ee076796be31a4a0958fbae133dbc43fc55c3968564cf5", - "sha256:2a791168e119cfddf4b5a40470620c872812042f0621e6a293983a2d52372db0", - "sha256:2b155e67fff215c09f130555002e42f7517d0ea72cbd58050abb83cb7c880cec", - "sha256:2b19795b26b979c845dba407fe79d66975d520947b74a8ab6cee1d22686f7967", - "sha256:2e03038bfa66d2d7cffa05d81c2f18fd6acbb25e7e3c068d52bb7469e07ff382", - "sha256:3028ee8ecc48250607fa8a0adce37b56275ec3b1acaccd84aee1f68487c8557b", - "sha256:35660bee3ce1204872574fa041c7ad7ec5175b3053a4cb6e181463fc07013de7", - "sha256:3c772d04fb0ebeece3109d91f6122b1503023086a9591a0b63d6ee7326bd73d9", - "sha256:3c84294f4470fcabd7830795d754d808133329e0a81d62fcc2e65886164be83b", - "sha256:40cced1a8852652813f30fb5d4b8f9b237112a0bbaeebb0f4cc3611502556764", - "sha256:4243a9c35667a349788461aae6471efde8d8800175b7db5148a6ab929628047f", - "sha256:42f211e366e026de110a4246801d43a907cd1a10948082f47e8a4e6da76fef52", - "sha256:4381023fa1ff32fd5076f5d8321249a9aa62128eb3f21d7ee6a55373e672b261", - "sha256:484759b5dbc5559e76fefaa9170147d1254468f555fd9649aea3bad46162a88b", - "sha256:49b9ed2472394d306d5dc967a7de48b0aab599016aa4477127b20c2ed982dbf9", - "sha256:53251e256017e2b87f7000aee0353ba42392c442ae0bafd0f6b948593d3f68c6", - "sha256:588c4b20fa2fae79d60a4e438cf7133d6773915df3cc0a7f1351da19eb90f720", - "sha256:5a2f3e9df346145c2be94e4d9eeffb82fab0cbfee85bd4a06810e834fe7c03fa", - "sha256:5d82b9651e3d34b23e4e8e201ecd3477c2baa17b638979deeabbb585bcb8ba74", - "sha256:5dd95b6b7bfb1584f806db89e1e0c8dbb9d25a30a4683880c195cc7f197eaf0c", - "sha256:692c9a50bea7a8537442834f9bc6b7d29d8729a5b6379df17c31b6ab4df948c2", - "sha256:6b0ccc2ec1781c7e5370d96aef0573dd1f97335343e4982bdb3a44c133e27786", - "sha256:6dede83a6b903e3ebcd7e8137e7ff46907ce9316e9d7e7f917d7e7cdc570ee05", - "sha256:7142ee354e9c06e29a2636b9bbcb592bb00600a88f02aa5e70e4f230347b373e", - "sha256:7183157edf0c982c0b8592686535c8b3e107f13904b36d85219c77be5cefd0d8", - "sha256:7420e801b00dee4a344ae2ee10e837d603461eb180e41d063699fb7efe08faf0", - "sha256:757dfd7392ec6346bd004f8826afb3bf01d18a723c97cbe9958c733ab1a51791", - "sha256:76c23ceaea27e790ddd35ef88b84cf9d721806ca366199a76fd47cfc0457a81b", - "sha256:7fec74c234d3097612ea80f2a80c60720eec34947066d33d34dc07a3092e8105", - "sha256:82300e5f8945d601c2daaaac139d5524d7c1fdf719aa799a9439927739917460", - "sha256:841eafba6913c4dfd53045835545ba01a41e9644e60920c65b89c8f7e60c00a9", - "sha256:8d7a072f10ee57c8413c8ab9593086d42aaff6ee65df4aa6663eecdb7c398dca", - "sha256:8e4da90e4c2b444d0a171d7444ea10152e07e95972bb40b834a13bdd6de1110c", - "sha256:96cd19934f76a1264e8ecfed9d9f5291fde04ecb667faef5f33bdbfd95fe2d1f", - "sha256:a03863714fa6936f90caa7b4b50ea59ea32bb498cc91f74dc25485b3f8fccfe9", - "sha256:a1788ebb5f5b655a15777e654ea433d198f593230277e74d51a2a1e29a986283", - "sha256:a3ee4f8f076aa92184e80308fc1a079ac356b99c39408fa422bbd00145be9854", - "sha256:a490cd645ef9d8524090551016f05f052e416c8adb2d8b85d35c9baa9d0428ab", - "sha256:a553cc1a80d97459d587529cc43a4c7c5ecf835f572b671107692fe9eddf3e24", - "sha256:a59472b43879012b90989603aa5a6937a869a72723b1bf2ff1a0d1edee2cc8e6", - "sha256:ac434fc71edda30d45db4a92ba5e7a42c7405e1a54cb4ec01d03cc668c6dcd40", - "sha256:ad9d74ef7c619b5b0577e909582a1928d93e07d271af18ba43e428dc3512c2a1", - "sha256:ae598a172e3a95df3383634589660d6b170cc1336fe7578115c584a99e0ba64d", - "sha256:b2ef4c0fd3256e357b70591ffb9e8ed1d439fb1f481ba03016e751a55261d7c1", - "sha256:b3e5af946f419c30f5cb98b69d40997fe8580efe78fc83c2f0f25b60d0e56efb", - "sha256:b53137d81e770c82189e07a8f32722d9e4260f13a0aec9914029206ead38cac3", - "sha256:b7e3375e4f2bfec77f907680328e4cd16cc64e137c84b1886d547ab340ba6928", - "sha256:bcc957c0a8bde8007f1a8a413a632a1a409890f31f73fe764ef4eac55f59ca87", - "sha256:be156f51f3a4f369e758505ed4ae64ea88900dcb2f89d5aabb5752676d3f3d7e", - "sha256:be368573255f8fbb0125a78330a1a40c65e9ba3c5ad129a426ff4289099bfb41", - "sha256:c1a23eee225dfb21c07f25c9fcf23eb055d0056b48e740fe241cbb4b22284379", - "sha256:c65f92881753aa1098c77818e2b04a95048f30edbe9c3094dc3707d67df4598b", - "sha256:ca3dfcf74f2b6962f411c33dd95b0adf3901266e770da6281bc96bb5a8b20de9", - "sha256:cd4ba4c18b149da11e7f1b3584813159f189dc20833709de5f3df8b1342a9759", - "sha256:d056e342989248d2bdd67f1955bb7c3b0ecfa239d8f67a8dfe6477b30872c607", - "sha256:d2f0274595cc5b2b929c80d4e71b35041104b577e118cf789b3fe0a77b37a4c5", - "sha256:d73dcfe789d37c6c8b108bf1e203e027714a239e50ad55572ced3c004424ed3b", - "sha256:d79aec8aeee02ab55d0ddb33cea3ecd7b69813a48e423c966a26d7aab025cdfe", - "sha256:da3e8c9f7e64bb17faefda085ff6862ecb3ad8b79b0f618a6cf4452028aa2222", - "sha256:dad55a514868dae4543ca48c4e1fc0fac704ead038dafedf8f1fc0cc263746c1", - "sha256:dec307b57ec2d5054d77d03ee4f654afcd2c18aee00c48014cb70bfed79597d6", - "sha256:e06c4242a1354cf9d48ee01f6f4e6e19c511d50bb1e8d7d20bcadbb83a2aea90", - "sha256:e19d519386e9db4a5335a4b29f25b8183a1c3f78cecb4c9c3112e7f86470e37f", - "sha256:e49b9575d16c56c696bc7b06a06bf0c3d4ef01e89137b3ddd4e2ce709af9fe06", - "sha256:ebcfb5bfd0a733514352cfc94224faad8791e576a80ffe2fd40b2177bf0e7198", - "sha256:ed0f712e0bb5fea327e92aec8a937afd07ba8de4c529735d82e4c4124c10d5a0", - "sha256:edf97c321fd641fea2793abce0e48fa4f91f3c202092672f8b5b4e781960b891", - "sha256:eef8b346ab331bec12bbc83ac75641249e6167fab3d84d8f5ca37fd8e6c7a08c", - "sha256:f056ba42fd2f32e06b2c2ba2443594873cfccc0c90c8b6327904fc2ddf6d5799", - "sha256:f382f7ffe384ce34345e1c0b2065451267d3453cadde78946fbd99a59f0cc23c", - "sha256:f59d19078cc332dbdf3b7b210852ba1f5db8c0a2cd8cc4c0ed84cc00c76e6802", - "sha256:fbc07e2e4ac696497c5f66ec35c21ddab3fc7a406640bffed64c26ab2f7ce6d6", - "sha256:fde9b14302a31af7bdafbf5cfbb100201ba21519be2b9dedcf4f1048e4fbe65d" + "sha256:012221629d54d3bee954148247f711eb86d4d390b589ebfe03172ea0b37a7531", + "sha256:080cb71b50cb6aff11d1c6aeb157f273e2da0b2bdb3f9d7b01257e49e69a8576", + "sha256:153d065e353371cc0aeff32b99999a5758266a64e958d1364189367c1c9f6814", + "sha256:16270b5529de83b7bae7457e952e4d9cf3fbf029a837dd32d415bb9e0eb8e599", + "sha256:1b86b93d93020c2b3edc1665d75c8855784845fc0a739b312c26c3a4bf0c80d5", + "sha256:1c54d3c85e522d3ac9ee39415f183c8fa184c4f87e7e5a37938f15a6d50e853a", + "sha256:1df2faf80201952e252413b6fac6f3e146080dcebb87bb1bb722508e67558ed8", + "sha256:1dfe4c24957474ce0ac75d886387e30e292b4be39228a6d71f76de414dc187db", + "sha256:259364199cbfeca33b1af369fc7951f71717aa285184a3fa5a7b1772da1b89db", + "sha256:28243086ed0e50808bb56632e5442c457241646aeafafd501ac87901f40a3237", + "sha256:28c011fb31f2c3f82f503aedd6097d3d3854e574e327a119a3b7eb2cf90b79ca", + "sha256:2a46220f86a5f9cb016af31525e0d0865cad437d02239aa0d8aed2ab8bff1f1c", + "sha256:2acd2514defce81e6ff4bbff50252d5e7df8e85a731442c4b83e44c86cf1c916", + "sha256:2d580d937146e803c8e5e1b87916cab8d6f84013b6392713e201efcda335c7d8", + "sha256:3168ff565d4b8c239cf11fb604dd2507d30e9bcaac76a4077c0ac23cf2c866ed", + "sha256:337e357f693130c4c6be740652542b260e36f622c59e01fa33d58f1d2750c930", + "sha256:3549123fca5bb817341025f98e8e49ca99f84596c7c4f92b658f8e5836040d4a", + "sha256:354ad5fe655beb7b279390cb58334903931c5452ecbad1b1666ffb06786498e2", + "sha256:35aeac852bca06023d6bbd50c1fc504ca5a9a3613d5e75a140f0be7601fa34ef", + "sha256:35bd4bc9c40e6994c5d6edea4b9319388b4d9711c13c66d543bb4c37624b4184", + "sha256:36ce7b68a7b90b787cdd73480a68d2f1ca63c31a3a9d5a79a8736f978e1e9344", + "sha256:380586664f2f63807050ddb95e7702888b4f0b425abf17655940c411f39287ad", + "sha256:3ae7c86914cb6673e97e187ba431b9c4cf4177d9ae77f8a1e5b2ba9a5628839e", + "sha256:3c1dc078ef371fce09f9f3eec2ca4eaa2a8cd412ec53941015b4f39f14d34407", + "sha256:4209816626d8d6ff8ae7dc248061c6059e618b70c6e6f6e4d7444ae3740b2b85", + "sha256:423c7c588b09d618601097b7a0017dfcb91132a2076bef29023c5f3cd2dc3de1", + "sha256:48e1eaea8fcd522fca7f04f0480663f0f0cfb77957092cce60a93f4462864996", + "sha256:495c0d8e14e6f12520eb7fc71b9ba9fcaafb47fc23a654e6e89b6c7985ec0020", + "sha256:4c279928651ce0e9e5220dcb25a00cc53b65e592a0861336a38299bcdca3a596", + "sha256:4edcceebb85ebfa49a3ddcde20ad891d36c08dc0fd592efdab0e7d313a4e36af", + "sha256:4fcfa23b5553b27f4016df77c53172ea743454cf12c28cfa7c35a309a2be93b3", + "sha256:53597fd72a9340bcdd80d3620f4957c2b92f9b569313b969a3abdaffd193aae6", + "sha256:5378c04102c7f084cde30a100154fa6d7e2baf0d51a6bdd2f912545559c1fb35", + "sha256:617949a70150e6fffdaed19253dd49f7a53528411dc8bf7663d499ba21e0f61e", + "sha256:62df2136068e2515ed8beb01756381ff62c29384d785e3bf46e3111d4ea3ba1e", + "sha256:6440ed0b3007c1c9286b0b88fe2ab2d9e83edd60cd62293b3dfabb732b4e8a30", + "sha256:6468f8bc8c3c50604f43631550ef9cfec873515dba5023ca34d461be94669fc8", + "sha256:6478f7803efebf5f644d0b758439c5b25728550fdfbb19783d150004c46a75a9", + "sha256:66b008bf2972740cd2dda5d382eb8bdb87265cd88198e71c7797bdc0d1f79d20", + "sha256:673ba2c343644805acdae1cb949c6a4de71aa2f62a998978551ebea59603af3f", + "sha256:6d67b649bf3e1b1722d04eca44d37919aef88305ce7ad05564502d013cf550fd", + "sha256:6ee98d88ae9ccc77ff61992ed33b2496478def5dc0da55c9a9aa06fcb725a352", + "sha256:7382c90170f60c846c81a07ddd80bb2e8c43c8383754486fa37f67391a571897", + "sha256:74c6773b11445b5e5cf93ca383171cd0ac0cdeafea11a7b2a5688f8bf8d813e6", + "sha256:7d830bc7a9b586a374147ec60b08b1f9ae5996b43f75cc514f37faef3866b519", + "sha256:7f18397c8d6a65fc0b288d2fc29bc7baeea6ba91eeb95163a3cd98f23cd3bc85", + "sha256:7f9fa14136a5b0cba1ec42531f7c3e0b0d3edb7fd6bc5e5ae7b498541f3855ab", + "sha256:80e51b22a7da83f9c87a97e92df07ed0612c74c35496590255f4b5d5b4212dfe", + "sha256:81a630ed2fc3ec5fc7400eb66bab1f87e282b4d47f0abe3e48c6634dfa13b5e4", + "sha256:824cc381cf81cbf8d158f6935664ec2a69e6ac3b1d39fa201988bf81a257f775", + "sha256:84c1032ae42628465b7a5cc35249906061e18a8193c9c27cbd2db54e9823a9a6", + "sha256:85a5b6e026393fe39fb61146b9c17c5af66fffbe1410e992c4bb06d9ec327bd3", + "sha256:87ac3a87f2251ae2e95fc9478ca5c759de6d141d04c84d3fec9f9cdcfc167b33", + "sha256:95a49c6b8bf1229743ae585dd5b7d57f0d15a7eb6e826866d5c9965ba958503c", + "sha256:9a07dffac439223b4f1025dbfc68f4445a3460a859309c9858c2a3fa29617cdc", + "sha256:9a74102fc5a2534fe91f7507838623e1f3a149d8e05648389c42bb42e14b1c3f", + "sha256:9bcc91ebd8fc69a6bd3b5711c8250f5f4e70606b4da75ef415f57ad209978205", + "sha256:9d457c89bac1471442002e70551e8268e639b3870b4a4521eae363c07253be87", + "sha256:a5637e6bf11b15b5aff6ee818c76bdec99ad208511b78985e6209ba648a6e3ee", + "sha256:a7895e04a22d6515bc91a850e0831f2405547605aa311d1ffec51e4818abc3c1", + "sha256:a945567c2b0b6e069454c9782d5234b0b6795718adf7a9f868bd3144afa6a023", + "sha256:ab269adfc64480f209e99f253391a10735edd5c09046e04899adab5fb132f20e", + "sha256:af7c19ec86e11488539380d3db1755be5d561a3c0e7b04ff9d07abd7f9a8e9d8", + "sha256:b593cc51aed887e93b78c2f94dfae9008be2b23d17afd3b1f1d3eb3913b58f26", + "sha256:bd82f9838519136b7083dd1e3149ee80344521f3dc37f744f227505ff0883efb", + "sha256:be69f7fd46b5c6467fe5e2fd4cff3816b0c03048eed8a4becb9a73e6000960e7", + "sha256:bec353f022011e6e5cd28ccb8700fbd2a33918197af0d4e0abb3c3f4845cc864", + "sha256:c17c5efee347a40a6f4c1eec59e3d7d1e22f7613a97f8b8a07733ef723483a04", + "sha256:cbd5894f23fdf5697499cf759523639838ac822bd1600e343fdce7313baa02ae", + "sha256:cda81d0e0ce0c13abfa46b24e10c1e85f9c6acb628f0a9a948f5779f9c2076a2", + "sha256:cf911e792ab0c431694c9bf2648afabfd92099103f2e31492893e078ddca5e1a", + "sha256:d07899506a5a8760448d9df036d528b55a554bf571714173635c79eef4a86e58", + "sha256:d41dd59a70decfce6595315367a2fea2af660d92a9d144acc6479030501014d7", + "sha256:d4f6de745fe6ce46a422d353ee10599013631d7d714a36d025f164b2d4e8c000", + "sha256:d57b98013b802621bbc8b12a46bfc9d36ac552ab51ca207f7ce167ad46adabeb", + "sha256:d8e5a437b9089df6242a718d9c31ab1742989e9400a0977af012ef483b63b4c2", + "sha256:dbee7f5ff11872b76505cbd87c814abc823e8757f11c69062eb3b25130a283da", + "sha256:ddc380ffaa90f204cc9ddcb779114b9ab6f015246d549de9d47871a97ef9f18a", + "sha256:de8adc12161bf282c60f12dc9233bb31632f71d446a010fe7469a69b8153427f", + "sha256:e06f6d270112f5db001f1cba5a97e1a48aee3d3dbdcbea3ec027c230462dbf9b", + "sha256:e1507fc5769aa109dda4de3a15f822a0f6a03e18d627bd0ba3ddbb253cf70e07", + "sha256:e48dde8ca83d11daa00900cf6a5d281a1297aef9b7bfa73801af6e8822be5019", + "sha256:e706f302c6a3ae0d74edd0d6ace46aee1ae07c563b436ccf5ff04db2b3571e60", + "sha256:eb2e51d01b9c6d6954a3e055c57a80d4685b4fc82719db5519fc153566bcd6bb", + "sha256:ed52461ae5a9ea4c400d38e2649c74a413f1a6d8fb8308b66f1fbd122514732f", + "sha256:ef3925daaa93eed20401012e219f569ff0c039ed5bf4ce2d3737b4f75d441622", + "sha256:f31314fd2e2f3dc3e519e6f93669462ce7953df2def1c344aa8f5345976d0eb2", + "sha256:f3a70f341c4c111bad910d2df69c78577a98af140319a996af24c9385939335d", + "sha256:f8b77779174b1b40aa70827692571ab457061897846255ad7d5d559e2edb1932", + "sha256:fe2a68be734e8e88af23385c68d6467e15818b6b1df1cbfebf7bff577226c957" ], "markers": "python_version >= '3.8'", - "version": "==3.6.1" + "version": "==3.6.2" }, "redis": { "extras": [ "hiredis" ], "hashes": [ - "sha256:3f82cc80d350e93042c8e6e7a5d0596e4dd68715babffba79492733e1f367037", - "sha256:4caa8e1fcb6f3c0ef28dba99535101d80934b7d4cd541bbb47f4a3826ee472d1" + "sha256:4973bae7444c0fbed64a06b87446f79361cb7e4ec1538c022d696ed7a5015580", + "sha256:5da9b8fe9e1254293756c16c008e8620b3d15fcc6dde6babde9541850e72a32d" ], "markers": "python_version >= '3.7'", - "version": "==5.0.2" + "version": "==5.0.3" }, "requests": { "hashes": [ @@ -1899,11 +1891,11 @@ }, "setuptools": { "hashes": [ - "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56", - "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8" + "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e", + "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c" ], "markers": "python_version >= '3.8'", - "version": "==69.1.1" + "version": "==69.2.0" }, "six": { "hashes": [ @@ -2019,11 +2011,11 @@ }, "torf": { "hashes": [ - "sha256:e84ec73cd81cb2d56e36b9f4604806f16ee0c70aeafadfe2f31ca3266ca03a87", - "sha256:f846ef58b6b00bc3b6a5c25bf225c59149ee871738d87a20a8953f620b77671c" + "sha256:261e861683f4d15cd8ae43c9db4828ef9da08f8efdf3b2878f82c0385d1e3707", + "sha256:9892b68c414c48a6dc0d5127cc6c76088fef3f9ffa91851cb0b4334d72e9565f" ], "markers": "python_version >= '3.7'", - "version": "==4.2.4" + "version": "==4.2.5" }, "twisted": { "hashes": [ @@ -2062,11 +2054,11 @@ "standard" ], "hashes": [ - "sha256:3d9a267296243532db80c83a959a3400502165ade2c1338dea4e67915fd4745a", - "sha256:5c89da2f3895767472a35556e539fd59f7edbe9b1e9c0e1c99eebeadc61838e4" + "sha256:6623abbbe6176204a4226e67607b4d52cc60ff62cda0ff177613645cefa2ece1", + "sha256:cab4473b5d1eaeb5a0f6375ac4bc85007ffc75c3cc1768816d9e5d589857b067" ], "markers": "python_version >= '3.8'", - "version": "==0.27.1" + "version": "==0.28.0" }, "uvloop": { "hashes": [ diff --git a/api/main.py b/api/main.py index 46c8a3e..be68185 100644 --- a/api/main.py +++ b/api/main.py @@ -18,8 +18,12 @@ from fastapi.responses import FileResponse, StreamingResponse from fastapi.templating import Jinja2Templates from api import middleware +from api.scheduler import setup_scheduler from db import database, crud, schemas from db.config import settings +from mediafusion_scrapy.task import run_spider +from scrapers.tamilmv import run_tamilmv_scraper +from scrapers.tamil_blasters import run_tamil_blasters_scraper from streaming_providers.routes import router as streaming_provider_router from utils import crypto, torrent, poster, const, wrappers from utils.parser import generate_manifest @@ -64,6 +68,8 @@ async def start_scheduler(): scheduler.add_job( crud.delete_search_history, CronTrigger(day="*/1"), name="delete_search_history" ) + setup_scheduler(scheduler) + scheduler.start() app.state.scheduler = scheduler @@ -96,7 +102,7 @@ async def get_home(request: Request): @app.get("/health", tags=["health"]) -@wrappers.exclude +@wrappers.exclude_rate_limit async def health(request: Request): return {"status": "healthy"} @@ -378,6 +384,7 @@ async def get_streams( user_data, secret_str, request.app.state.redis, video_id, season, episode ) else: + response.headers.update(const.NO_CACHE_HEADERS) fetched_streams = await crud.get_tv_streams(video_id) return {"streams": fetched_streams} @@ -391,7 +398,7 @@ async def encrypt_user_data(user_data: schemas.UserData): @app.get("/poster/{catalog_type}/{mediafusion_id}.jpg", tags=["poster"]) -@wrappers.exclude +@wrappers.exclude_rate_limit async def get_poster( catalog_type: Literal["movie", "series", "tv"], mediafusion_id: str, @@ -437,6 +444,38 @@ async def get_poster( raise HTTPException(status_code=404, detail="Failed to create poster.") +@app.get("/scraper", tags=["scraper"]) +async def get_scraper(request: Request): + return TEMPLATES.TemplateResponse( + "html/scraper.html", + { + "request": request, + "authentication_required": settings.api_password is not None, + }, + ) + + +@app.post("/scraper/run", tags=["scraper"]) +async def run_scraper_task(task: schemas.ScraperTask): + if settings.api_password and task.api_password != settings.api_password: + raise HTTPException(status_code=401, detail="Invalid API password.") + + if task.scraper_type == "tamilmv": + run_tamilmv_scraper.send(task.pages, task.start_page) + elif task.scraper_type == "tamilblasters": + run_tamil_blasters_scraper.send(task.pages, task.start_page) + elif task.scraper_type == "scrapy": + if not task.spider_name: + raise HTTPException( + status_code=400, detail="Spider name is required for scrapy tasks." + ) + run_spider.send(task.spider_name) + else: + raise HTTPException(status_code=400, detail="Invalid scraper type.") + + return {"status": f"Scraping {task.scraper_type} task has been scheduled."} + + app.include_router( streaming_provider_router, prefix="/streaming_provider", tags=["streaming_provider"] ) diff --git a/api/middleware.py b/api/middleware.py index af35eb9..22cacbc 100644 --- a/api/middleware.py +++ b/api/middleware.py @@ -102,7 +102,7 @@ class RateLimitMiddleware(BaseHTTPMiddleware): if not endpoint: return await call_next(request) - is_exclude = getattr(endpoint, "exclude", False) + is_exclude = getattr(endpoint, "exclude_rate_limit", False) if is_exclude: return await call_next(request) diff --git a/api/scheduler.py b/api/scheduler.py index 3381c59..399bd4f 100644 --- a/api/scheduler.py +++ b/api/scheduler.py @@ -1,73 +1,66 @@ -import asyncio -import logging - from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.cron import CronTrigger -from scrapy.crawler import CrawlerProcess -from scrapy.utils.project import get_project_settings from db.config import settings from scrapers import tamil_blasters, tamilmv - -# Configure logging -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" -) +from mediafusion_scrapy.task import run_spider +from utils.validation_helper import validate_tv_streams_in_db -def run_formula_tgx_spider(spider_name: str): +def setup_scheduler(scheduler: AsyncIOScheduler): """ - Function to start the formula_tgx spider. + Set up the scheduler with the required jobs. """ - process = CrawlerProcess(get_project_settings()) - process.crawl(spider_name) - process.start() + # Setup tamil blasters scraper + scheduler.add_job( + tamil_blasters.run_tamil_blasters_scraper.send, + CronTrigger.from_crontab(settings.tamil_blasters_scheduler_crontab), + name="tamil_blasters", + ) -scheduler = AsyncIOScheduler() + # Setup tamilmv scraper + scheduler.add_job( + tamilmv.run_tamilmv_scraper.send, + CronTrigger.from_crontab(settings.tamilmv_scheduler_crontab), + name="tamilmv", + ) -# Setup tamil blasters scraper -scheduler.add_job( - tamil_blasters.run_schedule_scrape, - CronTrigger.from_crontab(settings.tamil_blasters_scheduler_crontab), - name="tamil_blasters", -) + # Setup formula_tgx scraper + scheduler.add_job( + run_spider.send, + CronTrigger.from_crontab(settings.formula_tgx_scheduler_crontab), + name="formula_tgx", + kwargs={"spider_name": "formula_tgx"}, + ) -# Setup tamilmv scraper -scheduler.add_job( - tamilmv.run_schedule_scrape, - CronTrigger.from_crontab(settings.tamilmv_scheduler_crontab), - name="tamilmv", -) + # Setup mhdtvworld scraper + scheduler.add_job( + run_spider.send, + CronTrigger.from_crontab(settings.mhdtvworld_scheduler_crontab), + name="mhdtvworld", + kwargs={"spider_name": "mhdtvworld"}, + ) -# Setup formula_tgx scraper -scheduler.add_job( - run_formula_tgx_spider, - CronTrigger.from_crontab(settings.formula_tgx_scheduler_crontab), - name="formula_tgx", - kwargs={"spider_name": "formula_tgx"}, -) + # Setup mhdtvsports scraper + scheduler.add_job( + run_spider.send, + CronTrigger.from_crontab(settings.mhdtvsports_scheduler_crontab), + name="mhdtvsports", + kwargs={"spider_name": "mhdtvsports"}, + ) -# Setup mhdtvworld scraper -scheduler.add_job( - run_formula_tgx_spider, - CronTrigger.from_crontab(settings.mhdtvworld_scheduler_crontab), - name="mhdtvworld", - kwargs={"spider_name": "mhdtvworld"}, -) + # Setup tamilultra scraper + scheduler.add_job( + run_spider.send, + CronTrigger.from_crontab(settings.tamilultra_scheduler_crontab), + name="tamilultra", + kwargs={"spider_name": "tamilultra"}, + ) -# Setup mhdtvsports scraper -scheduler.add_job( - run_formula_tgx_spider, - CronTrigger.from_crontab(settings.mhdtvsports_scheduler_crontab), - name="mhdtvsports", - kwargs={"spider_name": "mhdtvsports"}, -) - -# Start the scheduler -scheduler.start() - -try: - asyncio.get_event_loop().run_forever() -except Exception as e: - logging.error(f"Error occurred: {e}") + # Schedule validate_tv_streams_in_db + scheduler.add_job( + validate_tv_streams_in_db.send, + CronTrigger.from_crontab(settings.validate_tv_streams_in_db_crontab), + name="validate_tv_streams_in_db", + ) diff --git a/api/task.py b/api/task.py index 15af892..71bce4e 100644 --- a/api/task.py +++ b/api/task.py @@ -1,10 +1,14 @@ -# import background actors import asyncio from db import database -from scrapers.helpers import update_torrent_movie_streams_metadata # noqa: F401 -from scrapers.prowlarr import parse_and_store_movie_stream_data # noqa: F401 + +# import background actors +from scrapers import helpers # noqa: F401 +from scrapers import prowlarr # noqa: F401 +from mediafusion_scrapy import task # noqa: F401 from utils import torrent +from utils import validation_helper # noqa: F401 +from scrapers import tamil_blasters, tamilmv # noqa: F401 async def async_setup(): diff --git a/db/config.py b/db/config.py index ad6f4be..72982b7 100644 --- a/db/config.py +++ b/db/config.py @@ -29,6 +29,8 @@ class Settings(BaseSettings): formula_tgx_scheduler_crontab: str = "0 */12 * * *" mhdtvworld_scheduler_crontab: str = "0 0 * * 5" mhdtvsports_scheduler_crontab: str = "0 10 * * *" + tamilultra_scheduler_crontab: str = "0 8 * * *" + validate_tv_streams_in_db_crontab: str = "0 */2 * * *" class Config: env_file = ".env" diff --git a/db/crud.py b/db/crud.py index c0d57d6..bfb2beb 100644 --- a/db/crud.py +++ b/db/crud.py @@ -6,6 +6,7 @@ from uuid import uuid4 from beanie import WriteRules from beanie.operators import In, Set +from pydantic import ValidationError from pymongo.errors import DuplicateKeyError from redis.asyncio import Redis @@ -78,7 +79,9 @@ async def get_meta_list( async def get_tv_meta_list( genre: Optional[str] = None, skip: int = 0, limit: int = 25 ) -> list[schemas.Meta]: - query = MediaFusionTVMetaData.find(fetch_links=True) + query = MediaFusionTVMetaData.find( + MediaFusionMovieMetaData.streams.is_working == True, fetch_links=True + ) if genre: query = query.find(In(MediaFusionTVMetaData.genres, [genre])) @@ -238,11 +241,21 @@ async def get_series_streams( async def get_tv_streams(video_id: str) -> list[Stream]: - tv_data = await get_tv_data_by_id(video_id, True) - if not tv_data: + tv_streams = await TVStreams.find( + {"meta_id": video_id, "is_working": True} + ).to_list() + if not tv_streams: return [] - return parse_tv_stream_data(tv_data) + return await parse_tv_stream_data(tv_streams) + + +async def get_tv_stream_by_id(stream_id: str) -> TVStreams | None: + try: + stream = await TVStreams.get(stream_id) + except ValidationError: + return None + return stream async def get_movie_meta(meta_id: str): diff --git a/db/models.py b/db/models.py index 9907340..2a1108c 100644 --- a/db/models.py +++ b/db/models.py @@ -63,6 +63,7 @@ class TVStreams(Document): created_at: datetime = Field(default_factory=datetime.now) meta_id: Optional[str] = None country: str | None = None + is_working: Optional[bool] = True class MediaFusionMetaData(Document): diff --git a/db/schemas.py b/db/schemas.py index b619e71..ea715d2 100644 --- a/db/schemas.py +++ b/db/schemas.py @@ -195,3 +195,11 @@ class TVMetaData(BaseModel): class TorrentStreamsList(BaseModel): streams: list[TorrentStreams] + + +class ScraperTask(BaseModel): + scraper_type: str + pages: int = 1 + start_page: int = 1 + spider_name: str = None + api_password: str = None diff --git a/deployment/docker-compose/docker-compose.yml b/deployment/docker-compose/docker-compose.yml index 277531d..b2e0d74 100644 --- a/deployment/docker-compose/docker-compose.yml +++ b/deployment/docker-compose/docker-compose.yml @@ -59,14 +59,6 @@ services: volumes: - prowlar-config:/config - scraper: - image: mhdzumair/mediafusion:v3.7.4 - command: ["pipenv", "run", "python", "-m", "api.scheduler"] - env_file: - - .env - depends_on: - - mongodb - volumes: mongo-data: prowlar-config: diff --git a/deployment/k8s/deployment.yaml b/deployment/k8s/deployment.yaml index 2c3ec3e..96c5fd8 100644 --- a/deployment/k8s/deployment.yaml +++ b/deployment/k8s/deployment.yaml @@ -129,56 +129,6 @@ spec: memory: "1Gi" cpu: "200m" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: scraper-deployment -spec: - replicas: 1 - selector: - matchLabels: - app: scraper - template: - metadata: - labels: - app: scraper - spec: - containers: - - name: scraper - image: mhdzumair/mediafusion:v3.7.4 - command: ["pipenv", "run", "python", "-m", "api.scheduler"] - env: - - name: MONGO_URI - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: MONGO_URI - - name: SECRET_KEY - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: SECRET_KEY - - name: PROWLARR_API_KEY - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: PROWLARR_API_KEY - - name: REDIS_URL - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: REDIS_URL - - name: db_max_connections - value: "30" - resources: - requests: - memory: "500Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "200m" - --- apiVersion: apps/v1 diff --git a/deployment/k8s/local-deployment.yaml b/deployment/k8s/local-deployment.yaml index 36a27c8..3fd5042 100644 --- a/deployment/k8s/local-deployment.yaml +++ b/deployment/k8s/local-deployment.yaml @@ -147,57 +147,6 @@ spec: --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: scraper-deployment -spec: - replicas: 1 - selector: - matchLabels: - app: scraper - template: - metadata: - labels: - app: scraper - spec: - containers: - - name: scraper - image: mhdzumair/mediafusion:beta - command: ["pipenv", "run", "python", "-m", "api.scheduler"] - env: - - name: MONGO_URI - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: MONGO_URI - - name: SECRET_KEY - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: SECRET_KEY - - name: PROWLARR_API_KEY - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: PROWLARR_API_KEY - - name: REDIS_URL - valueFrom: - secretKeyRef: - name: mediafusion-secrets - key: REDIS_URL - - name: db_max_connections - value: "30" - resources: - requests: - memory: "500Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "200m" - ---- - apiVersion: apps/v1 kind: Deployment metadata: diff --git a/mediafusion_scrapy/spiders/live_tv.py b/mediafusion_scrapy/spiders/live_tv.py index 18cbe92..58bc5c2 100644 --- a/mediafusion_scrapy/spiders/live_tv.py +++ b/mediafusion_scrapy/spiders/live_tv.py @@ -50,9 +50,6 @@ class LiveTVSpider(scrapy.Spider): ) def parse_categories(self, response): - # Process the current page immediately - yield from self.parse_page(response) - # Extract the total number of pages from the pagination text pagination_text = response.css("div.pagination span::text").get() total_pages = int(pagination_text.split(" ")[-1]) if pagination_text else 1 @@ -64,13 +61,16 @@ class LiveTVSpider(scrapy.Spider): else: base_url = base_url.rstrip("/") - # Since we've already processed the first page, start from the second page - page_urls = [f"{base_url}/page/{page}/" for page in range(2, total_pages + 1)] + # Generate URLs for all pages in reverse order, excluding the first page since it's already processed + page_urls = [f"{base_url}/page/{page}/" for page in range(total_pages, 1, -1)] - # Iterate over each subsequent page URL to scrape channels + # Iterate over each subsequent page URL in reverse order to scrape channels for page_url in page_urls: yield scrapy.Request(page_url, callback=self.parse_page) + # Process the first page last + yield from self.parse_page(response) + def parse_page(self, response): channel_elements = response.css("article.item.movies") source_name = response.css(".logo a img::attr(alt)").get(default="MHDTVWORLD") @@ -217,13 +217,17 @@ class LiveTVSpider(scrapy.Spider): }, ) - def get_behavior_hints(self, response): - """Generates behavior hints for requests.""" + def extract_m3u8_urls(self, response): + """Extracts M3U8 URLs using direct and fallback regex patterns.""" + m3u8_urls = self.direct_pattern.findall(response.text) + if not m3u8_urls: + m3u8_urls = self.fallback_pattern.findall(response.text) + user_agent = response.request.headers.get("User-Agent").decode() parsed_url = urlparse(response.url) referer = f"{parsed_url.scheme}://{parsed_url.netloc}" - return { + behavior_hints = { "notWebReady": True, "proxyHeaders": { "request": { @@ -233,21 +237,15 @@ class LiveTVSpider(scrapy.Spider): }, } - def extract_m3u8_urls(self, response): - """Extracts M3U8 URLs using direct and fallback regex patterns.""" - m3u8_urls = self.direct_pattern.findall(response.text) - if not m3u8_urls: - m3u8_urls = self.fallback_pattern.findall(response.text) - return m3u8_urls + return m3u8_urls, behavior_hints def request_and_extract_video_url(self, response): channel_data = response.meta.get("channel_data") stream_title = response.meta.get("stream_title") country_name = response.meta.get("country_name") - behavior_hints = self.get_behavior_hints(response) # Extract M3U8 URLs - m3u8_urls = self.extract_m3u8_urls(response) + m3u8_urls, behavior_hints = self.extract_m3u8_urls(response) if not m3u8_urls: self.logger.error( "No M3U8 URLs found for channel url: %s, stream title: %s", @@ -281,6 +279,9 @@ class LiveTVSpider(scrapy.Spider): if response.status == 200 and content_type in self.m3u8_valid_content_types: # Content type is valid, proceed with adding the stream + if response.meta.get("redirect_times", 0) > 0: + meta["behavior_hints"]["is_redirect"] = True + stream_info = { "name": f"{meta['stream_title']} - {meta['index']}" if meta["index"] diff --git a/mediafusion_scrapy/spiders/tamilultra.py b/mediafusion_scrapy/spiders/tamilultra.py new file mode 100644 index 0000000..692a3a7 --- /dev/null +++ b/mediafusion_scrapy/spiders/tamilultra.py @@ -0,0 +1,74 @@ +from urllib.parse import urljoin, urlparse + +import scrapy + +from mediafusion_scrapy.spiders.live_tv import LiveTVSpider + + +class TamilUltraSpider(LiveTVSpider): + name = "tamilultra" + start_urls = ["https://tamilultra.team/"] + + def extract_player_api_base(self, response): + """Extracts the admin-ajax URL for POST requests.""" + # Directly extract the URL used for admin-ajax POST requests + admin_ajax_url = response.xpath( + "//script[contains(text(), 'player_api')]/text()" + ).re_first(r'"url":"([^"]+)"') + if admin_ajax_url: + # Correctly format and return the full URL + admin_ajax_full_url = urljoin( + response.url, admin_ajax_url.replace("\\/", "/") + ) + return admin_ajax_full_url + else: + self.logger.error("Admin AJAX URL not found for TamilUltra.") + return None + + def process_player_option(self, element, channel_data, player_api_post_url): + """Processes each player option element to send a POST request.""" + stream_title, country_name = self.extract_stream_details(element) + data_post, data_nume, data_type = ( + element.attrib.get("data-post"), + element.attrib.get("data-nume"), + element.attrib.get("data-type"), + ) + + if all([data_post, data_nume, data_type]): + form_data = { + "action": "doo_player_ajax", + "post": data_post, + "nume": data_nume, + "type": data_type, + } + yield scrapy.FormRequest( + url=player_api_post_url, + formdata=form_data, + callback=self.parse_api_response, + meta={ + "channel_data": channel_data, + "stream_title": stream_title, + "country_name": country_name, + }, + ) + + def extract_m3u8_urls(self, response): + """Extracts M3U8 URLs using direct and fallback regex patterns.""" + query_string = urlparse(response.url).query + m3u8_urls = [urljoin(response.url, query_string)] + + user_agent = response.request.headers.get("User-Agent").decode() + parsed_url = urlparse(response.url) + referer = f"{parsed_url.scheme}://{parsed_url.netloc}" + + behavior_hints = { + "notWebReady": True, + "proxyHeaders": { + "request": { + "User-Agent": user_agent, + "Referer": referer, + } + }, + } + + return m3u8_urls, behavior_hints diff --git a/mediafusion_scrapy/task.py b/mediafusion_scrapy/task.py new file mode 100644 index 0000000..ac3d787 --- /dev/null +++ b/mediafusion_scrapy/task.py @@ -0,0 +1,24 @@ +from multiprocessing import Process + +import dramatiq +from scrapy.crawler import CrawlerProcess +from scrapy.utils.project import get_project_settings + + +def run_spider_in_process(spider_name, *args, **kwargs): + """ + Function to start a scrapy spider in a new process. + """ + process = CrawlerProcess(get_project_settings()) + process.crawl(spider_name, *args, **kwargs) + process.start() + + +@dramatiq.actor(priority=5, time_limit=60 * 60 * 1000) +def run_spider(spider_name: str, *args, **kwargs): + """ + Wrapper function to run the spider in a separate process. + """ + p = Process(target=run_spider_in_process, args=(spider_name, *args), kwargs=kwargs) + p.start() + p.join() diff --git a/resources/exceptions/source_not_live.mp4 b/resources/exceptions/source_not_live.mp4 new file mode 100644 index 0000000..ab38a54 Binary files /dev/null and b/resources/exceptions/source_not_live.mp4 differ diff --git a/resources/html/scraper.html b/resources/html/scraper.html new file mode 100644 index 0000000..b181d23 --- /dev/null +++ b/resources/html/scraper.html @@ -0,0 +1,72 @@ + + + + + + + + Scraper Controls - MediaFusion Stremio Addon + + + + + + + + +
+
+
+ +

Scraper Controls - Manage Scraping Tasks

+ + +
+

Scraper Selection

+
+
+ + +
+ + +
+
+ + + {% if authentication_required %} +
+

API Security Configuration

+
+
+ +
+ + +
+
+
+ {% endif %} + +
+ +
+
+
+
+ + + + + + + + + + diff --git a/resources/js/scraperControl.js b/resources/js/scraperControl.js new file mode 100644 index 0000000..1b9e5b2 --- /dev/null +++ b/resources/js/scraperControl.js @@ -0,0 +1,95 @@ +// ---- Helper Functions ---- + +function showNotification(message, type = 'info') { + toastr.options = { + closeButton: true, + newestOnTop: true, + progressBar: true, + positionClass: "toast-top-center", + preventDuplicates: true, + onclick: null, + showDuration: "300", + hideDuration: "1000", + timeOut: "5000", + extendedTimeOut: "1000", + showEasing: "swing", + hideEasing: "linear", + showMethod: "fadeIn", + hideMethod: "fadeOut", + }; + + toastr[type](message); +} + + +// Function to update form fields based on scraper selection +function updateFormFields() { + const scraperType = document.getElementById('scraperSelect').value; + let parametersHtml = ''; + const scraperParametersDiv = document.getElementById('scraperParameters'); + + if (scraperType === 'tamilmv' || scraperType === 'tamilblasters') { + parametersHtml += ` +
+ + +
+
+ + +
+ `; + } else if (scraperType === 'scrapy') { + parametersHtml += ` +
+ + +
+ `; + } + + // Update the form fields based on the selection + scraperParametersDiv.innerHTML = parametersHtml; +} + +// Function to submit the form and display the response using Toastr +async function submitScraperForm() { + const apiPassword = document.getElementById('api_password') ? document.getElementById('api_password').value : ''; + const scraperType = document.getElementById('scraperSelect').value; + let payload = { + scraper_type: scraperType, + api_password: apiPassword, + pages: document.getElementById('pages') ? document.getElementById('pages').value : 1, + start_page: document.getElementById('startPage') ? document.getElementById('startPage').value : 1, + spider_name: document.getElementById('spiderName') ? document.getElementById('spiderName').value : '' + }; + + try { + const response = await fetch("/scraper/run", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload) + }); + + const data = await response.json(); + if (response.ok) { + showNotification(data.status, 'success'); + } else { + showNotification(data.detail, 'error'); + } + } catch (error) { + console.error('Error submitting scraper form:', error); + showNotification('Error submitting scraper form. Please check the console for more details.', 'error'); + } +} + + +// Initial update for form fields on page load +document.addEventListener('DOMContentLoaded', updateFormFields); diff --git a/scrapers/helpers.py b/scrapers/helpers.py index ea33d93..f797423 100644 --- a/scrapers/helpers.py +++ b/scrapers/helpers.py @@ -129,7 +129,7 @@ def get_scraper_config(site_name: str, get_key: str) -> dict: return config.get(site_name, {}).get(get_key, {}) -@dramatiq.actor(time_limit=30 * 60 * 1000) +@dramatiq.actor(time_limit=30 * 60 * 1000, priority=10) async def update_torrent_movie_streams_metadata(info_hashes: list[str]): """Update torrent streams metadata.""" if not info_hashes: @@ -153,7 +153,7 @@ async def update_torrent_movie_streams_metadata(info_hashes: list[str]): logging.info(f"Updated {torrent_stream.id} metadata") -@dramatiq.actor(time_limit=30 * 60 * 1000) +@dramatiq.actor(time_limit=30 * 60 * 1000, priority=10) async def update_torrent_series_streams_metadata(info_hashes: list[str]): """Update torrent streams metadata.""" if not info_hashes: diff --git a/scrapers/prowlarr.py b/scrapers/prowlarr.py index 2c8ef29..380c887 100644 --- a/scrapers/prowlarr.py +++ b/scrapers/prowlarr.py @@ -141,6 +141,7 @@ async def scrap_movies_streams_from_prowlarr( min_backoff=2 * 60 * 1000, # 2 minutes max_backoff=60 * 60 * 1000, # 60 minutes retry_when=should_retry_prowlarr_scrap, + priority=100, ) async def background_movie_title_search(video_id: str, title: str, year: str): url = f"{settings.prowlarr_url}/api/v1/search" @@ -189,6 +190,7 @@ async def scrap_series_streams_from_prowlarr( min_backoff=2 * 60 * 1000, # 2 minutes max_backoff=60 * 60 * 1000, # 60 minutes retry_when=should_retry_prowlarr_scrap, + priority=100, ) async def background_series_title_search( video_id: str, title: str, season: int, episode: int diff --git a/scrapers/tamil_blasters.py b/scrapers/tamil_blasters.py index a09444e..6899452 100755 --- a/scrapers/tamil_blasters.py +++ b/scrapers/tamil_blasters.py @@ -6,7 +6,9 @@ import logging import math import random import re +from multiprocessing import Process +import dramatiq from bs4 import BeautifulSoup from dateutil.parser import parse as dateparser from playwright.async_api import async_playwright @@ -221,7 +223,6 @@ async def run_scraper( search_keyword: str = None, scrap_with_playwright: bool = None, ): - await database.init() if search_keyword: await scrap_search_keyword(search_keyword) return @@ -242,19 +243,33 @@ async def run_scraper( async def run_schedule_scrape( - pages: int = 1, - start_page: int = 1, - scrap_with_playwright: bool = None, + pages: int = 1, start_page: int = 1, scrap_with_playwright: bool = False ): - for language in TAMIL_BLASTER_CATALOGS: - for video_type in TAMIL_BLASTER_CATALOGS[language]: - await run_scraper( - language, - video_type, - pages=pages, - start_page=start_page, - scrap_with_playwright=scrap_with_playwright, - ) + await database.init() + async with asyncio.TaskGroup() as tg: + for language in TAMIL_BLASTER_CATALOGS: + for video_type in TAMIL_BLASTER_CATALOGS[language]: + tg.create_task( + run_scraper( + language, + video_type, + pages=pages, + start_page=start_page, + scrap_with_playwright=scrap_with_playwright, + ) + ) + + +def run_schedule_scrape_sync(pages, start_page, scrap_with_playwright): + asyncio.run(run_schedule_scrape(pages, start_page, scrap_with_playwright)) + + +@dramatiq.actor(priority=5, time_limit=60 * 60 * 1000) +def run_tamil_blasters_scraper(pages: int = 1, start_page: int = 1): + # Use a separate process to run the scraper + process = Process(target=run_schedule_scrape_sync, args=(pages, start_page, False)) + process.start() + process.join() if __name__ == "__main__": diff --git a/scrapers/tamilmv.py b/scrapers/tamilmv.py index ac36994..98aafd3 100644 --- a/scrapers/tamilmv.py +++ b/scrapers/tamilmv.py @@ -6,7 +6,9 @@ import logging import math import random import re +from multiprocessing import Process +import dramatiq from bs4 import BeautifulSoup from dateutil.parser import parse as dateparser from playwright.async_api import async_playwright @@ -196,7 +198,6 @@ async def run_scraper( search_keyword: str = None, scrap_with_playwright: bool = None, ): - await database.init() if search_keyword: await scrap_search_keyword(search_keyword) return @@ -228,15 +229,31 @@ async def run_schedule_scrape( start_page: int = 1, scrap_with_playwright: bool = None, ): - for language in TAMIL_MV_CATALOGS: - for video_type in TAMIL_MV_CATALOGS[language]: - await run_scraper( - language, - video_type, - pages=pages, - start_page=start_page, - scrap_with_playwright=scrap_with_playwright, - ) + await database.init() + async with asyncio.TaskGroup() as tg: + for language in TAMIL_MV_CATALOGS: + for video_type in TAMIL_MV_CATALOGS[language]: + tg.create_task( + run_scraper( + language, + video_type, + pages=pages, + start_page=start_page, + scrap_with_playwright=scrap_with_playwright, + ) + ) + + +def run_schedule_scrape_sync(pages, start_page, scrap_with_playwright): + asyncio.run(run_schedule_scrape(pages, start_page, scrap_with_playwright)) + + +@dramatiq.actor(priority=5, time_limit=60 * 60 * 1000) +def run_tamilmv_scraper(pages: int = 1, start_page: int = 1): + # Use a separate process to run the scraper + process = Process(target=run_schedule_scrape_sync, args=(pages, start_page, False)) + process.start() + process.join() if __name__ == "__main__": diff --git a/streaming_providers/pikpak/utils.py b/streaming_providers/pikpak/utils.py index 482d762..3eb8cb3 100644 --- a/streaming_providers/pikpak/utils.py +++ b/streaming_providers/pikpak/utils.py @@ -177,10 +177,10 @@ async def retrieve_or_download_file( pikpak, info_hash, stream.torrent_name, max_retries, retry_interval ) selected_file = await find_file_in_folder_tree(pikpak, folder_id, filename) - if selected_file is None: - raise ProviderException( - "No matching file available for this torrent", "no_matching_file.mp4" - ) + if selected_file is None: + raise ProviderException( + "No matching file available for this torrent", "no_matching_file.mp4" + ) return selected_file diff --git a/streaming_providers/routes.py b/streaming_providers/routes.py index 43ca694..d240bfb 100644 --- a/streaming_providers/routes.py +++ b/streaming_providers/routes.py @@ -30,7 +30,7 @@ router = APIRouter() @router.get("/{secret_str}/stream", tags=["streaming_provider"]) -@wrappers.exclude +@wrappers.exclude_rate_limit @wrappers.auth_required async def streaming_provider_endpoint( secret_str: str, diff --git a/utils/network.py b/utils/network.py index d1c4432..ca1d81e 100644 --- a/utils/network.py +++ b/utils/network.py @@ -110,3 +110,15 @@ async def batch_process_with_circuit_breaker( await asyncio.sleep(rate_limit_delay) # Always respect the rate limit return results + + +async def get_redirector_url(url: str, headers: dict) -> str | None: + """ + Get the final URL after following all redirects. + """ + try: + async with httpx.AsyncClient() as client: + response = await client.head(url, headers=headers, follow_redirects=True) + return str(response.url) + except httpx.HTTPError as e: + return diff --git a/utils/parser.py b/utils/parser.py index 7c2f5fc..0cd4b71 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -6,7 +6,7 @@ import requests from imdb import Cinemagoer from db.config import settings -from db.models import TorrentStreams, MediaFusionTVMetaData +from db.models import TorrentStreams, TVStreams from db.schemas import Stream, UserData from streaming_providers.alldebrid.utils import ( update_ad_cache_status, @@ -45,6 +45,7 @@ from streaming_providers.torbox.utils import ( fetch_downloaded_info_hashes_from_torbox, ) from utils import const +from utils.network import get_redirector_url ia = Cinemagoer() ADULT_CONTENT_KEYWORDS = re.compile( @@ -311,27 +312,37 @@ def get_imdb_data(video_id: str) -> tuple[str, str]: return movie.get("title"), movie.get("year") -def parse_tv_stream_data(tv_data: MediaFusionTVMetaData) -> list[Stream]: +async def parse_tv_stream_data(tv_streams: list[TVStreams]) -> list[Stream]: stream_list = [] - for stream in tv_data.streams: + for stream in tv_streams: if stream.behaviorHints.get("is_redirect", False): - response = requests.get( + stream_link = await get_redirector_url( stream.url, - headers=stream.behaviorHints["proxyHeaders"]["request"], - allow_redirects=False, + stream.behaviorHints.get("proxyHeaders", {}).get("request", {}), ) - if response.status_code == 302: - stream.url = response.headers["Location"] + if stream_link is None: + continue + stream.url = stream_link stream_list.append( Stream( name="MediaFusion", - description=f"{stream.name}, {tv_data.tv_language}, {stream.source}", + description=f"šŸ“ŗ {stream.name}\nšŸŒ {stream.country}\nšŸ”— {stream.source}", url=stream.url, ytId=stream.ytId, behaviorHints=stream.behaviorHints, ) ) + if not stream_list: + stream_list.append( + Stream( + name="MediaFusion", + description="No streams live at the moment.", + url=f"{settings.host_url}/static/exceptions/source_not_live.mp4", + behaviorHints={"notWebReady": True}, + ) + ) + return stream_list diff --git a/utils/validation_helper.py b/utils/validation_helper.py index b6c2021..4eb0365 100644 --- a/utils/validation_helper.py +++ b/utils/validation_helper.py @@ -3,6 +3,7 @@ import logging from urllib.parse import urlparse import aiohttp +import dramatiq from aiohttp import ClientError from db import schemas @@ -38,7 +39,7 @@ async def validate_m3u8_url(url: str, behaviour_hint: dict) -> bool: headers = behaviour_hint.get("proxyHeaders", {}).get("request", {}) async with aiohttp.ClientSession() as session: try: - async with session.get( + async with session.head( url, allow_redirects=True, headers=headers, @@ -152,3 +153,22 @@ def is_video_file(filename: str) -> bool: ".yuv", ) ) + + +@dramatiq.actor(time_limit=30 * 60 * 1000, priority=5) # time limit is 30 minutes +async def validate_tv_streams_in_db(): + """Validate TV streams in the database.""" + from db.models import TVStreams + + async def validate_and_update_tv_stream(stream): + is_valid = await validate_m3u8_url(stream.url, stream.behaviorHints) + stream.is_working = is_valid + await stream.save() + logging.info(f"Stream: {stream.name}, Status: {is_valid}") + + tv_streams = await TVStreams.all().to_list() + tasks = [ + asyncio.create_task(validate_and_update_tv_stream(stream)) + for stream in tv_streams + ] + await asyncio.gather(*tasks) diff --git a/utils/wrappers.py b/utils/wrappers.py index c19b650..c9d2570 100644 --- a/utils/wrappers.py +++ b/utils/wrappers.py @@ -23,12 +23,12 @@ def rate_limit(limit: int, window: int, scope: str = None): return decorator -def exclude(func): +def exclude_rate_limit(func): @wraps(func) async def wrapper(*args, **kwargs): return await func(*args, **kwargs) - wrapper.exclude = True + wrapper.exclude_rate_limit = True return wrapper