diff --git a/.gitignore b/.gitignore index be4ef3c..0f52d05 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,6 @@ dmypy.json *.service http-client.private.env.json + +localhost.pem +localhost-key.pem \ No newline at end of file diff --git a/Pipfile b/Pipfile index 3e9c166..cc59731 100644 --- a/Pipfile +++ b/Pipfile @@ -25,8 +25,11 @@ seedrcc = "*" parse-torrent-title = {editable = true, git = "https://github.com/mhdzumair/parse-torrent-title"} playwright = "==1.38.0" playwright-stealth = "*" +pillow = "*" +httpx = "*" [dev-packages] +pysocks = "*" [requires] python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock index 2db19ec..8c12d3a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "183ccb6cf1162b614d2096a00892cfebbeaacf4ebf80abf708242a8e938a31a3" + "sha256": "78844b6c17f6cdde79cec519db027b7ed84cdf96d0cf4998084c359b91385fe3" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "annotated-types": { "hashes": [ - "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802", - "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd" + "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43", + "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d" ], - "markers": "python_version >= '3.7'", - "version": "==0.5.0" + "markers": "python_version >= '3.8'", + "version": "==0.6.0" }, "anyio": { "hashes": [ @@ -38,15 +38,17 @@ "sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==3.10.4" }, "beanie": { "hashes": [ - "sha256:2b03cfe43ff383526d8ec67a88d8f0e60e251f2631af180c30fb4ac1fb9d900b", - "sha256:9721e3b0dc435e5a42b64e58d4f88bb9a447eb94c489de7ceb75854f45ec2a19" + "sha256:537e0c59c84644d78afaf7cf83821093c2a37f7a9c7f51e05191caf8b8142b92", + "sha256:6670849bffc1902a33e62ad5b5dc536a68ead6bafa2823e99a803d12a600b379" ], "index": "pypi", - "version": "==1.22.6" + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==1.23.0" }, "beautifulsoup4": { "hashes": [ @@ -54,6 +56,7 @@ "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a" ], "index": "pypi", + "markers": "python_full_version >= '3.6.0'", "version": "==4.12.2" }, "bencodepy": { @@ -68,7 +71,7 @@ "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2023.7.22" }, "charset-normalizer": { @@ -164,7 +167,7 @@ "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e", "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.3.0" }, "cinemagoer": { @@ -194,6 +197,7 @@ "sha256:8dcfae8c7460a2f84b4072e26f1c9f4101ca20c071649cb7c34e8b6a93d58984" ], "index": "pypi", + "markers": "python_version >= '3.8' and python_version < '4.0'", "version": "==2.4.2" }, "fastapi": { @@ -202,6 +206,7 @@ "sha256:75a11f6bfb8fc4d2bec0bd710c2d5f2829659c0e8c0afd5560fdda6ce25ec653" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==0.103.2" }, "greenlet": { @@ -282,6 +287,14 @@ "markers": "python_version >= '3.7'", "version": "==0.14.0" }, + "httpcore": { + "hashes": [ + "sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9", + "sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced" + ], + "markers": "python_version >= '3.8'", + "version": "==0.18.0" + }, "httptools": { "hashes": [ "sha256:03bfd2ae8a2d532952ac54445a2fb2504c804135ed28b53fefaf03d3a93eb1fd", @@ -322,6 +335,15 @@ ], "version": "==0.6.0" }, + "httpx": { + "hashes": [ + "sha256:181ea7f8ba3a82578be86ef4171554dd45fec26a02556a744db029a0a27b7100", + "sha256:47ecda285389cb32bb2691cc6e069e3ab0205956f681c5b2ad2325719751d875" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==0.25.0" + }, "idna": { "hashes": [ "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", @@ -336,6 +358,7 @@ "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==3.1.2" }, "lazy-model": { @@ -516,16 +539,74 @@ "sha256:c5eb400e27d722a3db03a9826656b6d13acf9b6c70c2fb4604f474eac9da5be4" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==3.3.1" }, "parse-torrent-title": { "editable": true, "git": "https://github.com/mhdzumair/parse-torrent-title", + "ref": "a1bb2b927669d5105622f381b856a24e822307d7" + }, + "pillow": { "hashes": [ - "sha256:556830ea268f808a3cee3fc6e9380ab3293b13a8172567a5a39b5d858a5387af", - "sha256:9852cfba2b746683733d77d2019a24f57d5e8c2c4db395f6ff03035ea628a4bb" + "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d", + "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de", + "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616", + "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839", + "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099", + "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a", + "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219", + "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106", + "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b", + "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412", + "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b", + "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7", + "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2", + "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7", + "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14", + "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f", + "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27", + "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57", + "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262", + "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28", + "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610", + "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172", + "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273", + "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e", + "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d", + "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818", + "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f", + "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9", + "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01", + "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7", + "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651", + "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312", + "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80", + "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666", + "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061", + "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b", + "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992", + "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593", + "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4", + "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db", + "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba", + "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd", + "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e", + "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212", + "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb", + "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2", + "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34", + "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256", + "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f", + "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2", + "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38", + "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996", + "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a", + "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793" ], - "ref": "0ccb3aa7523063375ca8a6e3677a493ddba31e26" + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==10.1.0" }, "playwright": { "hashes": [ @@ -538,6 +619,7 @@ "sha256:d0288c8932d7f14bc231e4a6761ecf76fff879d1601cfa3b6f6aefd544468911" ], "index": "pypi", + "markers": "python_version >= '3.8'", "version": "==1.38.0" }, "playwright-stealth": { @@ -546,6 +628,7 @@ "sha256:b504d951d00fac755c7d13665a29611d415180510bd7d23f14ebc89439ba2043" ], "index": "pypi", + "markers": "python_version >= '3' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.0.6" }, "pycryptodome": { @@ -584,6 +667,7 @@ "sha256:fc7a79590e2b5d08530175823a242de6790abc73638cc6dc9d2684e7be2f5e49" ], "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==3.19.0" }, "pydantic": { @@ -592,6 +676,7 @@ "sha256:bc3ddf669d234f4220e6e1c4d96b061abe0998185a8d7855c0126782b7abc8c1" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==2.4.2" }, "pydantic-core": { @@ -712,6 +797,7 @@ "sha256:ddd907b066622bd67603b75e2ff791875540dc485b7307c4fffc015719da8625" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==2.0.3" }, "pyee": { @@ -808,7 +894,7 @@ "sha256:fe48f50fb6348511a3268a893bfd4ab5f263f5ac220782449d03cd05964d1ae7", "sha256:fff7d17d30b2cd45afd654b3fc117755c5d84506ed25fda386494e4e0a3416e1" ], - "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==4.5.0" }, "pyparsing": { @@ -825,6 +911,7 @@ "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "python-dotenv": { @@ -903,6 +990,7 @@ "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==2.31.0" }, "requests-toolbelt": { @@ -919,6 +1007,7 @@ "sha256:e19b1086e304ce24e4fc706fc444523a9b873b9e9c7802a9c0d90bbac62ed417" ], "index": "pypi", + "markers": "python_version >= '3.0'", "version": "==1.0.1" }, "six": { @@ -947,50 +1036,58 @@ }, "sqlalchemy": { "hashes": [ - "sha256:014794b60d2021cc8ae0f91d4d0331fe92691ae5467a00841f7130fe877b678e", - "sha256:0268256a34806e5d1c8f7ee93277d7ea8cc8ae391f487213139018b6805aeaf6", - "sha256:05b971ab1ac2994a14c56b35eaaa91f86ba080e9ad481b20d99d77f381bb6258", - "sha256:141675dae56522126986fa4ca713739d00ed3a6f08f3c2eb92c39c6dfec463ce", - "sha256:1e7dc99b23e33c71d720c4ae37ebb095bebebbd31a24b7d99dfc4753d2803ede", - "sha256:2e617727fe4091cedb3e4409b39368f424934c7faa78171749f704b49b4bb4ce", - "sha256:3cf229704074bce31f7f47d12883afee3b0a02bb233a0ba45ddbfe542939cca4", - "sha256:3eb7c03fe1cd3255811cd4e74db1ab8dca22074d50cd8937edf4ef62d758cdf4", - "sha256:3f7d57a7e140efe69ce2d7b057c3f9a595f98d0bbdfc23fd055efdfbaa46e3a5", - "sha256:419b1276b55925b5ac9b4c7044e999f1787c69761a3c9756dec6e5c225ceca01", - "sha256:44ac5c89b6896f4740e7091f4a0ff2e62881da80c239dd9408f84f75a293dae9", - "sha256:4615623a490e46be85fbaa6335f35cf80e61df0783240afe7d4f544778c315a9", - "sha256:50a69067af86ec7f11a8e50ba85544657b1477aabf64fa447fd3736b5a0a4f67", - "sha256:513fd5b6513d37e985eb5b7ed89da5fd9e72354e3523980ef00d439bc549c9e9", - "sha256:6ff3dc2f60dbf82c9e599c2915db1526d65415be323464f84de8db3e361ba5b9", - "sha256:73c079e21d10ff2be54a4699f55865d4b275fd6c8bd5d90c5b1ef78ae0197301", - "sha256:7614f1eab4336df7dd6bee05bc974f2b02c38d3d0c78060c5faa4cd1ca2af3b8", - "sha256:785e2f2c1cb50d0a44e2cdeea5fd36b5bf2d79c481c10f3a88a8be4cfa2c4615", - "sha256:7ca38746eac23dd7c20bec9278d2058c7ad662b2f1576e4c3dbfcd7c00cc48fa", - "sha256:7f0c4ee579acfe6c994637527c386d1c22eb60bc1c1d36d940d8477e482095d4", - "sha256:87bf91ebf15258c4701d71dcdd9c4ba39521fb6a37379ea68088ce8cd869b446", - "sha256:89e274604abb1a7fd5c14867a412c9d49c08ccf6ce3e1e04fffc068b5b6499d4", - "sha256:8c323813963b2503e54d0944813cd479c10c636e3ee223bcbd7bd478bf53c178", - "sha256:a95aa0672e3065d43c8aa80080cdd5cc40fe92dc873749e6c1cf23914c4b83af", - "sha256:af520a730d523eab77d754f5cf44cc7dd7ad2d54907adeb3233177eeb22f271b", - "sha256:b19ae41ef26c01a987e49e37c77b9ad060c59f94d3b3efdfdbf4f3daaca7b5fe", - "sha256:b4eae01faee9f2b17f08885e3f047153ae0416648f8e8c8bd9bc677c5ce64be9", - "sha256:b69f1f754d92eb1cc6b50938359dead36b96a1dcf11a8670bff65fd9b21a4b09", - "sha256:b977bfce15afa53d9cf6a632482d7968477625f030d86a109f7bdfe8ce3c064a", - "sha256:bf8eebccc66829010f06fbd2b80095d7872991bfe8415098b9fe47deaaa58063", - "sha256:c111cd40910ffcb615b33605fc8f8e22146aeb7933d06569ac90f219818345ef", - "sha256:c2d494b6a2a2d05fb99f01b84cc9af9f5f93bf3e1e5dbdafe4bed0c2823584c1", - "sha256:c9cba4e7369de663611ce7460a34be48e999e0bbb1feb9130070f0685e9a6b66", - "sha256:cca720d05389ab1a5877ff05af96551e58ba65e8dc65582d849ac83ddde3e231", - "sha256:ccb99c3138c9bde118b51a289d90096a3791658da9aea1754667302ed6564f6e", - "sha256:d59cb9e20d79686aa473e0302e4a82882d7118744d30bb1dfb62d3c47141b3ec", - "sha256:e36339a68126ffb708dc6d1948161cea2a9e85d7d7b0c54f6999853d70d44430", - "sha256:ea7da25ee458d8f404b93eb073116156fd7d8c2a776d8311534851f28277b4ce", - "sha256:f9fefd6298433b6e9188252f3bff53b9ff0443c8fde27298b8a2b19f6617eeb9", - "sha256:fb87f763b5d04a82ae84ccff25554ffd903baafba6698e18ebaf32561f2fe4aa", - "sha256:fc6b15465fabccc94bf7e38777d665b6a4f95efd1725049d6184b3a39fd54880" + "sha256:0b0b3f2686c3f162123adba3cb8b626ed7e9b8433ab528e36ed270b4f70d1cdb", + "sha256:0c1fea8c0abcb070ffe15311853abfda4e55bf7dc1d4889497b3403629f3bf00", + "sha256:0e1ce8ebd2e040357dde01a3fb7d30d9b5736b3e54a94002641dfd0aa12ae6ce", + "sha256:129415f89744b05741c6f0b04a84525f37fbabe5dc3774f7edf100e7458c48cd", + "sha256:13790cb42f917c45c9c850b39b9941539ca8ee7917dacf099cc0b569f3d40da7", + "sha256:14cd3bcbb853379fef2cd01e7c64a5d6f1d005406d877ed9509afb7a05ff40a5", + "sha256:154a32f3c7b00de3d090bc60ec8006a78149e221f1182e3edcf0376016be9396", + "sha256:19c6986cf2fb4bc8e0e846f97f4135a8e753b57d2aaaa87c50f9acbe606bd1db", + "sha256:2096d6b018d242a2bcc9e451618166f860bb0304f590d205173d317b69986c95", + "sha256:2c9bac865ee06d27a1533471405ad240a6f5d83195eca481f9fc4a71d8b87df8", + "sha256:3076740335e4aaadd7deb3fe6dcb96b3015f1613bd190a4e1634e1b99b02ec86", + "sha256:3940677d341f2b685a999bffe7078697b5848a40b5f6952794ffcf3af150c301", + "sha256:3aa1472bf44f61dd27987cd051f1c893b7d3b17238bff8c23fceaef4f1133868", + "sha256:40b1206a0d923e73aa54f0a6bd61419a96b914f1cd19900b6c8226899d9742ad", + "sha256:4bb062784f37b2d75fd9b074c8ec360ad5df71f933f927e9e95c50eb8e05323c", + "sha256:4e869a8ff7ee7a833b74868a0887e8462445ec462432d8cbeff5e85f475186da", + "sha256:4f6ff392b27a743c1ad346d215655503cec64405d3b694228b3454878bf21590", + "sha256:505f503763a767556fa4deae5194b2be056b64ecca72ac65224381a0acab7ebe", + "sha256:53a766cb0b468223cafdf63e2d37f14a4757476157927b09300c8c5832d88560", + "sha256:5434cc601aa17570d79e5377f5fd45ff92f9379e2abed0be5e8c2fba8d353d2b", + "sha256:54bcceaf4eebef07dadfde424f5c26b491e4a64e61761dea9459103ecd6ccc95", + "sha256:55914d45a631b81a8a2cb1a54f03eea265cf1783241ac55396ec6d735be14883", + "sha256:564e9f9e4e6466273dbfab0e0a2e5fe819eec480c57b53a2cdee8e4fdae3ad5f", + "sha256:56a7e2bb639df9263bf6418231bc2a92a773f57886d371ddb7a869a24919face", + "sha256:58a3aba1bfb32ae7af68da3f277ed91d9f57620cf7ce651db96636790a78b736", + "sha256:625b72d77ac8ac23da3b1622e2da88c4aedaee14df47c8432bf8f6495e655de2", + "sha256:69fd9e41cf9368afa034e1c81f3570afb96f30fcd2eb1ef29cb4d9371c6eece2", + "sha256:6ac28bd6888fe3c81fbe97584eb0b96804bd7032d6100b9701255d9441373ec1", + "sha256:7c6c3e9350f9fb16de5b5e5fbf17b578811a52d71bb784cc5ff71acb7de2a7f9", + "sha256:7ee7ccf47aa503033b6afd57efbac6b9e05180f492aeed9fcf70752556f95624", + "sha256:875de9414393e778b655a3d97d60465eb3fae7c919e88b70cc10b40b9f56042d", + "sha256:8db5ba8b7da759b727faebc4289a9e6a51edadc7fc32207a30f7c6203a181592", + "sha256:92e512a6af769e4725fa5b25981ba790335d42c5977e94ded07db7d641490a85", + "sha256:9886a72c8e6371280cb247c5d32c9c8fa141dc560124348762db8a8b236f8692", + "sha256:9e55dff5ec115316dd7a083cdc1a52de63693695aecf72bc53a8e1468ce429e5", + "sha256:a42c9fa3abcda0dcfad053e49c4f752eef71ecd8c155221e18b99d4224621176", + "sha256:a571bc8ac092a3175a1d994794a8e7a1f2f651e7c744de24a19b4f740fe95034", + "sha256:af66001d7b76a3fab0d5e4c1ec9339ac45748bc4a399cbc2baa48c1980d3c1f4", + "sha256:b39a6e21110204a8c08d40ff56a73ba542ec60bab701c36ce721e7990df49fb9", + "sha256:b560f075c151900587ade06706b0c51d04b3277c111151997ea0813455378ae0", + "sha256:c8f1792d20d2f4e875ce7a113f43c3561ad12b34ff796b84002a256f37ce9437", + "sha256:cb9a758ad973e795267da334a92dd82bb7555cb36a0960dcabcf724d26299db8", + "sha256:ccca778c0737a773a1ad86b68bda52a71ad5950b25e120b6eb1330f0df54c3d0", + "sha256:ccd87c25e4c8559e1b918d46b4fa90b37f459c9b4566f1dfbce0eb8122571547", + "sha256:d143c5a9dada696bcfdb96ba2de4a47d5a89168e71d05a076e88a01386872f97", + "sha256:d80eeb5189d7d4b1af519fc3f148fe7521b9dfce8f4d6a0820e8f5769b005051", + "sha256:e04ab55cf49daf1aeb8c622c54d23fa4bec91cb051a43cc24351ba97e1dd09f5", + "sha256:f146c61ae128ab43ea3a0955de1af7e1633942c2b2b4985ac51cc292daf33222", + "sha256:f776c2c30f0e5f4db45c3ee11a5f2a8d9de68e81eb73ec4237de1e32e04ae81c" ], "markers": "python_version >= '3.7'", - "version": "==2.0.21" + "version": "==2.0.22" }, "starlette": { "hashes": [ @@ -1018,11 +1115,11 @@ }, "tzlocal": { "hashes": [ - "sha256:46eb99ad4bdb71f3f72b7d24f4267753e240944ecfc16f25d2719ba89827a803", - "sha256:f3596e180296aaf2dbd97d124fe76ae3a0e3d32b258447de7b939b3fd4be992f" + "sha256:2938498395d5f6a898ab8009555cb37a4d360913ad375d4747ef16826b03ef23", + "sha256:a5ccb2365b295ed964e0a98ad076fe10c495591e75505d34f154d60a7f1ed722" ], "markers": "python_version >= '3.7'", - "version": "==5.0.1" + "version": "==5.1" }, "urllib3": { "hashes": [ @@ -1040,43 +1137,49 @@ "sha256:1f9be6558f01239d4fdf22ef8126c39cb1ad0addf76c40e760549d2c2f43ab53", "sha256:4d3cc12d7727ba72b64d12d3cc7743124074c0a69f7b201512fc50c3e3f1569a" ], - "index": "pypi", + "markers": "python_version >= '3.8'", "version": "==0.23.2" }, "uvloop": { "hashes": [ - "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d", - "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1", - "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595", - "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b", - "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05", - "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8", - "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20", - "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded", - "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c", - "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8", - "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474", - "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f", - "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62", - "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376", - "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c", - "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e", - "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b", - "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4", - "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578", - "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811", - "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d", - "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738", - "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa", - "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9", - "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539", - "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c", - "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718", - "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667", - "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c", - "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024" + "sha256:1121087dfeb46e9e65920b20d1f46322ba299b8d93f7cb61d76c94b5a1adc20c", + "sha256:12af0d2e1b16780051d27c12de7e419b9daeb3516c503ab3e98d364cc55303bb", + "sha256:1f354d669586fca96a9a688c585b6257706d216177ac457c92e15709acaece10", + "sha256:1f4a549cd747e6f4f8446f4b4c8cb79504a8372d5d3a9b4fc20e25daf8e76c05", + "sha256:211ce38d84118ae282a91408f61b85cf28e2e65a0a8966b9a97e0e9d67c48722", + "sha256:25b714f07c68dcdaad6994414f6ec0f2a3b9565524fba181dcbfd7d9598a3e73", + "sha256:280904236a5b333a273292b3bcdcbfe173690f69901365b973fa35be302d7781", + "sha256:2b8b7cf7806bdc745917f84d833f2144fabcc38e9cd854e6bc49755e3af2b53e", + "sha256:4d90858f32a852988d33987d608bcfba92a1874eb9f183995def59a34229f30d", + "sha256:53aca21735eee3859e8c11265445925911ffe410974f13304edb0447f9f58420", + "sha256:54b211c46facb466726b227f350792770fc96593c4ecdfaafe20dc00f3209aef", + "sha256:56c1026a6b0d12b378425e16250acb7d453abaefe7a2f5977143898db6cfe5bd", + "sha256:585b7281f9ea25c4a5fa993b1acca4ad3d8bc3f3fe2e393f0ef51b6c1bcd2fe6", + "sha256:58e44650cbc8607a218caeece5a689f0a2d10be084a69fc32f7db2e8f364927c", + "sha256:61151cc207cf5fc88863e50de3d04f64ee0fdbb979d0b97caf21cae29130ed78", + "sha256:6132318e1ab84a626639b252137aa8d031a6c0550250460644c32ed997604088", + "sha256:680da98f12a7587f76f6f639a8aa7708936a5d17c5e7db0bf9c9d9cbcb616593", + "sha256:6e20bb765fcac07879cd6767b6dca58127ba5a456149717e0e3b1f00d8eab51c", + "sha256:74020ef8061678e01a40c49f1716b4f4d1cc71190d40633f08a5ef8a7448a5c6", + "sha256:75baba0bfdd385c886804970ae03f0172e0d51e51ebd191e4df09b929771b71e", + "sha256:847f2ed0887047c63da9ad788d54755579fa23f0784db7e752c7cf14cf2e7506", + "sha256:8849b8ef861431543c07112ad8436903e243cdfa783290cbee3df4ce86d8dd48", + "sha256:895a1e3aca2504638a802d0bec2759acc2f43a0291a1dff886d69f8b7baff399", + "sha256:99deae0504547d04990cc5acf631d9f490108c3709479d90c1dcd14d6e7af24d", + "sha256:ad79cd30c7e7484bdf6e315f3296f564b3ee2f453134a23ffc80d00e63b3b59e", + "sha256:b028776faf9b7a6d0a325664f899e4c670b2ae430265189eb8d76bd4a57d8a6e", + "sha256:b0a8f706b943c198dcedf1f2fb84899002c195c24745e47eeb8f2fb340f7dfc3", + "sha256:c65585ae03571b73907b8089473419d8c0aff1e3826b3bce153776de56cbc687", + "sha256:c6d341bc109fb8ea69025b3ec281fcb155d6824a8ebf5486c989ff7748351a37", + "sha256:d5d1135beffe9cd95d0350f19e2716bc38be47d5df296d7cc46e3b7557c0d1ff", + "sha256:db1fcbad5deb9551e011ca589c5e7258b5afa78598174ac37a5f15ddcfb4ac7b", + "sha256:e14de8800765b9916d051707f62e18a304cde661fa2b98a58816ca38d2b94029", + "sha256:e3d301e23984dcbc92d0e42253e0e0571915f0763f1eeaf68631348745f2dccc", + "sha256:ed3c28337d2fefc0bac5705b9c66b2702dc392f2e9a69badb1d606e7e7f773bb", + "sha256:edbb4de38535f42f020da1e3ae7c60f2f65402d027a08a8c60dc8569464873a6", + "sha256:f3b18663efe0012bc4c315f1b64020e44596f5fabc281f5b0d9bc9465288559c" ], - "version": "==0.17.0" + "version": "==0.18.0" }, "validators": { "hashes": [ @@ -1088,30 +1191,83 @@ }, "watchfiles": { "hashes": [ - "sha256:007dcc4a401093010b389c044e81172c8a2520dba257c88f8828b3d460c6bb38", - "sha256:08dc702529bb06a2b23859110c214db245455532da5eaea602921687cfcd23db", - "sha256:0d82dbc1832da83e441d112069833eedd4cf583d983fb8dd666fbefbea9d99c0", - "sha256:13f995d5152a8ba4ed7c2bbbaeee4e11a5944defc7cacd0ccb4dcbdcfd78029a", - "sha256:3796312bd3587e14926013612b23066912cf45a14af71cf2b20db1c12dadf4e9", - "sha256:5392dd327a05f538c56edb1c6ebba6af91afc81b40822452342f6da54907bbdf", - "sha256:570848706440373b4cd8017f3e850ae17f76dbdf1e9045fc79023b11e1afe490", - "sha256:608cd94a8767f49521901aff9ae0c92cc8f5a24d528db7d6b0295290f9d41193", - "sha256:728575b6b94c90dd531514677201e8851708e6e4b5fe7028ac506a200b622019", - "sha256:7d4e66a857621584869cfbad87039e65dadd7119f0d9bb9dbc957e089e32c164", - "sha256:835df2da7a5df5464c4a23b2d963e1a9d35afa422c83bf4ff4380b3114603644", - "sha256:87d9e1f75c4f86c93d73b5bd1ebe667558357548f11b4f8af4e0e272f79413ce", - "sha256:89d1de8218874925bce7bb2ae9657efc504411528930d7a83f98b1749864f2ef", - "sha256:99f4c65fd2fce61a571b2a6fcf747d6868db0bef8a934e8ca235cc8533944d95", - "sha256:9a0351d20d03c6f7ad6b2e8a226a5efafb924c7755ee1e34f04c77c3682417fa", - "sha256:9b5c8d3be7b502f8c43a33c63166ada8828dbb0c6d49c8f9ce990a96de2f5a49", - "sha256:a03d1e6feb7966b417f43c3e3783188167fd69c2063e86bad31e62c4ea794cc5", - "sha256:b17d4176c49d207865630da5b59a91779468dd3e08692fe943064da260de2c7c", - "sha256:d0002d81c89a662b595645fb684a371b98ff90a9c7d8f8630c82f0fde8310458", - "sha256:d97db179f7566dcf145c5179ddb2ae2a4450e3a634eb864b09ea04e68c252e8e", - "sha256:e43af4464daa08723c04b43cf978ab86cc55c684c16172622bdac64b34e36af0", - "sha256:eccc8942bcdc7d638a01435d915b913255bbd66f018f1af051cd8afddb339ea3" + "sha256:02b73130687bc3f6bb79d8a170959042eb56eb3a42df3671c79b428cd73f17cc", + "sha256:02d91cbac553a3ad141db016e3350b03184deaafeba09b9d6439826ee594b365", + "sha256:06247538e8253975bdb328e7683f8515ff5ff041f43be6c40bff62d989b7d0b0", + "sha256:08dca260e85ffae975448e344834d765983237ad6dc308231aa16e7933db763e", + "sha256:0d9ac347653ebd95839a7c607608703b20bc07e577e870d824fa4801bc1cb124", + "sha256:0dd5fad9b9c0dd89904bbdea978ce89a2b692a7ee8a0ce19b940e538c88a809c", + "sha256:11cd0c3100e2233e9c53106265da31d574355c288e15259c0d40a4405cbae317", + "sha256:18722b50783b5e30a18a8a5db3006bab146d2b705c92eb9a94f78c72beb94094", + "sha256:18d5b4da8cf3e41895b34e8c37d13c9ed294954907929aacd95153508d5d89d7", + "sha256:1ad7247d79f9f55bb25ab1778fd47f32d70cf36053941f07de0b7c4e96b5d235", + "sha256:1b8d1eae0f65441963d805f766c7e9cd092f91e0c600c820c764a4ff71a0764c", + "sha256:1bd467213195e76f838caf2c28cd65e58302d0254e636e7c0fca81efa4a2e62c", + "sha256:1c9198c989f47898b2c22201756f73249de3748e0fc9de44adaf54a8b259cc0c", + "sha256:1fd9a5205139f3c6bb60d11f6072e0552f0a20b712c85f43d42342d162be1235", + "sha256:214cee7f9e09150d4fb42e24919a1e74d8c9b8a9306ed1474ecaddcd5479c293", + "sha256:27b4035013f1ea49c6c0b42d983133b136637a527e48c132d368eb19bf1ac6aa", + "sha256:3a23092a992e61c3a6a70f350a56db7197242f3490da9c87b500f389b2d01eef", + "sha256:3ad692bc7792be8c32918c699638b660c0de078a6cbe464c46e1340dadb94c19", + "sha256:3ccceb50c611c433145502735e0370877cced72a6c70fd2410238bcbc7fe51d8", + "sha256:3d0f32ebfaa9c6011f8454994f86108c2eb9c79b8b7de00b36d558cadcedaa3d", + "sha256:3f92944efc564867bbf841c823c8b71bb0be75e06b8ce45c084b46411475a915", + "sha256:40bca549fdc929b470dd1dbfcb47b3295cb46a6d2c90e50588b0a1b3bd98f429", + "sha256:43babacef21c519bc6631c5fce2a61eccdfc011b4bcb9047255e9620732c8097", + "sha256:4566006aa44cb0d21b8ab53baf4b9c667a0ed23efe4aaad8c227bfba0bf15cbe", + "sha256:49f56e6ecc2503e7dbe233fa328b2be1a7797d31548e7a193237dcdf1ad0eee0", + "sha256:4c48a10d17571d1275701e14a601e36959ffada3add8cdbc9e5061a6e3579a5d", + "sha256:4ea10a29aa5de67de02256a28d1bf53d21322295cb00bd2d57fcd19b850ebd99", + "sha256:511f0b034120cd1989932bf1e9081aa9fb00f1f949fbd2d9cab6264916ae89b1", + "sha256:51ddac60b96a42c15d24fbdc7a4bfcd02b5a29c047b7f8bf63d3f6f5a860949a", + "sha256:57d430f5fb63fea141ab71ca9c064e80de3a20b427ca2febcbfcef70ff0ce895", + "sha256:59137c0c6826bd56c710d1d2bda81553b5e6b7c84d5a676747d80caf0409ad94", + "sha256:5a03651352fc20975ee2a707cd2d74a386cd303cc688f407296064ad1e6d1562", + "sha256:5eb86c6acb498208e7663ca22dbe68ca2cf42ab5bf1c776670a50919a56e64ab", + "sha256:642d66b75eda909fd1112d35c53816d59789a4b38c141a96d62f50a3ef9b3360", + "sha256:6674b00b9756b0af620aa2a3346b01f8e2a3dc729d25617e1b89cf6af4a54eb1", + "sha256:668c265d90de8ae914f860d3eeb164534ba2e836811f91fecc7050416ee70aa7", + "sha256:66fac0c238ab9a2e72d026b5fb91cb902c146202bbd29a9a1a44e8db7b710b6f", + "sha256:6c107ea3cf2bd07199d66f156e3ea756d1b84dfd43b542b2d870b77868c98c03", + "sha256:6c889025f59884423428c261f212e04d438de865beda0b1e1babab85ef4c0f01", + "sha256:6cb8fdc044909e2078c248986f2fc76f911f72b51ea4a4fbbf472e01d14faa58", + "sha256:6e9be3ef84e2bb9710f3f777accce25556f4a71e15d2b73223788d528fcc2052", + "sha256:7f762a1a85a12cc3484f77eee7be87b10f8c50b0b787bb02f4e357403cad0c0e", + "sha256:83a696da8922314ff2aec02987eefb03784f473281d740bf9170181829133765", + "sha256:853853cbf7bf9408b404754b92512ebe3e3a83587503d766d23e6bf83d092ee6", + "sha256:8ad3fe0a3567c2f0f629d800409cd528cb6251da12e81a1f765e5c5345fd0137", + "sha256:8c6ed10c2497e5fedadf61e465b3ca12a19f96004c15dcffe4bd442ebadc2d85", + "sha256:8d5f400326840934e3507701f9f7269247f7c026d1b6cfd49477d2be0933cfca", + "sha256:927c589500f9f41e370b0125c12ac9e7d3a2fd166b89e9ee2828b3dda20bfe6f", + "sha256:9a0aa47f94ea9a0b39dd30850b0adf2e1cd32a8b4f9c7aa443d852aacf9ca214", + "sha256:9b37a7ba223b2f26122c148bb8d09a9ff312afca998c48c725ff5a0a632145f7", + "sha256:9c873345680c1b87f1e09e0eaf8cf6c891b9851d8b4d3645e7efe2ec20a20cc7", + "sha256:9d09869f2c5a6f2d9df50ce3064b3391d3ecb6dced708ad64467b9e4f2c9bef3", + "sha256:9d353c4cfda586db2a176ce42c88f2fc31ec25e50212650c89fdd0f560ee507b", + "sha256:a1e3014a625bcf107fbf38eece0e47fa0190e52e45dc6eee5a8265ddc6dc5ea7", + "sha256:a3b9bec9579a15fb3ca2d9878deae789df72f2b0fdaf90ad49ee389cad5edab6", + "sha256:ab03a90b305d2588e8352168e8c5a1520b721d2d367f31e9332c4235b30b8994", + "sha256:aff06b2cac3ef4616e26ba17a9c250c1fe9dd8a5d907d0193f84c499b1b6e6a9", + "sha256:b3cab0e06143768499384a8a5efb9c4dc53e19382952859e4802f294214f36ec", + "sha256:b4a21f71885aa2744719459951819e7bf5a906a6448a6b2bbce8e9cc9f2c8128", + "sha256:b6d45d9b699ecbac6c7bd8e0a2609767491540403610962968d258fd6405c17c", + "sha256:be6dd5d52b73018b21adc1c5d28ac0c68184a64769052dfeb0c5d9998e7f56a2", + "sha256:c550a56bf209a3d987d5a975cdf2063b3389a5d16caf29db4bdddeae49f22078", + "sha256:c76c635fabf542bb78524905718c39f736a98e5ab25b23ec6d4abede1a85a6a3", + "sha256:c81818595eff6e92535ff32825f31c116f867f64ff8cdf6562cd1d6b2e1e8f3e", + "sha256:cfb92d49dbb95ec7a07511bc9efb0faff8fe24ef3805662b8d6808ba8409a71a", + "sha256:d23bcd6c8eaa6324fe109d8cac01b41fe9a54b8c498af9ce464c1aeeb99903d6", + "sha256:d5b1dc0e708fad9f92c296ab2f948af403bf201db8fb2eb4c8179db143732e49", + "sha256:d78f30cbe8b2ce770160d3c08cff01b2ae9306fe66ce899b73f0409dc1846c1b", + "sha256:d8f57c4461cd24fda22493109c45b3980863c58a25b8bec885ca8bea6b8d4b28", + "sha256:d9792dff410f266051025ecfaa927078b94cc7478954b06796a9756ccc7e14a9", + "sha256:e7941bbcfdded9c26b0bf720cb7e6fd803d95a55d2c14b4bd1f6a2772230c586", + "sha256:ebe684d7d26239e23d102a2bad2a358dedf18e462e8808778703427d1f584400", + "sha256:ec8c8900dc5c83650a63dd48c4d1d245343f904c4b64b48798c67a3767d7e165", + "sha256:f564bf68404144ea6b87a78a3f910cc8de216c6b12a4cf0b27718bf4ec38d303", + "sha256:fd7ac678b92b29ba630d8c842d8ad6c555abda1b9ef044d6cc092dacbfc9719d" ], - "version": "==0.20.0" + "version": "==0.21.0" }, "websockets": { "hashes": [ @@ -1189,5 +1345,16 @@ "version": "==11.0.3" } }, - "develop": {} + "develop": { + "pysocks": { + "hashes": [ + "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299", + "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", + "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.7.1" + } + } } diff --git a/README.md b/README.md index 6c99518..5d89437 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ - **Python**: This project uses Python version 3.11. Ensure you have it installed. - **MongoDB**: Set up a MongoDB server. You can use [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) to create a free MongoDB cluster. +- **mkcert**: To set up local HTTPS, you'll need to use mkcert to generate SSL certificates. If not installed, get it from [here](https://github.com/FiloSottile/mkcert). ### Setup @@ -44,9 +45,35 @@ ```bash MONGO_URI= SECRET_KEY= - HOST_URL=http://127.0.0.1:8000 + HOST_URL=https://localhost:8443 ``` -4. **For scraping instructions**: refer to the [scrapping README](/scrappers/README.md). +4. **Local HTTPS Setup**: + + - Navigate to the MediaFusion directory. + - Generate local SSL certificates using mkcert: + + ```bash + mkcert -install + mkcert localhost + ``` + + This will generate two files: localhost.pem and localhost-key.pem. + +5. **Run Servers**: + + - To serve your application over HTTPS on port 8443: + + ```bash + pipenv run uvicorn main:app --host 0.0.0.0 --port 8443 --ssl-keyfile localhost-key.pem --ssl-certfile localhost.pem + ``` + + - Since Stremio doesn't support localhost HTTPS servers to install add-on, also run an HTTP server on port 8000: + + ```bash + pipenv run uvicorn main:app --host 0.0.0.0 --port 8000 + ``` + +6. **For scraping instructions**: refer to the [scrapping README](/scrappers/README.md). ## :books: References diff --git a/api/main.py b/api/main.py index 330f3b4..1f08478 100644 --- a/api/main.py +++ b/api/main.py @@ -3,9 +3,10 @@ import logging from typing import Literal from apscheduler.schedulers.asyncio import AsyncIOScheduler -from fastapi import FastAPI, Request, Response, BackgroundTasks, Depends, HTTPException +from apscheduler.triggers.cron import CronTrigger +from fastapi import FastAPI, Request, Response, Depends, HTTPException from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import RedirectResponse, FileResponse +from fastapi.responses import RedirectResponse, FileResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -16,8 +17,11 @@ from streaming_providers.realdebrid.api import router as realdebrid_router from streaming_providers.realdebrid.utils import get_direct_link_from_realdebrid from streaming_providers.seedr.api import router as seedr_router from streaming_providers.seedr.utils import get_direct_link_from_seedr -from utils import crypto, torrent +from streaming_providers.debridlink.api import router as debridlink_router +from streaming_providers.debridlink.utils import get_direct_link_from_debridlink +from utils import crypto, torrent, poster from utils.const import CATALOG_ID_DATA, CATALOG_NAME_DATA +from scrappers import tamil_blasters, tamilmv logging.basicConfig( format="%(levelname)s::%(asctime)s - %(message)s", @@ -55,7 +59,8 @@ async def init_db(): @app.on_event("startup") async def start_scheduler(): scheduler = AsyncIOScheduler() - # scheduler.add_job(tb_scrapper.run_schedule_scrape, CronTrigger(hour="*/3")) + scheduler.add_job(tamil_blasters.run_schedule_scrape, CronTrigger(hour="*/3")) + scheduler.add_job(tamilmv.run_schedule_scrape, CronTrigger(hour="*/3")) scheduler.start() app.state.scheduler = scheduler @@ -174,28 +179,35 @@ async def get_catalog( @app.get( - "/{secret_str}/catalog/{catalog_type}/tamil_blasters/search={search_query}.json", - response_model=schemas.Metas, + "/catalog/{catalog_type}/{catalog_id}/search={search_query}.json", tags=["search"], -) -@app.get( - "/catalog/{catalog_type}/tamil_blasters/search={search_query}.json", response_model=schemas.Metas, - tags=["search"], + response_model_exclude_none=True, ) async def search_movie( response: Response, catalog_type: Literal["movie", "series"], + catalog_id: Literal["mediafusion_search_movies", "mediafusion_search_series"], search_query: str, ): response.headers.update(headers) - logging.debug("Searching for %s", search_query) + logging.debug("Searching for %s : %s", catalog_id, search_query) return await crud.process_search_query(search_query, catalog_type) -@app.get("/{secret_str}/meta/{catalog_type}/{meta_id}.json", tags=["meta"]) -@app.get("/meta/{catalog_type}/{meta_id}.json", tags=["meta"]) +@app.get( + "/{secret_str}/meta/{catalog_type}/{meta_id}.json", + tags=["meta"], + response_model=schemas.Meta, + response_model_exclude_none=True, +) +@app.get( + "/meta/{catalog_type}/{meta_id}.json", + tags=["meta"], + response_model=schemas.Meta, + response_model_exclude_none=True, +) async def get_meta( catalog_type: Literal["movie", "series"], meta_id: str, response: Response ): @@ -284,10 +296,14 @@ async def streaming_provider_endpoint( video_url = await get_direct_link_from_seedr( info_hash, magnet_link, user_data, stream, episode_data, 3, 1 ) - else: + elif user_data.streaming_provider.service == "realdebrid": video_url = get_direct_link_from_realdebrid( info_hash, magnet_link, user_data, stream, episode_data, 3, 1 ) + else: + video_url = get_direct_link_from_debridlink( + info_hash, magnet_link, user_data, stream, episode_data, 3, 1 + ) except ProviderException as error: logging.info("Exception occurred: %s", error.message) video_url = f"{settings.host_url}/static/exceptions/{error.video_file_name}" @@ -295,5 +311,27 @@ async def streaming_provider_endpoint( return RedirectResponse(url=video_url, headers=response.headers) +@app.get("/poster/{catalog_type}/{mediafusion_id}.jpg", tags=["poster"]) +async def get_poster(catalog_type: Literal["movie", "series"], mediafusion_id: str): + # Query the MediaFusion data + if catalog_type == "movie": + mediafusion_data = await crud.get_movie_data_by_id(mediafusion_id) + else: + mediafusion_data = await crud.get_series_data_by_id(mediafusion_id) + + if not mediafusion_data: + raise HTTPException(status_code=404, detail="MediaFusion ID not found.") + + try: + image_byte_io = await poster.create_poster(mediafusion_data) + return StreamingResponse( + image_byte_io, media_type="image/jpeg", headers=headers + ) + except Exception as e: + logging.error(f"Unexpected error while creating poster: {e}", exc_info=True) + raise HTTPException(status_code=404, detail="Failed to create poster.") + + app.include_router(seedr_router, prefix="/seedr", tags=["seedr"]) app.include_router(realdebrid_router, prefix="/realdebrid", tags=["realdebrid"]) +app.include_router(debridlink_router, prefix="/debridlink", tags=["debridlink"]) \ No newline at end of file diff --git a/beamup.json b/beamup.json index affa6ac..ed06e13 100644 --- a/beamup.json +++ b/beamup.json @@ -1,4 +1,4 @@ { "projectName": "mediafusion", - "lastCommit": "b3d3866" + "lastCommit": "f8743ed" } \ No newline at end of file diff --git a/db/crud.py b/db/crud.py index ac7223d..1790977 100644 --- a/db/crud.py +++ b/db/crud.py @@ -3,76 +3,43 @@ from typing import Optional from uuid import uuid4 from beanie import WriteRules -from imdb import Cinemagoer, IMDbDataAccessError +from beanie.operators import In from db import schemas +from db.config import settings from db.models import ( MediaFusionMovieMetaData, MediaFusionSeriesMetaData, Streams, Season, Episode, - MediaFusionMetaData, ) from db.schemas import Stream, MetaIdProjection -from utils.parser import parse_stream_data, get_catalogs - -ia = Cinemagoer() +from utils.parser import parse_stream_data, get_catalogs, search_imdb async def get_meta_list( - catalog_type: str, catalog: str, skip: int = 0, limit: int = 100 + catalog_type: str, catalog: str, skip: int = 0, limit: int = 25 ) -> list[schemas.Meta]: - - pipeline = [ - # Lookup to join with the Streams collection - { - "$lookup": { - "from": "Streams", - "localField": "streams.$id", - "foreignField": "_id", - "as": "joined_streams", - } - }, - # Unwind the joined streams to process each stream separately - {"$unwind": "$joined_streams"}, - # Match the series based on catalog - {"$match": {"joined_streams.catalog": catalog}}, - # Group by series ID and determine the latest date - { - "$group": { - "_id": "$_id", - "latest_date": {"$max": "$joined_streams.created_at"}, - "title": {"$first": "$title"}, - "poster": {"$first": "$poster"}, - "type": {"$first": "$type"}, - } - }, - # Rename _id to id - { - "$project": { - "id": "$_id", - "latest_date": 1, - "title": 1, - "poster": 1, - "type": 1, - } - }, - # Sort by the latest_date in descending order - {"$sort": {"latest_date": -1}}, - # Pagination - {"$skip": skip}, - {"$limit": limit}, - ] - if catalog_type == "movie": - meta_list = await MediaFusionMovieMetaData.aggregate( - pipeline, projection_model=schemas.Meta - ).to_list() + meta_class = MediaFusionMovieMetaData else: - meta_list = await MediaFusionSeriesMetaData.aggregate( - pipeline, projection_model=schemas.Meta - ).to_list() + meta_class = MediaFusionSeriesMetaData + + meta_list = ( + await meta_class.find( + In(meta_class.streams.catalog, [catalog]), + fetch_links=True, + ) + .sort(-meta_class.streams.created_at) + .skip(skip) + .limit(limit) + .project(schemas.Meta) + .to_list() + ) + for meta in meta_list: + meta.poster = f"{settings.host_url}/poster/{catalog_type}/{meta.id}.jpg" + return meta_list @@ -124,10 +91,10 @@ async def get_movie_meta(meta_id: str): return { "meta": { - "id": meta_id, + "_id": meta_id, "type": "movie", - "name": movie_data.title, - "poster": movie_data.poster, + "title": movie_data.title, + "poster": f"{settings.host_url}/poster/movie/{meta_id}.jpg", "background": movie_data.poster, } } @@ -141,10 +108,10 @@ async def get_series_meta(meta_id: str): metadata = { "meta": { - "id": meta_id, + "_id": meta_id, "type": "series", - "name": series_data.title, - "poster": series_data.poster, + "title": series_data.title, + "poster": f"{settings.host_url}/poster/series/{meta_id}.jpg", "background": series_data.poster, "videos": [], } @@ -168,20 +135,6 @@ async def get_series_meta(meta_id: str): return metadata -def search_imdb(title: str, year: int, retry: int = 5) -> dict: - try: - result = ia.search_movie(f"{title} {year}") - except IMDbDataAccessError: - return search_imdb(title, year, retry - 1) if retry > 0 else {} - for movie in result: - if movie.get("year") == year and movie.get("title").lower() in title.lower(): - return { - "imdb_id": f"tt{movie.movieID}", - "poster": movie.get("full-size cover url"), - } - return {} - - async def save_movie_metadata(metadata: dict): # Try to get the existing movie existing_movie = await MediaFusionMovieMetaData.find_one( @@ -200,8 +153,10 @@ async def save_movie_metadata(metadata: dict): meta_id = f"mf{uuid4().fields[-1]}" # Update the poster from IMDb if available poster = imdb_data.get("poster") or metadata["poster"] + background = imdb_data.get("background") or metadata["poster"] else: poster = existing_movie.poster + background = existing_movie.background meta_id = existing_movie.id # Determine file index for the main movie file (largest file) @@ -255,6 +210,7 @@ async def save_movie_metadata(metadata: dict): title=metadata["title"], year=metadata["year"], poster=poster, + background=background, streams=[new_stream], ) await movie_data.insert(link_rule=WriteRules.WRITE) @@ -277,6 +233,7 @@ async def save_series_metadata(metadata: dict): if not series: meta_id = meta_id or f"mf{uuid4().fields[-1]}" poster = imdb_data.get("poster") or metadata["poster"] + background = imdb_data.get("background") or metadata["poster"] # Create an initial entry for the series series = MediaFusionSeriesMetaData( @@ -284,6 +241,7 @@ async def save_series_metadata(metadata: dict): title=metadata["title"], year=metadata["year"], poster=poster, + background=background, streams=[], ) await series.insert() @@ -353,15 +311,18 @@ async def save_series_metadata(metadata: dict): async def process_search_query(search_query: str, catalog_type: str) -> dict: - # Define the query with the text search and catalog type - query = {"$text": {"$search": search_query}, "type": catalog_type} + if catalog_type == "movie": + meta_class = MediaFusionMovieMetaData + else: + meta_class = MediaFusionSeriesMetaData - # Perform the search search_results = ( - await MediaFusionMetaData.find(query).project(MetaIdProjection).to_list() + await meta_class.find({"$text": {"$search": search_query}}) + .project(MetaIdProjection) + .to_list() ) - logging.debug( + logging.info( "Found %s results for %s in %s", len(search_results), search_query, catalog_type ) @@ -377,7 +338,7 @@ async def process_search_query(search_query: str, catalog_type: str) -> dict: if not meta: continue - metas.append(meta["meta"]) # Extract the 'meta' key from the result + metas.append(meta["meta"]) return {"metas": metas} diff --git a/db/models.py b/db/models.py index 45d0f97..feade45 100644 --- a/db/models.py +++ b/db/models.py @@ -55,7 +55,9 @@ class MediaFusionMetaData(Document): title: str year: Optional[int] poster: str + background: str streams: list[Link[Streams]] + type: str class Settings: is_root = True diff --git a/db/schemas.py b/db/schemas.py index 5fdbe54..0ba4df5 100644 --- a/db/schemas.py +++ b/db/schemas.py @@ -12,10 +12,11 @@ class Catalog(BaseModel): class Meta(BaseModel): - id: str + id: str = Field(alias="_id") name: str = Field(alias="title") type: str = Field(default="movie") poster: str + background: str videos: list | None = None @@ -37,7 +38,7 @@ class Streams(BaseModel): class StreamingProvider(BaseModel): - service: Literal["realdebrid", "seedr"] + service: Literal["realdebrid", "seedr", "debridlink"] token: str class Config: @@ -57,4 +58,4 @@ class AuthorizeData(BaseModel): class MetaIdProjection(BaseModel): - id: str + id: str = Field(alias="_id") diff --git a/resources/exceptions/need_premium.mp4 b/resources/exceptions/need_premium.mp4 new file mode 100644 index 0000000..3e22a4b Binary files /dev/null and b/resources/exceptions/need_premium.mp4 differ diff --git a/resources/html/configure.html b/resources/html/configure.html index 41e4872..d16238b 100644 --- a/resources/html/configure.html +++ b/resources/html/configure.html @@ -28,6 +28,7 @@ + diff --git a/resources/images/logo_text.png b/resources/images/logo_text.png new file mode 100644 index 0000000..8a5b6f0 Binary files /dev/null and b/resources/images/logo_text.png differ diff --git a/resources/js/config_script.js b/resources/js/config_script.js index cc62155..35e3e75 100644 --- a/resources/js/config_script.js +++ b/resources/js/config_script.js @@ -93,7 +93,7 @@ function adjustOAuthSectionDisplay() { const providerToken = document.getElementById('provider_token').value; setOAuthBtnTextContent(provider) - if ((provider === 'seedr' || provider === 'realdebrid') && !providerToken) { + if ((provider === 'seedr' || provider === 'realdebrid' || provider === 'debridlink') && !providerToken) { setElementDisplay('oauth_section', 'block'); } else { setElementDisplay('oauth_section', 'none'); @@ -136,6 +136,8 @@ oAuthBtn.addEventListener('click', async function () { await initiateOAuthFlow('/seedr/get-device-code', '/seedr/authorize'); } else if (provider === 'realdebrid') { await initiateOAuthFlow('/realdebrid/get-device-code', '/realdebrid/authorize'); + } else if (provider === 'debridlink') { + await initiateOAuthFlow('/debridlink/get-device-code', '/debridlink/authorize') } }); diff --git a/resources/manifest.json b/resources/manifest.json index d25941f..4dc32b0 100644 --- a/resources/manifest.json +++ b/resources/manifest.json @@ -1,6 +1,6 @@ { "id": "mhdzumair.addons.mediafusion", - "version": "3.3.0", + "version": "3.4.0", "name": "Media Fusion", "contactEmail": "mhdzumair@gmail", "description": "Watch Tamil, Hindi, Malayalam, Kannada, English, and dubbed movies & series with the Media Fusion add-on. Source: https://github.com/mhdzumair/MediaFusion", @@ -83,6 +83,17 @@ } ] }, + { + "id": "tamil_old", + "type": "movie", + "name": "Tamil Old Movies", + "extra": [ + { + "name": "skip", + "isRequired": false + } + ] + }, { "id": "tamil_dubbed", "type": "movie", @@ -127,6 +138,17 @@ } ] }, + { + "id": "malayalam_old", + "type": "movie", + "name": "Malayalam Old Movies", + "extra": [ + { + "name": "skip", + "isRequired": false + } + ] + }, { "id": "malayalam_dubbed", "type": "movie", @@ -171,6 +193,17 @@ } ] }, + { + "id": "telugu_old", + "type": "movie", + "name": "Telugu Old Movies", + "extra": [ + { + "name": "skip", + "isRequired": false + } + ] + }, { "id": "telugu_dubbed", "type": "movie", @@ -215,6 +248,17 @@ } ] }, + { + "id": "hindi_old", + "type": "movie", + "name": "Hindi Old Movies", + "extra": [ + { + "name": "skip", + "isRequired": false + } + ] + }, { "id": "hindi_dubbed", "type": "movie", @@ -259,6 +303,17 @@ } ] }, + { + "id": "kannada_old", + "type": "movie", + "name": "Kannada Old Movies", + "extra": [ + { + "name": "skip", + "isRequired": false + } + ] + }, { "id": "kannada_dubbed", "type": "movie", diff --git a/scrappers/README.md b/scrappers/README.md index aefed1e..d1b4565 100644 --- a/scrappers/README.md +++ b/scrappers/README.md @@ -1,6 +1,8 @@ # Scrapping with Media Fusion -Currently, the project supports scraping from TamilBlasters. Here's how to use the scraper: +The project offers scraping capabilities for TamilBlasters and TamilMV. Here are the guidelines on how to use these scrapers: + +## TamilBlasters 1. **Setup Playwright**: @@ -10,38 +12,12 @@ Currently, the project supports scraping from TamilBlasters. Here's how to use t pipenv run playwright install firefox ``` -2. **Get Help on Available Options **: +2. **Get Help on Available Options**: To understand the available options for the scraper, run: ```bash - pipenv run python3 -m scrappers.tamil_blasters_scrapper --help - ``` - - This will display the available options: - - ```bash - usage: tamil_blasters_scrapper.py [-h] [--all] [-l {tamil,malayalam,telugu,hindi,kannada,english}] [-t {hdrip,tcrip,dubbed,series}] [-p PAGES] [-s START_PAGES] [-k SEARCH_KEYWORD] [--scrap-with-playwright] [--proxy-url PROXY_URL] - - Scrap Movie metadata from TamilBlasters - - options: - -h, --help show this help message and exit - --all scrap all type of movies & series - -l {tamil,malayalam,telugu,hindi,kannada,english}, --language {tamil,malayalam,telugu,hindi,kannada,english} - scrap movie language - -t {hdrip,tcrip,dubbed,series}, --video-type {hdrip,tcrip,dubbed,series} - scrap movie video type - -p PAGES, --pages PAGES - number of scrap pages - -s START_PAGES, --start-pages START_PAGES - page number to start scrap. - -k SEARCH_KEYWORD, --search-keyword SEARCH_KEYWORD - search keyword to scrap movies & series. ex: 'bigg boss' - --scrap-with-playwright - scrap with playwright - --proxy-url PROXY_URL - proxy url to scrap. ex: socks5://127.0.0.1:1080 + pipenv run python3 -m scrappers.tamil_blasters --help ``` 3. **Run the Scraper**: @@ -51,7 +27,30 @@ Currently, the project supports scraping from TamilBlasters. Here's how to use t e.g. To scrap all movies & series, run: ```bash - pipenv run python3 -m scrappers.tamil_blasters_scrapper --all --scrap-with-playwright + pipenv run python3 -m scrappers.tamil_blasters --all --scrap-with-playwright ``` Note: You may have to solve the cloudflare validation challenge manually when its required. + + +## TamilMV + +1. **Get Help on Available Options**: + + To understand the available options for the TamilMV scraper, run: + + ```bash + pipenv run python3 -m scrappers.tamilmv --help + ``` + +2. **Run the TamilMV Scraper**: + + Execute the scraper with the desired options based on your requirements. + + e.g. To scrap all movies & series from TamilMV for 5 pages, run: + + ```bash + pipenv run python3 -m scrappers.tamilmv --all -p 5 + ``` + + Note: Ensure you have Playwright set up as mentioned in the TamilBlasters section if you intend to use it with the TamilMV scraper. \ No newline at end of file diff --git a/scrappers/helpers.py b/scrappers/helpers.py index 10987f5..5458028 100644 --- a/scrappers/helpers.py +++ b/scrappers/helpers.py @@ -1,5 +1,6 @@ import logging +import PTN import cloudscraper import requests from requests.adapters import HTTPAdapter @@ -73,7 +74,12 @@ async def download_and_save_torrent( logging.error(f"Info hash not found for {torrent_link}") return False - metadata.update({"torrent_metadata": torrent_metadata}) + parsed_data = PTN.parse(torrent_metadata["torrent_name"]) + metadata.update({"torrent_metadata": torrent_metadata, **parsed_data}) + + if not metadata.get("year"): + logging.error(f"Year not found for {page_link}") + return False # Saving the metadata if media_type == "series": diff --git a/scrappers/tamil_blasters_scrapper.py b/scrappers/tamil_blasters.py similarity index 91% rename from scrappers/tamil_blasters_scrapper.py rename to scrappers/tamil_blasters.py index a942377..7601077 100755 --- a/scrappers/tamil_blasters_scrapper.py +++ b/scrappers/tamil_blasters.py @@ -7,7 +7,6 @@ import math import random import re -import PTN from bs4 import BeautifulSoup from dateutil.parser import parse as dateparser from playwright.async_api import async_playwright @@ -20,37 +19,42 @@ from scrappers.helpers import ( download_and_save_torrent, ) -HOMEPAGE = "https://www.tamilblasters.life" +HOMEPAGE = "https://www.1tamilblasters.art" TAMIL_BLASTER_LINKS = { "tamil": { "hdrip": "7-tamil-new-movies-hdrips-bdrips-dvdrips-hdtv", "tcrip": "8-tamil-new-movies-tcrip-dvdscr-hdcam-predvd", "dubbed": "9-tamil-dubbed-movies-bdrips-hdrips-dvdscr-hdcam-in-multi-audios", "series": "63-tamil-new-web-series-tv-shows", + "old": "56-tamil-old-mid-movies-bdrips-hdrips-dvdrips-hdtv", }, "malayalam": { "tcrip": "75-malayalam-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": "74-malayalam-new-movies-hdrips-bdrips-dvdrips-hdtv", "dubbed": "76-malayalam-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", "series": "98-malayalam-new-web-series-tv-shows", + "old": "77-malayalam-old-mid-movies-bdrips-hdrips-dvdrips", }, "telugu": { "tcrip": "79-telugu-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": "78-telugu-new-movies-hdrips-bdrips-dvdrips-hdtv", "dubbed": "80-telugu-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", "series": "96-telugu-new-web-series-tv-shows", + "old": "81-telugu-old-mid-movies-bdrips-hdrips-dvdrips", }, "hindi": { "tcrip": "87-hindi-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": "86-hindi-new-movies-hdrips-bdrips-dvdrips-hdtv", "dubbed": "88-hindi-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", - "series": "forums/forum/89-hindi-new-web-series-tv-shows", + "series": "89-hindi-new-web-series-tv-shows", + "old": "102-hindi-old-mid-movies-bdrips-hdrips-dvdrips", }, "kannada": { "tcrip": "83-kannada-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": "82-kannada-new-movies-hdrips-bdrips-dvdrips-hdtv", "dubbed": "84-kannada-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", "series": "103-kannada-new-web-series-tv-shows", + "old": "85-kannada-old-mid-movies-bdrips-hdrips-dvdrips", }, "english": { "tcrip": "52-english-movies-hdcam-dvdscr-predvd", @@ -95,7 +99,6 @@ async def process_movie( logging.error(f"Movie link not found") return - metadata = PTN.parse(re.sub(r"\s+", " ", movie_link.text)) page_link = movie_link.get("href") try: @@ -118,16 +121,14 @@ async def process_movie( dateparser(datetime_element.get("datetime")) if datetime_element else None ) - # Updating metadata - metadata.update( - { - "catalog": f"{language}_{media_type}", - "poster": poster, - "created_at": created_at, - "scrap_language": language.title(), - "source": "TamilBlasters", - } - ) + # Define metadata + metadata = { + "catalog": f"{language}_{media_type}", + "poster": poster, + "created_at": created_at, + "scrap_language": language.title(), + "source": "TamilBlasters", + } # Extracting torrent details torrent_elements = movie_page.select("a[data-fileext='torrent']") @@ -141,7 +142,7 @@ async def process_movie( torrent_element, scraper=scraper, page=page, - metadata=metadata, + metadata=metadata.copy(), media_type=media_type, page_link=page_link, ) @@ -254,12 +255,12 @@ async def run_scraper( await scrap_search_keyword(search_keyword, proxy_url) return try: - scrap_link = f"{HOMEPAGE}/index.php?/forums/forum/{TAMIL_BLASTER_LINKS[language][video_type]}" + scrap_link_prefix = f"{HOMEPAGE}/index.php?/forums/forum/{TAMIL_BLASTER_LINKS[language][video_type]}" except KeyError: logging.error(f"Unsupported language or video type: {language}_{video_type}") return for page in range(start_page, pages + start_page): - scrap_link = f"{scrap_link}/page/{page}/" + scrap_link = f"{scrap_link_prefix}/page/{page}/" logging.info(f"Scrap page: {page}") if scrap_with_playwright is True: await scrap_page_with_playwright( @@ -308,7 +309,7 @@ if __name__ == "__main__": "--video-type", help="scrap movie video type", default="hdrip", - choices=["hdrip", "tcrip", "dubbed", "series"], + choices=["hdrip", "tcrip", "dubbed", "series", "old"], ) parser.add_argument( "-p", "--pages", type=int, default=1, help="number of scrap pages" diff --git a/scrappers/tamilmv.py b/scrappers/tamilmv.py new file mode 100644 index 0000000..19090cf --- /dev/null +++ b/scrappers/tamilmv.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import logging +import math +import random +import re + +from bs4 import BeautifulSoup +from dateutil.parser import parse as dateparser +from playwright.async_api import async_playwright +from playwright_stealth import stealth_async + +from db import database +from scrappers.helpers import ( + get_page_content, + get_scrapper_session, + download_and_save_torrent, +) + +HOMEPAGE = "https://www.1tamilmv.prof" +TAMIL_MV_LINKS = { + "tamil": { + "hdrip": [ + "11-web-hd-itunes-hd-bluray", + "12-hd-rips-dvd-rips-br-rips", + "14-hdtv-sdtv-hdtv-rips", + ], + "tcrip": "10-predvd-dvdscr-cam-tc", + "dubbed": "17-hollywood-movies-in-multi-audios", + "series": "19-web-series-tv-shows", + }, + "malayalam": { + "hdrip": [ + "36-web-hd-itunes-hd-bluray", + "37-hd-rips-dvd-rips-br-rips", + "39-hdtv-sdtv-hdtv-rips", + ], + "tcrip": "35-predvd-dvdscr-cam-tc", + "dubbed": "42-malayalam-dubbed-subtitled-movies", + "series": "44-web-series-tv-shows", + }, + "telugu": { + "tcrip": "23-predvd-dvdscr-cam-tc", + "hdrip": [ + "24-web-hd-itunes-hd-bluray", + "25-hd-rips-dvd-rips-br-rips", + "27-hdtv-sdtv-hdtv-rips", + ], + "dubbed": "31-telugu-dubbed-movies", + "series": "33-web-series-tv-shows", + }, + "hindi": { + "tcrip": "57-predvd-dvdscr-cam-tc", + "hdrip": [ + "58-web-hd-itunes-hd-bluray", + "59-hd-rips-dvd-rips-br-rips", + "61-hdtv-sdtv-hdtv-rips", + ], + "dubbed": "64-hindi-dubbed-movies", + "series": "66-web-series-tv-shows", + }, + "kannada": { + "tcrip": "68-predvd-dvdscr-cam-tc", + "hdrip": [ + "69-web-hd-itunes-hd-bluray", + "70-hd-rips-dvd-rips-br-rips", + "72-hdtv-sdtv-hdtv-rips", + ], + "dubbed": "75-watch-kannada-movies-online", + "series": "77-web-series-tv-shows", + }, + "english": { + "tcrip": "46-predvd-dvdscr-cam-tc", + "hdrip": ["49-web-hd-itunes-hd-bluray", "50-hd-rips-dvd-rips-br-rips"], + "series": "55-web-series-tv-shows", + }, +} + + +async def process_movie( + movie, + scraper=None, + page=None, + keyword=None, + language=None, + media_type=None, + supported_forums=None, +): + if keyword: + movie_link = movie.find("a", {"data-linktype": "link"}) + forum_link = movie.find("a", href=re.compile(r"forums/forum/")).get("href") + forum_id = re.search(r"forums/forum/([^/]+)/", forum_link)[1] + if forum_id not in supported_forums: + logging.error(f"Unsupported forum {forum_id}") + return + # Extracting language and media_type from supported_forums + language = supported_forums[forum_id]["language"] + media_type = supported_forums[forum_id]["media_type"] + else: + movie_link = movie.find("a") + + if not movie_link: + logging.error(f"Movie link not found") + return + + page_link = movie_link.get("href") + + try: + if scraper: # If using the scraper + response = scraper.get(page_link) + movie_page_content = response.content + else: # If using playwright + movie_page_content = await get_page_content(page, page_link) + + movie_page = BeautifulSoup(movie_page_content, "html.parser") + + # Extracting other details + poster_element = movie_page.select_one("div[data-commenttype='forums'] img") + poster = poster_element.get("src") if poster_element else None + + datetime_element = movie_page.select_one("time") + created_at = ( + dateparser(datetime_element.get("datetime")) if datetime_element else None + ) + + # Define metadata + metadata = { + "catalog": f"{language}_{media_type}", + "poster": poster, + "created_at": created_at, + "scrap_language": language.title(), + "source": "TamilMV", + } + + # Extracting torrent details + torrent_elements = movie_page.select("a[data-fileext='torrent']") + + if not torrent_elements: + logging.error(f"No torrents found for {page_link}") + return + + for torrent_element in torrent_elements: + await download_and_save_torrent( + torrent_element, + scraper=scraper, + page=page, + metadata=metadata.copy(), + media_type=media_type, + page_link=page_link, + ) + + return True + except Exception as e: + logging.error( + f"Error processing movie {page_link}: {e}", exc_info=True, stack_info=True + ) + return False + + +async def scrap_page(url, language, media_type, proxy_url=None): + scraper = get_scrapper_session(proxy_url) + response = scraper.get(url) + if response.status_code == 403: + logging.error( + "Cloudflare validation required. Run with --scrap-with-playwright" + ) + return + + response.raise_for_status() + tamil_blasters = BeautifulSoup(response.content, "html.parser") + movies = tamil_blasters.select("li[data-rowid]") + + for movie in movies: + await process_movie( + movie, scraper=scraper, language=language, media_type=media_type + ) + + +async def scrap_page_with_playwright(url, language, media_type, proxy_url=None): + async with async_playwright() as p: + # Launch a new browser session + browser = await p.firefox.launch( + headless=False, + proxy={"server": proxy_url} if proxy_url else None, + ) + page = await browser.new_page() + await stealth_async(page) + await asyncio.sleep(2) + + page_content = await get_page_content(page, url) + tamil_blasters = BeautifulSoup(page_content, "html.parser") + + movies = tamil_blasters.select("li[data-rowid]") + + for movie in movies: + await process_movie( + movie, page=page, language=language, media_type=media_type + ) + + await browser.close() + + +async def get_search_results(scraper, keyword, page_number=1): + search_link = f"{HOMEPAGE}/index.php?/search/&q={keyword}&type=forums_topic&page={page_number}&search_and_or=or&search_in=titles&sortby=relevancy" + # Get page content and initialize BeautifulSoup + response = scraper.get(search_link) + response.raise_for_status() + page_content = response.content + soup = BeautifulSoup(page_content, "html.parser") + + return soup + + +async def scrap_search_keyword(keyword, proxy_url=None): + supported_forums = {} + for language in TAMIL_MV_LINKS: + for video_type in TAMIL_MV_LINKS[language]: + forum_ids = TAMIL_MV_LINKS[language][video_type] + if isinstance(forum_ids, list): + for forum_id in forum_ids: + supported_forums[forum_id] = { + "language": language, + "media_type": video_type, + } + else: + supported_forums[forum_ids] = { + "language": language, + "media_type": video_type, + } + + scraper = get_scrapper_session(proxy_url) + soup = await get_search_results(scraper, keyword) + results_element = soup.find("div", {"data-role": "resultsArea"}) + + results_count = int(re.search(r"\d+", results_element.find("p").text).group()) + logging.info(f"Found {results_count} results for {keyword}") + + movies = results_element.select("li[data-role='activityItem']") + if results_count > 25: + number_of_pages = math.ceil(results_count / 25) + logging.info(f"Found {number_of_pages} pages for {keyword}") + for page_number in range(2, number_of_pages + 1): + soup = await get_search_results(scraper, keyword, page_number) + movies.extend(soup.select("li[data-role='activityItem']")) + await asyncio.sleep(random.randint(2, 5)) + + for movie in movies: + await process_movie( + movie, scraper=scraper, keyword=keyword, supported_forums=supported_forums + ) + + +async def run_scraper( + language: str = None, + video_type: str = None, + pages: int = None, + start_page: int = None, + search_keyword: str = None, + scrap_with_playwright: bool = None, + proxy_url: str = None, +): + await database.init() + if search_keyword: + await scrap_search_keyword(search_keyword, proxy_url) + return + link_prefix = f"{HOMEPAGE}/index.php?/forums/forum/" + try: + forum_ids = TAMIL_MV_LINKS[language][video_type] + scrap_links = ( + [link_prefix + link for link in forum_ids] + if isinstance(forum_ids, list) + else [link_prefix + forum_ids] + ) + except KeyError: + logging.error(f"Unsupported language or video type: {language}_{video_type}") + return + for scrap_link_prefix in scrap_links: + for page in range(start_page, pages + start_page): + scrap_link = f"{scrap_link_prefix}/page/{page}/" + logging.info(f"Scrap page: {scrap_link}") + if scrap_with_playwright is True: + await scrap_page_with_playwright( + scrap_link, language, video_type, proxy_url + ) + else: + await scrap_page(scrap_link, language, video_type, proxy_url) + + logging.info(f"Scrap completed for : {language}_{video_type}") + + +async def run_schedule_scrape( + pages: int = 1, + start_page: int = 1, + scrap_with_playwright: bool = None, + proxy_url: str = None, +): + for language in TAMIL_MV_LINKS: + for video_type in TAMIL_MV_LINKS[language]: + await run_scraper( + language, + video_type, + pages=pages, + start_page=start_page, + scrap_with_playwright=scrap_with_playwright, + proxy_url=proxy_url, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Scrap Movie metadata from TamilMV") + parser.add_argument( + "--all", action="store_true", help="scrap all type of movies & series" + ) + parser.add_argument( + "-l", + "--language", + help="scrap movie language", + default="tamil", + choices=["tamil", "malayalam", "telugu", "hindi", "kannada", "english"], + ) + parser.add_argument( + "-t", + "--video-type", + help="scrap movie video type", + default="hdrip", + choices=["hdrip", "tcrip", "dubbed", "series"], + ) + parser.add_argument( + "-p", "--pages", type=int, default=1, help="number of scrap pages" + ) + parser.add_argument( + "-s", "--start-pages", type=int, default=1, help="page number to start scrap." + ) + parser.add_argument( + "-k", + "--search-keyword", + help="search keyword to scrap movies & series. ex: 'bigg boss'", + default=None, + ) + parser.add_argument( + "--scrap-with-playwright", action="store_true", help="scrap with playwright" + ) + parser.add_argument( + "--proxy-url", + help="proxy url to scrap. ex: socks5://127.0.0.1:1080", + default=None, + ) + args = parser.parse_args() + + logging.basicConfig( + format="%(levelname)s::%(asctime)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", + level=logging.INFO, + ) + if args.all: + asyncio.run( + run_schedule_scrape( + args.pages, args.start_pages, args.scrap_with_playwright, args.proxy_url + ) + ) + else: + asyncio.run( + run_scraper( + args.language, + args.video_type, + args.pages, + args.start_pages, + args.search_keyword, + args.scrap_with_playwright, + args.proxy_url, + ) + ) diff --git a/streaming_providers/debridlink/__init__.py b/streaming_providers/debridlink/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/streaming_providers/debridlink/api.py b/streaming_providers/debridlink/api.py new file mode 100644 index 0000000..2b877f3 --- /dev/null +++ b/streaming_providers/debridlink/api.py @@ -0,0 +1,21 @@ +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from db.schemas import AuthorizeData +from streaming_providers.debridlink.client import DebridLink + +router = APIRouter() +headers = {"Cache-Control": "no-store, no-cache, must-revalidate, max-age=0", "Pragma": "no-cache", "Expires": "0"} + + +@router.get("/get-device-code") +async def get_device_code(): + dl_client = DebridLink() + return JSONResponse(content=dl_client.get_device_code(), headers=headers) + + +@router.post("/authorize") +async def authorize(data: AuthorizeData): + dl_client = DebridLink() + response = dl_client.authorize(data.device_code) + return JSONResponse(content=response, headers=headers) diff --git a/streaming_providers/debridlink/client.py b/streaming_providers/debridlink/client.py new file mode 100644 index 0000000..8e2ca9a --- /dev/null +++ b/streaming_providers/debridlink/client.py @@ -0,0 +1,183 @@ +import PTN +import traceback + +from requests import RequestException, JSONDecodeError +from typing import Any + +import requests +from base64 import b64encode, b64decode + +from streaming_providers.exceptions import ProviderException + + +class DebridLink: + BASE_URL = "https://debrid-link.com/api/v2" + OAUTH_URL = "https://debrid-link.com/api/oauth" + OPENSOURCE_CLIENT_ID = "RyrV22FOg30DsxjYPziRKA" + + def __init__(self, encoded_token=None): + self.encoded_token = encoded_token + self.headers = {} + self.initialize_headers() + + def __del__(self): + if self.encoded_token: + self.disable_access_token() + + def _make_request( + self, + method: str, + url: str, + data=None, + params=None, + is_return_none=False, + is_expected_to_fail=False, + ) -> dict: + if method == "GET": + response = requests.get(url, params=params, headers=self.headers) + elif method == "POST": + response = requests.post(url, data=data, headers=self.headers) + elif method == "DELETE": + response = requests.delete(url, headers=self.headers) + else: + raise ValueError(f"Unsupported method: {method}") + + try: + response.raise_for_status() + except RequestException as error: + if is_expected_to_fail: + pass + elif error.response.status_code == 401: + raise ProviderException("Invalid token", "invalid_token.mp4") + elif ( + error.response.status_code == 400 + and response.json().get("error") == "freeServerOverload" + ): + raise ProviderException( + "Debrid-Link free servers are overloaded", "need_premium.mp4" + ) + else: + formatted_traceback = "".join(traceback.format_exception(error)) + raise ProviderException( + f"status code: {error.response.status_code}, data: {error.response.content}, trace log:\n {formatted_traceback}", + "api_error.mp4", + ) + + if is_return_none: + return {} + try: + return response.json() + except JSONDecodeError as error: + formatted_traceback = "".join(traceback.format_exception(error)) + raise ProviderException( + f"Failed to parse response. content: {response.text}, trace log:\n {formatted_traceback}", + "api_error.mp4", + ) + + def initialize_headers(self): + if self.encoded_token: + token_data = self.decode_token_str(self.encoded_token) + access_token_data = self.refresh_token( + token_data["client_id"], token_data["code"] + ) + self.headers = { + "Authorization": f"Bearer {access_token_data['access_token']}" + } + + @staticmethod + def encode_token_data(client_id: str, code: str): + token = f"{client_id}:{code}" + return b64encode(str(token).encode()).decode() + + @staticmethod + def decode_token_str(token: str) -> dict[str, str]: + try: + client_id, code = b64decode(token).decode().split(":") + except ValueError: + raise ProviderException("Invalid token", "invalid_token.mp4") + return {"client_id": client_id, "code": code} + + def get_device_code(self): + return self._make_request( + "POST", + f"{self.OAUTH_URL}/device/code", + data={ + "client_id": self.OPENSOURCE_CLIENT_ID, + "scope": "get.post.downloader get.post.seedbox get.account get.files get.post.stream", + }, + ) + + def get_token(self, client_id, device_code): + return self._make_request( + "POST", + f"{self.OAUTH_URL}/token", + data={ + "client_id": client_id, + "code": device_code, + "grant_type": "http://oauth.net/grant_type/device/1.0", + }, + is_expected_to_fail=True, + ) + + def refresh_token(self, client_id, refresh_token): + return self._make_request( + "POST", + f"{self.OAUTH_URL}/token", + data={ + "client_id": client_id, + "refresh_token": refresh_token, + "grant_type": "refresh_token", + }, + ) + + def authorize(self, device_code): + token_data = self.get_token(self.OPENSOURCE_CLIENT_ID, device_code) + + if "error" in token_data: + return token_data + + if "access_token" in token_data: + token = self.encode_token_data( + self.OPENSOURCE_CLIENT_ID, token_data["refresh_token"] + ) + return {"token": token} + else: + return token_data + + def add_magent_link(self, magnet_link): + return self._make_request( + "POST", f"{self.BASE_URL}/seedbox/add", data={"url": magnet_link} + ) + + def get_user_torrent_list(self): + return self._make_request("GET", f"{self.BASE_URL}/seedbox/list") + + def get_torrent_info(self, torrent_id): + return self._make_request( + "GET", f"{self.BASE_URL}/seedbox/list", data={"url": torrent_id} + ) + + def get_torrent_files_list(self, torrent_id): + return self._make_request("GET", f"{self.BASE_URL}/files/{torrent_id}/list") + + def get_torrent_instant_availability(self, torrent_hash): + return self._make_request( + "GET", f"{self.BASE_URL}/seedbox/cached/", data={"url": torrent_hash} + ) + + def disable_access_token(self): + return self._make_request( + "GET", f"{self.OAUTH_URL}/revoke", is_return_none=True + ) + + def get_available_torrent(self, info_hash: str) -> dict[str, Any] | None: + torrent_list_response = self.get_user_torrent_list() + if "error" in torrent_list_response: + raise ProviderException( + "Failed to get torrent info from Debrid-Link", "transfer_error.mp4" + ) + + available_torrents = torrent_list_response["value"] + for torrent in available_torrents: + if torrent["hashString"] == info_hash: + return torrent diff --git a/streaming_providers/debridlink/utils.py b/streaming_providers/debridlink/utils.py new file mode 100644 index 0000000..a881074 --- /dev/null +++ b/streaming_providers/debridlink/utils.py @@ -0,0 +1,103 @@ +import time +from typing import Any + +from db.models import Streams, Episode +from db.schemas import UserData +from streaming_providers.exceptions import ProviderException +from streaming_providers.debridlink.client import DebridLink + +import PTN + + +def get_direct_link_from_debridlink( + info_hash: str, + magnet_link: str, + user_data: UserData, + stream: Streams, + episode_data: Episode = None, + max_retries=5, + retry_interval=5, +) -> str: + dl_client = DebridLink(encoded_token=user_data.streaming_provider.token) + filename = episode_data.filename if episode_data else stream.filename + + # Check if the torrent already exists + downloadUrl = check_existing_torrent(dl_client, info_hash, episode_data, max_retries, retry_interval) + if downloadUrl: + return downloadUrl + + # If torrent doesn't exist, add it + response_data = dl_client.add_magent_link(magnet_link) + if "error" in response_data: + raise ProviderException("Failed to add magnet link to Debrid-Link", "transfer_error.mp4") + + torrent_id = response_data["value"]["id"] + + return wait_for_torrent_download(dl_client, torrent_id, filename, max_retries, retry_interval) + + +def check_existing_torrent( + dl_client: DebridLink, info_hash: str, episode_data: Episode | None, max_retries: int, retry_interval: int +) -> str: + """Check if the torrent is already in torrent list and return the direct link if available.""" + retries = 0 + + torrent_info = dl_client.get_available_torrent(info_hash) + if not torrent_info: + return None + + torrent_id = torrent_info.get("id") + while retries < max_retries: + torrent_info_response = dl_client.get_torrent_info(torrent_id) + if not torrent_info_response["success"] and not torrent_info_response["value"]: + raise ProviderException("Failed to get torrent info from Debrid-Link", "transfer_error.mp4") + + torrent_info = torrent_info_response["value"][0] + if torrent_info["downloadPercent"] == 100: + return get_direct_link(torrent_info, episode_data) + + time.sleep(retry_interval) + retries += 1 + raise ProviderException("Torrent not downloaded yet.", "torrent_not_downloaded.mp4") + + +def wait_for_torrent_download(dl_client, torrent_id: str, episode_data: Episode | None, max_retries: int, retry_interval: int) -> str: + """Wait for the torrent to be downloaded and return the direct link.""" + retries = 0 + while retries < max_retries: + torrent_info_response = dl_client.get_torrent_info(torrent_id) + + if not torrent_info_response["success"] and not torrent_info_response["value"]: + raise ProviderException("Failed to get torrent info from Debrid-Link", "transfer_error.mp4") + + torrent_info = torrent_info_response["value"][0] + if not torrent_info["files"]: + raise ProviderException("No files available for this torrent", "transfer_error.mp4") + + if torrent_info["downloadPercent"] == 100: + return get_direct_link(torrent_info, episode_data) + + time.sleep(retry_interval) + retries += 1 + raise ProviderException("Torrent not downloaded yet.", "torrent_not_downloaded.mp4") + + +def get_direct_link(torrent_info, episode_data: Episode | None) -> str: + if episode_data: + selected_file = select_episode_file(torrent_info["files"], episode_data.episode_number, "name") + else: + # Otherwise, select the largest file for download + selected_file = max(torrent_info["files"], key=lambda x: x["size"]) + return selected_file["downloadUrl"] + + +def select_episode_file(torrent_files: list, episode: int, file_name_key: str) -> dict: + """Select the file with the specified episode number.""" + + for file in torrent_files: + torrent_data = PTN.parse(file[file_name_key]) + file_episode = torrent_data.get("episode") + if file_episode and int(file_episode) == episode: + return file + else: + raise ProviderException(f"Episode {episode} not found in this torrent", "episode_not_found.mp4") \ No newline at end of file diff --git a/streaming_providers/realdebrid/client.py b/streaming_providers/realdebrid/client.py index fd11a0c..1d43a95 100644 --- a/streaming_providers/realdebrid/client.py +++ b/streaming_providers/realdebrid/client.py @@ -47,6 +47,13 @@ class RealDebrid: pass elif error.response.status_code == 401: raise ProviderException("Invalid token", "invalid_token.mp4") + elif ( + error.response.status_code == 403 + and response.json().get("error_code") == 9 + ): + raise ProviderException( + "Real-Debrid Permission denied for free account", "need_premium.mp4" + ) else: formatted_traceback = "".join(traceback.format_exception(error)) raise ProviderException( diff --git a/utils/const.py b/utils/const.py index f3ff203..4f7e3a5 100644 --- a/utils/const.py +++ b/utils/const.py @@ -3,22 +3,27 @@ CATALOG_ID_DATA = [ "mediafusion_search_series", "tamil_hdrip", "tamil_tcrip", + "tamil_old", "tamil_dubbed", "tamil_series", "malayalam_tcrip", "malayalam_hdrip", + "malayalam_old", "malayalam_dubbed", "malayalam_series", "telugu_tcrip", "telugu_hdrip", + "telugu_old", "telugu_dubbed", "telugu_series", "hindi_tcrip", "hindi_hdrip", + "hindi_old", "hindi_dubbed", "hindi_series", "kannada_tcrip", "kannada_hdrip", + "kannada_old", "kannada_dubbed", "kannada_series", "english_hdrip", @@ -31,22 +36,27 @@ CATALOG_NAME_DATA = [ "MediaFusion Search Series", "Tamil HD Movies", "Tamil TCRip/DVDScr/HDCAM Movies", + "Tamil Old Movies", "Tamil Dubbed Movies", "Tamil Series", "Malayalam TCRip/DVDScr/HDCAM Movies", "Malayalam HD Movies", + "Malayalam Old Movies", "Malayalam Dubbed Movies", "Malayalam Series", "Telugu TCRip/DVDScr/HDCAM Movies", "Telugu HD Movies", + "Telugu Old Movies", "Telugu Dubbed Movies", "Telugu Series", "Hindi TCRip/DVDScr/HDCAM Movies", "Hindi HD Movies", + "Hindi Old Movies", "Hindi Dubbed Movies", "Hindi Series", "Kannada TCRip/DVDScr/HDCAM Movies", "Kannada HD Movies", + "Kannada Old Movies", "Kannada Dubbed Movies", "Kannada Series", "English HD Movies", diff --git a/utils/exception_video.py b/utils/exception_video.py index 573fa8d..573d299 100644 --- a/utils/exception_video.py +++ b/utils/exception_video.py @@ -1,11 +1,11 @@ import os -def create_text_video(output_path, text, duration=30, resolution=(1280, 720), fontsize=45, bgcolor="black"): +def create_text_video( + output_path, text, duration=30, resolution=(1280, 720), fontsize=45, bgcolor="black" +): # Create a solid color background using FFmpeg - background_command = ( - f"ffmpeg -f lavfi -i color=c={bgcolor}:s={resolution[0]}x{resolution[1]} -t {duration} -y background.mp4" - ) + background_command = f"ffmpeg -f lavfi -i color=c={bgcolor}:s={resolution[0]}x{resolution[1]} -t {duration} -y background.mp4" os.system(background_command) # Overlay the text with shadow and fade-in and fade-out effects diff --git a/utils/parser.py b/utils/parser.py index 48cf425..e4573f3 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -1,6 +1,9 @@ import math import re +import requests +from imdb import Cinemagoer, IMDbDataAccessError + from db.config import settings from db.models import Streams from db.schemas import Stream, UserData @@ -8,6 +11,8 @@ from streaming_providers.realdebrid.utils import ( order_streams_by_instant_availability_and_date, ) +ia = Cinemagoer() + def parse_stream_data( streams: list[Streams], @@ -120,3 +125,27 @@ def get_catalogs(catalog: str, languages: list[str]) -> list[str]: # Generate the catalog for each supported language return [f"{lang.lower()}_{base_catalog}" for lang in languages] + + +def search_imdb(title: str, year: int, retry: int = 5) -> dict: + try: + result = ia.search_movie(f"{title} {year}") + except IMDbDataAccessError: + return search_imdb(title, year, retry - 1) if retry > 0 else {} + for movie in result: + if movie.get("year") == year and movie.get("title").lower() in title.lower(): + imdb_id = f"tt{movie.movieID}" + poster = f"https://live.metahub.space/poster/small/{imdb_id}/img" + if requests.get(poster).status_code == 200: + return { + "imdb_id": imdb_id, + "poster": poster.replace("small", "medium"), + "background": f"https://live.metahub.space/background/medium/{imdb_id}/img", + } + poster = movie.get("full-size cover url") + return { + "imdb_id": imdb_id, + "poster": poster, + "background": poster, + } + return {} diff --git a/utils/poster.py b/utils/poster.py new file mode 100644 index 0000000..ce46988 --- /dev/null +++ b/utils/poster.py @@ -0,0 +1,76 @@ +from io import BytesIO + +from PIL import Image, ImageDraw, ImageFont +from imdb import Cinemagoer +import requests + +from db.models import MediaFusionMetaData + +ia = Cinemagoer() + + +async def create_poster(mediafusion_data: MediaFusionMetaData) -> BytesIO: + response = requests.get(mediafusion_data.poster, timeout=10) + + response.raise_for_status() + imdb_rating = None + if mediafusion_data.id.startswith("tt"): + result = ia.get_movie(mediafusion_data.id[2:], info="main") + imdb_rating = result.get("rating") + + image = Image.open(BytesIO(response.content)) + image = image.resize((300, 450)) + image = add_elements_to_poster(image, imdb_rating) + image = image.convert("RGB") + + byte_io = BytesIO() + image.save(byte_io, "JPEG") + byte_io.seek(0) + + return byte_io + + +def add_elements_to_poster( + image: Image.Image, imdb_rating: float = None +) -> Image.Image: + draw = ImageDraw.Draw(image) + margin = 10 + + # Adding IMDb rating at the bottom left with a semi-transparent background + if imdb_rating: + font = ImageFont.truetype("Arial.ttf", size=24) + imdb_text = f"IMDb: {imdb_rating}/10" + + # Calculate text bounding box using the draw instance + left, top, right, bottom = draw.textbbox((0, 0), imdb_text, font=font) + text_width = right - left + text_height = bottom - top + + # Draw a semi-transparent rectangle behind the text for better visibility + rectangle_x0 = margin + rectangle_y0 = image.height - text_height - margin - 5 # 5 for a little padding + rectangle_x1 = rectangle_x0 + text_width + 10 # 10 for padding + rectangle_y1 = image.height - margin + + draw.rectangle( + (rectangle_x0, rectangle_y0, rectangle_x1, rectangle_y1), + fill=(0, 0, 0, 128), + ) + draw.text( + (rectangle_x0 + 5, rectangle_y0), imdb_text, font=font, fill="#F5C518" + ) # 5 for padding + + # Add MediaFusion watermark at the top right + watermark = Image.open("resources/images/logo_text.png") + + # Resizing the watermark to fit the new poster size + aspect_ratio = watermark.width / watermark.height + new_width = int(image.width * 0.5) # Reduced size for better aesthetics + new_height = int(new_width / aspect_ratio) + watermark = watermark.resize((new_width, new_height)) + + # Position watermark at top right + watermark_position = (image.width - watermark.width - margin, margin) + image.paste(watermark, watermark_position, watermark) + + return image