Initial comments support

Random User Agent generator
Merge pull request #131 from chrisnovakovic/video-title
2024-12-06 19:27:30 +01:00 · 2023-03-06 21:20:14 +01:00 · 2023-03-05 13:18:34 +01:00 · 2023-02-28 11:40:30 +00:00 · 2023-02-27 23:11:36 +00:00 · 2023-02-19 11:08:22 +00:00
19 changed files with 228 additions and 112 deletions
@@ -1,5 +1,6 @@
 node_modules
 /.env
+/.git
 /.vscode
 /vendor
 /cache/latte/*
@@ -20,3 +20,5 @@
 # PROXY_PORT=8080
 # PROXY_USERNAME=username
 # PROXY_PASSWORD=password
+
+# USER_AGENT="Mozilla/5.0 (Android 12; Mobile; rv:109.0) Gecko/109.0 Firefox/109.0"
@@ -0,0 +1,25 @@
+<?php
+namespace App\Cache;
+
+use TikScraper\Interfaces\CacheInterface;
+
+class ApcuCache implements CacheInterface {
+    function __construct() {
+        if (!(extension_loaded('apcu') && apcu_enabled())) {
+            throw new \Exception('APCu not enabled');
+        }
+    }
+
+    public function get(string $cache_key): ?object {
+        $data = apcu_fetch($cache_key);
+        return $data !== false ? json_decode($data) : null;
+    }
+
+    public function exists(string $cache_key): bool {
+        return apcu_exists($cache_key);
+    }
+
+    public function set(string $cache_key, string $data, $timeout = 3600): void {
+        apcu_store($cache_key, $data, $timeout);
+    }
+}
@@ -3,5 +3,6 @@ namespace App\Constants;

 abstract class CacheMethods {
    const JSON = 'json';
+    const APCU = 'apcu';
    const REDIS = 'redis';
 }
@@ -3,16 +3,16 @@ namespace App\Controllers;

 use App\Helpers\ErrorHandler;
 use App\Helpers\Wrappers;
-use App\Models\FeedTemplate;
+use App\Models\DiscoverTemplate;

 class DiscoverController {
    static public function get() {
        $api = Wrappers::api();
-        $feed = $api->discover();
-        if ($feed->meta->success) {
-            Wrappers::latte('discover', new FeedTemplate('Discover', $feed));
+        $data = $api->discover();
+        if ($data->meta->success) {
+            Wrappers::latte('discover', new DiscoverTemplate($data));
        } else {
-            ErrorHandler::showMeta($feed->meta);
+            ErrorHandler::showMeta($data->meta);
        }
    }
 }
@@ -2,45 +2,19 @@
 namespace App\Controllers;

 use App\Helpers\Cookies;
+use App\Helpers\Misc;
+use TikScraper\Constants\UserAgents as TikScraperUserAgents;

 class ProxyController {
    const VALID_TIKTOK_DOMAINS = [
        "tiktokcdn.com", "tiktokcdn-us.com", "tiktok.com"
    ];

-    static private function isValidDomain(string $url) {
-        $host = parse_url($url, PHP_URL_HOST);
-        $host_split = explode('.', $host);
-        $host_count = count($host_split);
-        if ($host_count === 2) {
-            // Using no watermark
-            return in_array($host_split[0] . '.' . $host_split[1], self::VALID_TIKTOK_DOMAINS);
-        } elseif ($host_count === 3) {
-            return in_array($host_split[1] . '.' . $host_split[2], self::VALID_TIKTOK_DOMAINS);
-        }
-        return false;
-    }
-
-    static private function checkUrl() {
-        if (!isset($_GET['url'])) {
-            die('You need to send a URL');
-        }
-
-        if (!filter_var($_GET['url'], FILTER_VALIDATE_URL) || !self::isValidDomain($_GET['url'])) {
-            die('Not a valid URL');
-        }
-
-    }
-
-    static private function getFilename(string $id, string $user): string {
-        $filename = 'tiktok-video-' . $id . '-' . $user;
-        return $filename;
-    }
-
    static public function stream() {
        self::checkUrl();
        $url = $_GET['url'];
-        $streamer = new \TikScraper\Stream();
+        $config['user_agent'] = Misc::env("USER_AGENT", TikScraperUserAgents::DEFAULT);
+        $streamer = new \TikScraper\Stream($config);
        $streamer->url($url);
    }

@@ -59,4 +33,34 @@ class ProxyController {
        // Running
        $downloader->url($url, $filename, $watermark);
    }
+
+    static private function isValidDomain(string $url): bool {
+        $valid = false;
+        $host = parse_url($url, PHP_URL_HOST);
+        $host_split = explode('.', $host);
+        $host_count = count($host_split);
+        if ($host_count === 2) {
+            // Using no watermark
+            $valid = in_array($host_split[0] . '.' . $host_split[1], self::VALID_TIKTOK_DOMAINS);
+        } elseif ($host_count === 3) {
+            $valid = in_array($host_split[1] . '.' . $host_split[2], self::VALID_TIKTOK_DOMAINS);
+        }
+        return $valid;
+    }
+
+    static private function checkUrl(): void {
+        if (!isset($_GET['url'])) {
+            die('You need to send a URL');
+        }
+
+        if (!filter_var($_GET['url'], FILTER_VALIDATE_URL) || !self::isValidDomain($_GET['url'])) {
+            die('Not a valid URL');
+        }
+
+    }
+
+    static private function getFilename(string $id, string $user): string {
+        $filename = 'tiktok-video-' . $id . '-' . $user;
+        return $filename;
+    }
 }
@@ -18,10 +18,6 @@ class UserController {
        if ($user->ok()) {
            $info = $user->getInfo();
            $feed = $user->getFeed();
-            if ($info->detail->privateAccount) {
-                ErrorHandler::showText(401, "Private account detected! Not supported");
-                return;
-            }
            Wrappers::latte('user', new FullTemplate($info->detail->nickname, $info, $feed));
        } else {
            ErrorHandler::showMeta($user->error());
@@ -1,11 +1,14 @@
 <?php
 namespace App\Helpers;

+use App\Cache\ApcuCache;
 use App\Cache\JSONCache;
 use App\Cache\RedisCache;
 use App\Constants\CacheMethods;
 use App\Models\BaseTemplate;

+use TikScraper\Constants\UserAgents as TikScraperUserAgents;
+
 class Wrappers {
    /**
     * Setup of Latte template engine
@@ -138,6 +141,9 @@ class Wrappers {
                case CacheMethods::JSON:
                    $cacheEngine = new JSONCache();
                    break;
+                case CacheMethods::APCU:
+                    $cacheEngine = new ApcuCache();
+                    break;
                case CacheMethods::REDIS:
                    if (!(isset($_ENV['REDIS_URL']) || isset($_ENV['REDIS_HOST'], $_ENV['REDIS_PORT']))) {
                        throw new \Exception('You need to set REDIS_URL or REDIS_HOST and REDIS_PORT to use Redis Cache!');
@@ -158,6 +164,12 @@ class Wrappers {
            }
        }

+        $customUa = Misc::env("USER_AGENT", '');
+
+        if ($customUa) {
+            $options['user_agent'] = $customUa;
+        }
+
        return new \TikScraper\Api($options, $cacheEngine);
    }
 }
@@ -0,0 +1,13 @@
+<?php
+namespace App\Models;
+
+use TikScraper\Models\Discover;
+
+class DiscoverTemplate extends BaseTemplate {
+    public Discover $data;
+
+    function __construct(Discover $data) {
+        parent::__construct("Discover");
+        $this->data = $data;
+    }
+}
@@ -1,16 +1,16 @@
 <?php
 namespace App\Models;

+use TikScraper\Models\Feed;
+
 /**
 * Base for templates with a feed
 */
 class FeedTemplate extends BaseTemplate {
-    public object $data;
+    public Feed $feed;

-    function __construct(string $title, object $feed) {
+    function __construct(string $title, Feed $feed) {
        parent::__construct($title);
-        $this->data = (object) [
-            'feed' => $feed
-        ];
+        $this->feed = $feed;
    }
 }
@@ -1,7 +1,7 @@
 {
    "name": "pablouser1/proxitok",
    "description": "An alternative frontend for TikTok",
-    "version": "2.4.4.0",
+    "version": "2.4.7.0",
    "license": "AGPL-3.0-or-later",
    "type": "project",
    "authors": [
@@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "f9cec7d0ed07b8f1d6db885ceefd3226",
+    "content-hash": "97152c7473ad08a262f9db1af867faaa",
    "packages": [
        {
            "name": "bramus/router",
@@ -57,6 +57,46 @@
            },
            "time": "2021-11-18T19:24:07+00:00"
        },
+        {
+            "name": "deemon47/user-agent-generator",
+            "version": "v1.0.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/Deemon47/user-agent-generator.git",
+                "reference": "5af13fd2e905c5441eb4d61aad3a71bd1e801709"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/Deemon47/user-agent-generator/zipball/5af13fd2e905c5441eb4d61aad3a71bd1e801709",
+                "reference": "5af13fd2e905c5441eb4d61aad3a71bd1e801709",
+                "shasum": ""
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Deemon47\\": "./"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "GPL-3.0-only"
+            ],
+            "authors": [
+                {
+                    "name": "phpfail",
+                    "email": "phpfail@gihub.com"
+                },
+                {
+                    "name": "Dee",
+                    "email": "deemon47@gmail.com"
+                }
+            ],
+            "description": "Generate random User-Agent http header with parameters",
+            "support": {
+                "source": "https://github.com/Deemon47/user-agent-generator/tree/v1.0.1"
+            },
+            "time": "2020-04-11T12:05:34+00:00"
+        },
        {
            "name": "josegonzalez/dotenv",
            "version": "dev-master",
@@ -263,19 +303,20 @@
        },
        {
            "name": "pablouser1/tikscraper",
-            "version": "v2.3.3.2",
+            "version": "v2.3.6.0",
            "source": {
                "type": "git",
                "url": "https://github.com/pablouser1/TikScraperPHP.git",
-                "reference": "dff2feebe7973d576f4ba097e624fdecd9f7d337"
+                "reference": "23eff4120eecfb01ed27e049dc36ed14a967cef8"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/pablouser1/TikScraperPHP/zipball/dff2feebe7973d576f4ba097e624fdecd9f7d337",
-                "reference": "dff2feebe7973d576f4ba097e624fdecd9f7d337",
+                "url": "https://api.github.com/repos/pablouser1/TikScraperPHP/zipball/23eff4120eecfb01ed27e049dc36ed14a967cef8",
+                "reference": "23eff4120eecfb01ed27e049dc36ed14a967cef8",
                "shasum": ""
            },
            "require": {
+                "deemon47/user-agent-generator": "^1.0",
                "ext-curl": "*",
                "ext-json": "*",
                "ext-xml": "*",
@@ -305,43 +346,44 @@
            "description": "Get data from TikTok API",
            "support": {
                "issues": "https://github.com/pablouser1/TikScraperPHP/issues",
-                "source": "https://github.com/pablouser1/TikScraperPHP/tree/v2.3.3.2"
+                "source": "https://github.com/pablouser1/TikScraperPHP/tree/v2.3.6.0"
            },
-            "time": "2023-01-25T12:58:06+00:00"
+            "time": "2023-03-06T20:19:11+00:00"
        },
        {
            "name": "php-webdriver/webdriver",
-            "version": "1.13.1",
+            "version": "1.14.0",
            "source": {
                "type": "git",
                "url": "https://github.com/php-webdriver/php-webdriver.git",
-                "reference": "6dfe5f814b796c1b5748850aa19f781b9274c36c"
+                "reference": "3ea4f924afb43056bf9c630509e657d951608563"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/php-webdriver/php-webdriver/zipball/6dfe5f814b796c1b5748850aa19f781b9274c36c",
-                "reference": "6dfe5f814b796c1b5748850aa19f781b9274c36c",
+                "url": "https://api.github.com/repos/php-webdriver/php-webdriver/zipball/3ea4f924afb43056bf9c630509e657d951608563",
+                "reference": "3ea4f924afb43056bf9c630509e657d951608563",
                "shasum": ""
            },
            "require": {
                "ext-curl": "*",
                "ext-json": "*",
                "ext-zip": "*",
-                "php": "^5.6 || ~7.0 || ^8.0",
+                "php": "^7.3 || ^8.0",
                "symfony/polyfill-mbstring": "^1.12",
-                "symfony/process": "^2.8 || ^3.1 || ^4.0 || ^5.0 || ^6.0"
+                "symfony/process": "^5.0 || ^6.0"
            },
            "replace": {
                "facebook/webdriver": "*"
            },
            "require-dev": {
-                "ondram/ci-detector": "^2.1 || ^3.5 || ^4.0",
+                "ergebnis/composer-normalize": "^2.20.0",
+                "ondram/ci-detector": "^4.0",
                "php-coveralls/php-coveralls": "^2.4",
-                "php-mock/php-mock-phpunit": "^1.1 || ^2.0",
+                "php-mock/php-mock-phpunit": "^2.0",
                "php-parallel-lint/php-parallel-lint": "^1.2",
-                "phpunit/phpunit": "^5.7 || ^7 || ^8 || ^9",
+                "phpunit/phpunit": "^9.3",
                "squizlabs/php_codesniffer": "^3.5",
-                "symfony/var-dumper": "^3.3 || ^4.0 || ^5.0 || ^6.0"
+                "symfony/var-dumper": "^5.0 || ^6.0"
            },
            "suggest": {
                "ext-SimpleXML": "For Firefox profile creation"
@@ -370,9 +412,9 @@
            ],
            "support": {
                "issues": "https://github.com/php-webdriver/php-webdriver/issues",
-                "source": "https://github.com/php-webdriver/php-webdriver/tree/1.13.1"
+                "source": "https://github.com/php-webdriver/php-webdriver/tree/1.14.0"
            },
-            "time": "2022-10-11T11:49:44+00:00"
+            "time": "2023-02-09T12:12:19+00:00"
        },
        {
            "name": "sapistudio/seleniumstealth",
@@ -572,16 +614,16 @@
        },
        {
            "name": "symfony/process",
-            "version": "v5.4.19",
+            "version": "v5.4.21",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/process.git",
-                "reference": "c5ba874c9b636dbccf761e22ce750e88ec3f55e1"
+                "reference": "d4ce417ebcb0b7d090b4c178ed6d3accc518e8bd"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/process/zipball/c5ba874c9b636dbccf761e22ce750e88ec3f55e1",
-                "reference": "c5ba874c9b636dbccf761e22ce750e88ec3f55e1",
+                "url": "https://api.github.com/repos/symfony/process/zipball/d4ce417ebcb0b7d090b4c178ed6d3accc518e8bd",
+                "reference": "d4ce417ebcb0b7d090b4c178ed6d3accc518e8bd",
                "shasum": ""
            },
            "require": {
@@ -614,7 +656,7 @@
            "description": "Executes commands in sub-processes",
            "homepage": "https://symfony.com",
            "support": {
-                "source": "https://github.com/symfony/process/tree/v5.4.19"
+                "source": "https://github.com/symfony/process/tree/v5.4.21"
            },
            "funding": [
                {
@@ -630,7 +672,7 @@
                    "type": "tidelift"
                }
            ],
-            "time": "2023-01-01T08:32:19+00:00"
+            "time": "2023-02-21T19:46:44+00:00"
        }
    ],
    "packages-dev": [],
@@ -1,4 +1,4 @@
-<div n:ifset="$info" class="buttons">
+<div class="buttons">
 	{* is_numeric is used to avoid having a back button with ttwid cursors *}
 	{if isset($_GET['cursor']) && is_numeric($_GET['cursor']) && $_GET['cursor'] != 0}
    <a class="button is-danger" href="?cursor=0">First</a>
@@ -1,22 +1,8 @@
-{embed '../../form.latte', path: '/redirect/download', method: 'GET'}
-	{block fields}
-		<div class="field has-addons has-addons-centered">
-			<div class="control">
-				<div class="select">
-					<select name="watermark">
-						<option value="yes">WM</option>
-						<option value="no">No WM</option>
-					</select>
-				</div>
-			</div>
-			<div class="control">
-				<button type="submit" class="button is-success">
-					{include '../../icon.latte', icon: 'software-download', text: 'Download'}
-				</a>
-			</div>
-		</div>
-		<input type="hidden" name="playAddr" value="{$playAddr}" />
-		<input type="hidden" name="videoId" value="{$id}" />
-		<input type="hidden" name="authorUsername" value="{$uniqueId}" />
-	{/block}
-{/embed}
+<div class="field is-grouped is-grouped-centered">
+  <p class="control">
+		<a target="_blank" href="{url_download($playAddr, $uniqueId, $id, true)}" class="button is-info">Watermark</a>
+  </p>
+  <p class="control">
+		<a target="_blank" href="{url_download(url_video_external($uniqueId, $id), $uniqueId, $id, false)}" class="button is-success">No watermark</a>
+  </p>
+</div>
@@ -1,14 +1,12 @@
-<nav class="breadcrumb is-centered" aria-label="breadcrumbs">
-  <ul>
-    <li>
-			<a class="has-text-success" href="{url_video_internal($uniqueId, $id)}">
-				{include '../../icon.latte', icon: 'lock', text: 'Instance link'}
-			</a>
-		</li>
-    <li>
-			<a class="has-text-warning" href="{url_video_external($uniqueId, $id)}">
-				{include '../../icon.latte', icon: 'lock-unlock', text: 'Original Link'}
-			</a>
-		</li>
-  </ul>
-</nav>
+<div class="field is-grouped is-grouped-centered">
+  <p class="control">
+		<a class="button is-success" href="{url_video_internal($uniqueId, $id)}">
+			{include '../../icon.latte', icon: 'lock', text: 'Instance'}
+		</a>
+  </p>
+  <p class="control">
+		<a class="button is-warning" href="{url_video_external($uniqueId, $id)}">
+			{include '../../icon.latte', icon: 'lock-unlock', text: 'Original'}
+		</a>
+  </p>
+</div>
@@ -24,10 +24,10 @@
 					<source src="{url_stream($item->video->playAddr)}" type="video/mp4" />
 				</video>
 			</div>
-			<div class="has-text-centered">
-				{include './common/download.latte', playAddr: $item->video->playAddr, id: $item->id, uniqueId: $item->author->uniqueId}
-				<div class="mt-2">{include './common/share.latte', uniqueId: $item->author->uniqueId, id: $item->id}</div>
-			</div>
+			<p class="has-text-centered"><b>Download video</b></p>
+			{include './common/download.latte', playAddr: $item->video->playAddr, id: $item->id, uniqueId: $item->author->uniqueId}
+			<p class="has-text-centered"><b>Share link</b></p>
+			{include './common/share.latte', uniqueId: $item->author->uniqueId, id: $item->id}
  	</div>
 	</article>
 	{/foreach}
@@ -5,7 +5,7 @@
 {/block}

 {block content}
-	{foreach $data->feed->items as $type => $items}
+	{foreach $data->items as $type => $items}
 		<p class="title">{$type|firstUpper}</p>
 		<div class="columns is-multiline is-vcentered">
 			{foreach $items as $item}
@@ -12,6 +12,10 @@
 	</figure>
 	<p class="title">{$info->detail->uniqueId}</p>
 	<p class="subtitle">{include '../components/rss.latte'}</p>
+	{if $info->detail->privateAccount}
+	<p><strong>Private account</strong></p>
+	{/if}
+	<p></p>
 	<p>{$info->detail->signature}</p>
 	<p>Following: {number($info->stats->followingCount)} / Followers: {number($info->stats->followerCount)}</p>
 	<p>Hearts: {number($info->stats->heartCount)} / Videos: {$info->stats->videoCount}</p>
@@ -4,6 +4,10 @@
 {var $og_content = $item->video->originCover}
 {var $og_url = url_video_internal($info->detail->uniqueId, $item->id)}

+{if !empty($item->desc)}
+	{var $title = trim($item->desc)}
+{/if}
+
 {block content}
 	<div class="columns is-centered is-vcentered is-gapless">
 		<div class="column has-text-centered">
@@ -41,6 +45,34 @@
  				</div>
 				</article>
 			</div>
+			{if !empty($item->comments)}
+			<div class="box">
+				<p class="is-size-5 has-text-centered">Comments</p>
+				{foreach $item->comments as $comment}
+				<article class="media">
+				  <figure class="media-left">
+				    <p class="image is-64x64">
+				      <img src="{url_stream($comment->user->avatar_thumb->url_list[0])}" />
+				    </p>
+				  </figure>
+				  <div class="media-content">
+				    <div class="content">
+				      <p>
+				        <strong>{$comment->user->unique_id}</strong>
+								<small>
+									<a href="{url_user($comment->user->unique_id)}">@{$comment->user->nickname}</a>
+								</small>
+								<small title="{date('M d, Y H:i:s e', $comment->create_time)}">{date('M d, Y', $comment->create_time)}</small>
+				        <br>
+								{$comment->text}
+				      </p>
+				    </div>
+						<p>{include '../components/icon.latte', icon: 'heart', text: number($comment->digg_count)}</p>
+				  </div>
+				</article>
+				{/foreach}
+			</div>
+			{/if}
 		</div>
 	</div>
 {/block}
Author	SHA1	Message	Date
Pablo Ferreiro	cac97ff05e	Initial comments support	2023-03-06 21:20:14 +01:00
Pablo Ferreiro	eeaa8cdf1c	Random User Agent generator	2023-03-05 13:18:34 +01:00
Pablo Ferreiro	ecba96eaf4	Merge pull request #131 from chrisnovakovic/video-title Set page title to video description in video view	2023-02-28 11:40:30 +00:00
Chris Novakovic	013feeb679	Set page title to video description in video view If the upstream video has a description set, use it as the page title in place of "Video" in the video view. This helps to disambiguate video pages in the browser history.	2023-02-27 23:11:36 +00:00
Pablo Ferreiro	bf86e0d36a	Merge pull request #125 from samip5/configureable_user_agent Ability to pass custom user agent to the scraper and ignore .git on Docker	2023-02-19 11:08:22 +00:00
Skyler Mäntysaari	72ea5d43fd	Addressing PR comments about not using hardcoded string.	2023-02-17 22:32:36 +02:00
Skyler Mäntysaari	288f87504c	Add .git to dockerignore Otherwise bad things end up in container.	2023-02-16 02:26:25 +02:00
Skyler Mäntysaari	3d5890d821	Ability to pass own user agent to the scraper.	2023-02-08 12:59:17 +02:00
Pablo Ferreiro	5dea2468f8	Better handling of private accounts	2023-02-01 15:22:53 +01:00
Pablo Ferreiro	e8ec0ab293	bump	2023-01-28 17:32:07 +01:00
Pablo Ferreiro	78fe44070c	Fixed Trending and normalized Discover	2023-01-26 19:21:14 +01:00
Pablo Ferreiro	7d2f6e8fd6	bump scraper	2023-01-26 17:20:38 +01:00
Pablo Ferreiro	32d49954f9	hotfix	2023-01-25 21:10:47 +01:00
Pablo Ferreiro	5c770def41	better ui	2023-01-25 21:06:59 +01:00
Pablo Ferreiro	d273b35e53	Added apcu caching method and bump	2023-01-25 15:56:23 +01:00
Pablo Ferreiro	faf1dcee51	bump, video showing up	2023-01-25 15:18:46 +01:00