From 06119eac379f4d0dc83b3b686d64efc444a92fb8 Mon Sep 17 00:00:00 2001 From: WardPearce Date: Thu, 5 Jun 2025 11:46:44 +1200 Subject: [PATCH] Fixes to feed item duplication --- materialious/electron/package-lock.json | 4 +-- materialious/package-lock.json | 20 +++++++++--- materialious/package.json | 3 +- materialious/src/lib/misc.ts | 31 +++++++++++++++++++ materialious/src/lib/store.ts | 8 +++-- .../routes/(app)/search/[slug]/+page.svelte | 13 +++----- .../src/routes/(app)/search/[slug]/+page.ts | 9 +++++- .../src/routes/(app)/subscriptions/+page.ts | 8 ++++- 8 files changed, 74 insertions(+), 22 deletions(-) diff --git a/materialious/electron/package-lock.json b/materialious/electron/package-lock.json index 6564b0ba..456718a2 100644 --- a/materialious/electron/package-lock.json +++ b/materialious/electron/package-lock.json @@ -1,12 +1,12 @@ { "name": "Materialious", - "version": "1.9.0", + "version": "1.9.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "Materialious", - "version": "1.9.0", + "version": "1.9.1", "license": "MIT", "dependencies": { "@capacitor-community/electron": "^5.0.0", diff --git a/materialious/package-lock.json b/materialious/package-lock.json index a8e8ae9d..d84a24a3 100644 --- a/materialious/package-lock.json +++ b/materialious/package-lock.json @@ -1,12 +1,12 @@ { "name": "materialious", - "version": "1.8.11", + "version": "1.9.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "materialious", - "version": "1.8.11", + "version": "1.9.1", "hasInstallScript": true, "dependencies": { "@capacitor-community/electron": "^5.0.1", @@ -37,7 +37,6 @@ "svelte-i18n": "^4.0.1", "svelte-infinite-loading": "^1.4.0", "svelte-persisted-store": "^0.12.0", - "terser": "^5.34.1", "youtubei.js": "^13.4.0" }, "devDependencies": { @@ -2903,7 +2902,9 @@ "version": "0.3.6", "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.25" @@ -4632,7 +4633,9 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "license": "MIT" + "dev": true, + "license": "MIT", + "peer": true }, "node_modules/call-bind": { "version": "1.0.8", @@ -11673,6 +11676,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" @@ -11692,7 +11696,9 @@ "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, "license": "MIT", + "peer": true, "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" @@ -12273,7 +12279,9 @@ "version": "5.39.0", "resolved": "https://registry.npmjs.org/terser/-/terser-5.39.0.tgz", "integrity": "sha512-LBAhFyLho16harJoWMg/nZsQYgTrg5jXOn2nCYjRUcZZEdE3qa2zb8QEDRUGVZBW4rlazf2fxkg8tztybTaqWw==", + "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.8.2", @@ -12291,7 +12299,9 @@ "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", - "license": "MIT" + "dev": true, + "license": "MIT", + "peer": true }, "node_modules/text-decoder": { "version": "1.2.3", diff --git a/materialious/package.json b/materialious/package.json index 16855e1b..99d1088b 100644 --- a/materialious/package.json +++ b/materialious/package.json @@ -68,7 +68,6 @@ "svelte-i18n": "^4.0.1", "svelte-infinite-loading": "^1.4.0", "svelte-persisted-store": "^0.12.0", - "terser": "^5.34.1", "youtubei.js": "^13.4.0" } -} \ No newline at end of file +} diff --git a/materialious/src/lib/misc.ts b/materialious/src/lib/misc.ts index ab1c4ca2..5518fb96 100644 --- a/materialious/src/lib/misc.ts +++ b/materialious/src/lib/misc.ts @@ -4,6 +4,7 @@ import he from 'he'; import type Peer from 'peerjs'; import { get } from 'svelte/store'; import { instanceStore, interfaceAllowInsecureRequests } from './store'; +import type { Channel, HashTag, Playlist, PlaylistPageVideo, Video, VideoBase } from './api/model'; export function truncate(value: string, maxLength: number = 50): string { return value.length > maxLength ? `${value.substring(0, maxLength)}...` : value; @@ -71,3 +72,33 @@ export async function insecureRequestImageHandler(source: string): Promise { + existingIds.push(extractUniqueId(item)); + }); + + const nonDuplicatedNewItems: feedItems = []; + newItems.forEach((item) => { + if (!existingIds.includes(extractUniqueId(item))) { + nonDuplicatedNewItems.push(item); + } + }); + + return [...nonDuplicatedNewItems, ...currentItems]; +} diff --git a/materialious/src/lib/store.ts b/materialious/src/lib/store.ts index a67edd11..84d63a1c 100644 --- a/materialious/src/lib/store.ts +++ b/materialious/src/lib/store.ts @@ -18,7 +18,9 @@ function platformDependentDefault(givenValue: any, defaultValue: any): any { export const instanceStore: Writable = persisted( 'invidiousInstance', platformDependentDefault( - ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_INVIDIOUS_INSTANCE), + !import.meta.env.VITE_DEFAULT_INVIDIOUS_INSTANCE + ? undefined + : ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_INVIDIOUS_INSTANCE), 'https://invidious.materialio.us' ) ); @@ -26,7 +28,9 @@ export const instanceStore: Writable = persisted( export const companionStore: Writable = persisted( 'companionInstance', platformDependentDefault( - ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_COMPANION_INSTANCE), + !import.meta.env.VITE_DEFAULT_COMPANION_INSTANCE + ? undefined + : ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_COMPANION_INSTANCE), undefined ) ); diff --git a/materialious/src/routes/(app)/search/[slug]/+page.svelte b/materialious/src/routes/(app)/search/[slug]/+page.svelte index 5c220d7b..d622f7d2 100644 --- a/materialious/src/routes/(app)/search/[slug]/+page.svelte +++ b/materialious/src/routes/(app)/search/[slug]/+page.svelte @@ -6,6 +6,7 @@ import PageLoading from '$lib/components/PageLoading.svelte'; import PlaylistThumbnail from '$lib/components/PlaylistThumbnail.svelte'; import Thumbnail from '$lib/components/Thumbnail.svelte'; + import { extractUniqueId } from '$lib/misc.js'; import { feedLastItemId, searchCacheStore } from '$lib/store.js'; import { onMount } from 'svelte'; import { _ } from 'svelte-i18n'; @@ -27,6 +28,7 @@ async function changeType(type: 'playlist' | 'all' | 'video' | 'channel') { currentType = type; currentPage = 1; + data.searchStoreId = type + data.slug; searchCacheStore.set({ [data.searchStoreId]: await getSearch(data.slug, { type: type }) }); } @@ -91,15 +93,8 @@
- feedLastItemId.set( - 'videoId' in item ? item.videoId : 'authorId' in item ? item.authorId : item.title - )} - id={'videoId' in item - ? item.videoId - : 'authorId' in item - ? item.authorId - : item.title} + onclick={() => feedLastItemId.set(extractUniqueId(item))} + id={extractUniqueId(item)} > {#if item.type === 'video'} diff --git a/materialious/src/routes/(app)/search/[slug]/+page.ts b/materialious/src/routes/(app)/search/[slug]/+page.ts index 533b0f46..597f0f15 100644 --- a/materialious/src/routes/(app)/search/[slug]/+page.ts +++ b/materialious/src/routes/(app)/search/[slug]/+page.ts @@ -1,4 +1,6 @@ import { getSearch } from '$lib/api/index'; +import type { Channel, HashTag, Playlist, Video } from '$lib/api/model'; +import { excludeDuplicateFeeds } from '$lib/misc.js'; import { searchCacheStore } from '$lib/store.js'; import { error } from '@sveltejs/kit'; import { get } from 'svelte/store'; @@ -25,7 +27,12 @@ export async function load({ params, url }) { } else { getSearch(params.slug, { type: type }).then((newSearch) => { searchCacheStore.set({ - [searchStoreId]: [...new Set([...newSearch, ...search])] + [searchStoreId]: excludeDuplicateFeeds(search, newSearch) as ( + | Channel + | Video + | Playlist + | HashTag + )[] }); }); } diff --git a/materialious/src/routes/(app)/subscriptions/+page.ts b/materialious/src/routes/(app)/subscriptions/+page.ts index c4364ea2..991eaac8 100644 --- a/materialious/src/routes/(app)/subscriptions/+page.ts +++ b/materialious/src/routes/(app)/subscriptions/+page.ts @@ -1,4 +1,6 @@ import { getFeed } from '$lib/api/index'; +import type { PlaylistPageVideo, Video, VideoBase } from '$lib/api/model'; +import { excludeDuplicateFeeds } from '$lib/misc'; import { feedCacheStore } from '$lib/store'; import { error } from '@sveltejs/kit'; import { get } from 'svelte/store'; @@ -21,7 +23,11 @@ export async function load() { await getFeed(100, 1).then((feeds) => { const newVideos = [...feeds.notifications, ...feeds.videos, ...videos]; feedCacheStore.set({ - subscription: [...new Set(newVideos)] + subscription: excludeDuplicateFeeds(videos, newVideos) as ( + | VideoBase + | Video + | PlaylistPageVideo + )[] }); }); }