blade/materialious
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes to feed item duplication

Browse Source
This commit is contained in:
WardPearce
2025-06-05 11:46:44 +12:00
parent 07e0e9d9cd
commit 06119eac37
8 changed files with 74 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
materialious/electron/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "Materialious",
"version": "1.9.0",
"version": "1.9.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "Materialious",
"version": "1.9.0",
"version": "1.9.1",
"license": "MIT",
"dependencies": {
"@capacitor-community/electron": "^5.0.0",
materialious/package-lock.json
Generated
+15 -5
View File
@@ -1,12 +1,12 @@
{
"name": "materialious",
"version": "1.8.11",
"version": "1.9.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "materialious",
"version": "1.8.11",
"version": "1.9.1",
"hasInstallScript": true,
"dependencies": {
"@capacitor-community/electron": "^5.0.1",
@@ -37,7 +37,6 @@
"svelte-i18n": "^4.0.1",
"svelte-infinite-loading": "^1.4.0",
"svelte-persisted-store": "^0.12.0",
"terser": "^5.34.1",
"youtubei.js": "^13.4.0"
},
"devDependencies": {
@@ -2903,7 +2902,9 @@
"version": "0.3.6",
"resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz",
"integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@jridgewell/gen-mapping": "^0.3.5",
"@jridgewell/trace-mapping": "^0.3.25"
@@ -4632,7 +4633,9 @@
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
"license": "MIT"
"dev": true,
"license": "MIT",
"peer": true
},
"node_modules/call-bind": {
"version": "1.0.8",
@@ -11673,6 +11676,7 @@
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"dev": true,
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.10.0"
@@ -11692,7 +11696,9 @@
"version": "0.5.21",
"resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz",
"integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"buffer-from": "^1.0.0",
"source-map": "^0.6.0"
@@ -12273,7 +12279,9 @@
"version": "5.39.0",
"resolved": "https://registry.npmjs.org/terser/-/terser-5.39.0.tgz",
"integrity": "sha512-LBAhFyLho16harJoWMg/nZsQYgTrg5jXOn2nCYjRUcZZEdE3qa2zb8QEDRUGVZBW4rlazf2fxkg8tztybTaqWw==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"@jridgewell/source-map": "^0.3.3",
"acorn": "^8.8.2",
@@ -12291,7 +12299,9 @@
"version": "2.20.3",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
"license": "MIT"
"dev": true,
"license": "MIT",
"peer": true
},
"node_modules/text-decoder": {
"version": "1.2.3",
materialious/package.json
+1 -2
View File
@@ -68,7 +68,6 @@
"svelte-i18n": "^4.0.1",
"svelte-infinite-loading": "^1.4.0",
"svelte-persisted-store": "^0.12.0",
"terser": "^5.34.1",
"youtubei.js": "^13.4.0"
}
}
}
materialious/src/lib/misc.ts
+31
View File
@@ -4,6 +4,7 @@ import he from 'he';
import type Peer from 'peerjs';
import { get } from 'svelte/store';
import { instanceStore, interfaceAllowInsecureRequests } from './store';
import type { Channel, HashTag, Playlist, PlaylistPageVideo, Video, VideoBase } from './api/model';
export function truncate(value: string, maxLength: number = 50): string {
return value.length > maxLength ? `${value.substring(0, maxLength)}...` : value;
@@ -71,3 +72,33 @@ export async function insecureRequestImageHandler(source: string): Promise<HTMLI
return img;
}
export type feedItem = VideoBase | Video | PlaylistPageVideo | Channel | Video | Playlist | HashTag;
export type feedItems = feedItem[];
export function extractUniqueId(item: feedItem): string {
if ('videoId' in item) {
return item.videoId;
} else if ('authorId' in item) {
return item.authorId;
} else {
return item.title;
}
}
export function excludeDuplicateFeeds(currentItems: feedItems, newItems: feedItems): feedItems {
const existingIds: string[] = [];
currentItems.forEach((item) => {
existingIds.push(extractUniqueId(item));
});
const nonDuplicatedNewItems: feedItems = [];
newItems.forEach((item) => {
if (!existingIds.includes(extractUniqueId(item))) {
nonDuplicatedNewItems.push(item);
}
});
return [...nonDuplicatedNewItems, ...currentItems];
}
materialious/src/lib/store.ts
+6 -2
View File
@@ -18,7 +18,9 @@ function platformDependentDefault(givenValue: any, defaultValue: any): any {
export const instanceStore: Writable<string> = persisted(
'invidiousInstance',
platformDependentDefault(
ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_INVIDIOUS_INSTANCE),
!import.meta.env.VITE_DEFAULT_INVIDIOUS_INSTANCE
? undefined
: ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_INVIDIOUS_INSTANCE),
'https://invidious.materialio.us'
)
);
@@ -26,7 +28,9 @@ export const instanceStore: Writable<string> = persisted(
export const companionStore: Writable<string | undefined> = persisted(
'companionInstance',
platformDependentDefault(
ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_COMPANION_INSTANCE),
!import.meta.env.VITE_DEFAULT_COMPANION_INSTANCE
? undefined
: ensureNoTrailingSlash(import.meta.env.VITE_DEFAULT_COMPANION_INSTANCE),
undefined
)
);
materialious/src/routes/(app)/search/[slug]/+page.svelte
+4 -9
View File
@@ -6,6 +6,7 @@
import PageLoading from '$lib/components/PageLoading.svelte';
import PlaylistThumbnail from '$lib/components/PlaylistThumbnail.svelte';
import Thumbnail from '$lib/components/Thumbnail.svelte';
import { extractUniqueId } from '$lib/misc.js';
import { feedLastItemId, searchCacheStore } from '$lib/store.js';
import { onMount } from 'svelte';
import { _ } from 'svelte-i18n';
@@ -27,6 +28,7 @@
async function changeType(type: 'playlist' | 'all' | 'video' | 'channel') {
currentType = type;
currentPage = 1;
data.searchStoreId = type + data.slug;
searchCacheStore.set({ [data.searchStoreId]: await getSearch(data.slug, { type: type }) });
}
@@ -91,15 +93,8 @@
<article
class="no-padding"
style="height: 100%;"
onclick={() =>
feedLastItemId.set(
'videoId' in item ? item.videoId : 'authorId' in item ? item.authorId : item.title
)}
id={'videoId' in item
? item.videoId
: 'authorId' in item
? item.authorId
: item.title}
onclick={() => feedLastItemId.set(extractUniqueId(item))}
id={extractUniqueId(item)}
>
{#if item.type === 'video'}
<Thumbnail video={item} />
materialious/src/routes/(app)/search/[slug]/+page.ts
+8 -1
View File
@@ -1,4 +1,6 @@
import { getSearch } from '$lib/api/index';
import type { Channel, HashTag, Playlist, Video } from '$lib/api/model';
import { excludeDuplicateFeeds } from '$lib/misc.js';
import { searchCacheStore } from '$lib/store.js';
import { error } from '@sveltejs/kit';
import { get } from 'svelte/store';
@@ -25,7 +27,12 @@ export async function load({ params, url }) {
} else {
getSearch(params.slug, { type: type }).then((newSearch) => {
searchCacheStore.set({
[searchStoreId]: [...new Set([...newSearch, ...search])]
[searchStoreId]: excludeDuplicateFeeds(search, newSearch) as (
| Channel
| Video
| Playlist
| HashTag
)[]
});
});
}
materialious/src/routes/(app)/subscriptions/+page.ts
+7 -1
View File
@@ -1,4 +1,6 @@
import { getFeed } from '$lib/api/index';
import type { PlaylistPageVideo, Video, VideoBase } from '$lib/api/model';
import { excludeDuplicateFeeds } from '$lib/misc';
import { feedCacheStore } from '$lib/store';
import { error } from '@sveltejs/kit';
import { get } from 'svelte/store';
@@ -21,7 +23,11 @@ export async function load() {
await getFeed(100, 1).then((feeds) => {
const newVideos = [...feeds.notifications, ...feeds.videos, ...videos];
feedCacheStore.set({
subscription: [...new Set(newVideos)]
subscription: excludeDuplicateFeeds(videos, newVideos) as (
| VideoBase
| Video
| PlaylistPageVideo
)[]
});
});
}