Skip to content

Deepen Currently Listening module #62

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
BradNut wants to merge 1 commit into
base: development
Choose a base branch
from
Open

Deepen Currently Listening module #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions src/lib/server/currentlyListening.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';

const redisGetMock = vi.fn();
const redisTtlMock = vi.fn();
const redisSetWithExpiryMock = vi.fn();
const scrapeItMock = vi.fn();

vi.mock('$lib/server/redis', () => ({
redisService: {
get: (data: unknown) => redisGetMock(data),
ttl: (data: unknown) => redisTtlMock(data),
setWithExpiry: (data: unknown) => redisSetWithExpiryMock(data),
},
REDIS_PREFIXES: {
ARTICLES: 'articles',
BANDCAMP_ALBUMS: 'bandcampAlbums',
PAGE_CACHE: 'pageCache',
},
}));

vi.mock('scrape-it', () => ({ default: (...args: unknown[]) => scrapeItMock(...args) }));

vi.mock('$lib/util/retry', () => ({
retryWithBackoff: (fn: () => Promise<unknown>) => fn(),
}));

import { getCurrentlyListeningAlbums } from './currentlyListening';

const makeAlbum = () => ({
url: 'https://bandcamp.com/album/123',
artwork: 'https://img.bandcamp.com/art.jpg',
title: 'Test Album',
artist: 'Test Artist',
});

beforeEach(() => {
vi.resetAllMocks();
});

describe('getCurrentlyListeningAlbums', () => {
it('returns cached albums with TTL cache metadata', async () => {
const cached = [makeAlbum()];
redisGetMock.mockResolvedValueOnce(JSON.stringify(cached));
redisTtlMock.mockResolvedValueOnce(3600);

const result = await getCurrentlyListeningAlbums();

expect(result).toEqual({ albums: cached, cacheMaxAge: 3600 });
expect(redisGetMock).toHaveBeenCalledWith({ prefix: 'bandcampAlbums', key: 'albums' });
expect(redisTtlMock).toHaveBeenCalledWith({ prefix: 'bandcampAlbums', key: 'albums' });
expect(scrapeItMock).not.toHaveBeenCalled();
});

it('returns cached albums with fallback cache metadata when TTL is unavailable', async () => {
const cached = [makeAlbum()];
redisGetMock.mockResolvedValueOnce(JSON.stringify(cached));
redisTtlMock.mockResolvedValueOnce(0);

const result = await getCurrentlyListeningAlbums();

expect(result).toEqual({ albums: cached, cacheMaxAge: 43200 });
expect(scrapeItMock).not.toHaveBeenCalled();
});

it('scrapes, caches, and returns albums with default cache metadata on cache miss', async () => {
const scraped = [makeAlbum()];
redisGetMock.mockResolvedValueOnce(null);
scrapeItMock.mockResolvedValueOnce({ data: { collectionItems: scraped } });

const result = await getCurrentlyListeningAlbums();

expect(result).toEqual({ albums: scraped, cacheMaxAge: 43200 });
expect(scrapeItMock).toHaveBeenCalledWith('https://bandcamp.com/test-user', expect.any(Object));
expect(redisSetWithExpiryMock).toHaveBeenCalledWith({
prefix: 'bandcampAlbums',
key: 'albums',
value: JSON.stringify(scraped),
expiry: 43200,
});
});

it('returns empty albums without cache metadata when scrape returns no items', async () => {
redisGetMock.mockResolvedValueOnce(null);
scrapeItMock.mockResolvedValueOnce({ data: { collectionItems: [] } });

const result = await getCurrentlyListeningAlbums();

expect(result).toEqual({ albums: [], cacheMaxAge: null });
expect(redisSetWithExpiryMock).not.toHaveBeenCalled();
});

it('returns empty albums without cache metadata when scrape fails', async () => {
vi.spyOn(console, 'error').mockImplementation(() => undefined);
redisGetMock.mockResolvedValueOnce(null);
scrapeItMock.mockRejectedValueOnce(new Error('scrape failed'));

const result = await getCurrentlyListeningAlbums();

expect(result).toEqual({ albums: [], cacheMaxAge: null });
expect(redisSetWithExpiryMock).not.toHaveBeenCalled();
});
});
71 changes: 71 additions & 0 deletions src/lib/server/currentlyListening.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import scrapeIt, { type ScrapeResult } from 'scrape-it';
import { ENV } from 'varlock/env';
import { REDIS_PREFIXES, redisService } from '$lib/server/redis';
import type { Album, BandCampResults } from '$lib/types/album';
import { retryWithBackoff } from '$lib/util/retry';

const BANDCAMP_ALBUMS_CACHE_KEY = 'albums';
const BANDCAMP_ALBUMS_CACHE_MAX_AGE = 43200;

export type CurrentlyListeningAlbumsResult = {
albums: Album[];
cacheMaxAge: number | null;
};

export async function getCurrentlyListeningAlbums(): Promise<CurrentlyListeningAlbumsResult> {
try {
const cached = await getCachedAlbums();
if (cached) return cached;

const albums = await scrapeCurrentlyListeningAlbums();
if (albums.length === 0) return { albums: [], cacheMaxAge: null };

await cacheAlbums(albums);
return { albums, cacheMaxAge: BANDCAMP_ALBUMS_CACHE_MAX_AGE };
} catch (error) {
console.error(error);
return { albums: [], cacheMaxAge: null };
}
}

async function getCachedAlbums(): Promise<CurrentlyListeningAlbumsResult | null> {
if (!ENV.USE_REDIS_CACHE) return null;

const cached = await redisService.get({ prefix: REDIS_PREFIXES.BANDCAMP_ALBUMS, key: BANDCAMP_ALBUMS_CACHE_KEY });
if (!cached) return null;

const albums: Album[] = JSON.parse(cached);
const ttl = await redisService.ttl({ prefix: REDIS_PREFIXES.BANDCAMP_ALBUMS, key: BANDCAMP_ALBUMS_CACHE_KEY });
return { albums, cacheMaxAge: ttl || BANDCAMP_ALBUMS_CACHE_MAX_AGE };
}

async function scrapeCurrentlyListeningAlbums(): Promise<Album[]> {
// Scrape Bandcamp with realistic headers, plus retry/backoff
const { data }: ScrapeResult<BandCampResults> = await retryWithBackoff(
async () =>
await scrapeIt(`https://bandcamp.com/${ENV.BANDCAMP_USERNAME}`, {
collectionItems: {
listItem: '.collection-item-container',
data: {
url: { selector: '.collection-title-details > a.item-link', attr: 'href' },
artwork: { selector: 'div.collection-item-art-container a img', attr: 'src' },
title: { selector: 'span.item-link-alt > div.collection-item-title' },
artist: { selector: 'span.item-link-alt > div.collection-item-artist' },
},
},
}),
);

return data?.collectionItems || [];
}

async function cacheAlbums(albums: Album[]): Promise<void> {
if (!ENV.USE_REDIS_CACHE) return;

await redisService.setWithExpiry({
prefix: REDIS_PREFIXES.BANDCAMP_ALBUMS,
key: BANDCAMP_ALBUMS_CACHE_KEY,
value: JSON.stringify(albums),
expiry: BANDCAMP_ALBUMS_CACHE_MAX_AGE,
});
}
61 changes: 7 additions & 54 deletions src/routes/api/bandcamp/albums/+server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,60 +1,13 @@
import { json, type RequestEvent } from '@sveltejs/kit';
import scrapeIt, { type ScrapeResult } from 'scrape-it';
import { ENV } from 'varlock/env';
import { REDIS_PREFIXES, redisService } from '$lib/server/redis';
import type { Album, BandCampResults } from '$lib/types/album';
import { retryWithBackoff } from '$lib/util/retry';
import { getCurrentlyListeningAlbums } from '$lib/server/currentlyListening';

export async function GET(event: RequestEvent) {
const { setHeaders } = event;
const { setHeaders } = event;
const { albums, cacheMaxAge } = await getCurrentlyListeningAlbums();

try {
if (ENV.USE_REDIS_CACHE) {
const cached: string | null = await redisService.get({ prefix: REDIS_PREFIXES.BANDCAMP_ALBUMS, key: 'albums' });
if (cacheMaxAge !== null) {
setHeaders({ 'cache-control': `max-age=${cacheMaxAge}` });
}

if (cached) {
const response: Album[] = JSON.parse(cached);
const ttl = await redisService.ttl({ prefix: REDIS_PREFIXES.BANDCAMP_ALBUMS, key: 'albums' });
if (ttl) {
setHeaders({
'cache-control': `max-age=${ttl}`,
});
} else {
setHeaders({
'cache-control': 'max-age=43200',
});
}
return json(response);
}
}

// Scrape Bandcamp with realistic headers, plus retry/backoff
const { data }: ScrapeResult<BandCampResults> = await retryWithBackoff(
async () =>
await scrapeIt(`https://bandcamp.com/${ENV.BANDCAMP_USERNAME}`, {
collectionItems: {
listItem: '.collection-item-container',
data: {
url: { selector: '.collection-title-details > a.item-link', attr: 'href' },
artwork: { selector: 'div.collection-item-art-container a img', attr: 'src' },
title: { selector: 'span.item-link-alt > div.collection-item-title' },
artist: { selector: 'span.item-link-alt > div.collection-item-artist' },
},
},
}),
);

const albums: Album[] = data?.collectionItems || [];
if (albums && albums.length > 0) {
if (ENV.USE_REDIS_CACHE) {
await redisService.setWithExpiry({ prefix: REDIS_PREFIXES.BANDCAMP_ALBUMS, key: 'albums', value: JSON.stringify(albums), expiry: 43200 });
}
setHeaders({ 'cache-control': 'max-age=43200' });
return json(albums);
}
return json([]);
} catch (error) {
console.error(error);
return json([]);
}
return json(albums);
}
122 changes: 30 additions & 92 deletions src/routes/api/bandcamp/albums/server.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,115 +1,53 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';

const scrapeItMock = vi.fn();
const redisGetMock = vi.fn();
const redisTtlMock = vi.fn();
const redisSetWithExpiryMock = vi.fn();
const getCurrentlyListeningAlbumsMock = vi.fn();

vi.mock('scrape-it', () => ({ default: (...args: unknown[]) => scrapeItMock(...args) }));

vi.mock('$lib/server/redis', () => ({
redisService: {
get: (d: unknown) => redisGetMock(d),
ttl: (d: unknown) => redisTtlMock(d),
setWithExpiry: (d: unknown) => redisSetWithExpiryMock(d),
set: vi.fn(),
delete: vi.fn(),
scan: vi.fn(),
redis: null,
},
REDIS_PREFIXES: {
ARTICLES: 'articles',
BANDCAMP_ALBUMS: 'bandcampAlbums',
PAGE_CACHE: 'pageCache',
},
}));

vi.mock('$lib/util/retry', () => ({
retryWithBackoff: (fn: () => Promise<unknown>) => fn(),
vi.mock('$lib/server/currentlyListening', () => ({
getCurrentlyListeningAlbums: () => getCurrentlyListeningAlbumsMock(),
}));

import { GET } from './+server.js';

const makeAlbum = () => ({
url: 'https://bandcamp.com/album/123',
artwork: 'https://img.bandcamp.com/art.jpg',
title: 'Test Album',
artist: 'Test Artist',
url: 'https://bandcamp.com/album/123',
artwork: 'https://img.bandcamp.com/art.jpg',
title: 'Test Album',
artist: 'Test Artist',
});

function makeRequestEvent() {
const capturedHeaders: Record<string, string> = {};
const event = {
setHeaders: (h: Record<string, string>) => Object.assign(capturedHeaders, h),
} as unknown as Parameters<typeof GET>[0];
return { event, capturedHeaders };
const capturedHeaders: Record<string, string> = {};
const event = {
setHeaders: (h: Record<string, string>) => Object.assign(capturedHeaders, h),
} as unknown as Parameters<typeof GET>[0];
return { event, capturedHeaders };
}

beforeEach(() => {
vi.resetAllMocks();
vi.resetAllMocks();
});

describe('GET /api/bandcamp/albums', () => {
it('returns cached albums with TTL-based cache-control', async () => {
const cached = [makeAlbum()];
redisGetMock.mockResolvedValueOnce(JSON.stringify(cached));
redisTtlMock.mockResolvedValueOnce(3600);

const { event, capturedHeaders } = makeRequestEvent();
const response = await GET(event);
const body = await response.json();

expect(body).toHaveLength(1);
expect(body[0].title).toBe('Test Album');
expect(capturedHeaders['cache-control']).toBe('max-age=3600');
expect(scrapeItMock).not.toHaveBeenCalled();
});

it('returns cached albums with fallback cache-control when no TTL', async () => {
const cached = [makeAlbum()];
redisGetMock.mockResolvedValueOnce(JSON.stringify(cached));
redisTtlMock.mockResolvedValueOnce(0);

const { event, capturedHeaders } = makeRequestEvent();
await GET(event);

expect(capturedHeaders['cache-control']).toBe('max-age=43200');
});

it('scrapes and returns albums on cache miss', async () => {
redisGetMock.mockResolvedValueOnce(null);
scrapeItMock.mockResolvedValueOnce({ data: { collectionItems: [makeAlbum()] } });

const { event, capturedHeaders } = makeRequestEvent();
const response = await GET(event);
const body = await response.json();

expect(body).toHaveLength(1);
expect(body[0].artist).toBe('Test Artist');
expect(capturedHeaders['cache-control']).toBe('max-age=43200');
expect(redisSetWithExpiryMock).toHaveBeenCalledOnce();
});

it('returns empty array when scrape returns no items', async () => {
redisGetMock.mockResolvedValueOnce(null);
scrapeItMock.mockResolvedValueOnce({ data: { collectionItems: [] } });
it('returns albums and applies cache metadata from the Currently Listening module', async () => {
const albums = [makeAlbum()];
getCurrentlyListeningAlbumsMock.mockResolvedValueOnce({ albums, cacheMaxAge: 3600 });

const { event } = makeRequestEvent();
const response = await GET(event);
const body = await response.json();
const { event, capturedHeaders } = makeRequestEvent();
const response = await GET(event);
const body = await response.json();

expect(body).toEqual([]);
expect(redisSetWithExpiryMock).not.toHaveBeenCalled();
});
expect(body).toEqual(albums);
expect(capturedHeaders['cache-control']).toBe('max-age=3600');
});

it('returns empty array when scrape throws', async () => {
redisGetMock.mockResolvedValueOnce(null);
scrapeItMock.mockRejectedValueOnce(new Error('scrape failed'));
it('returns albums without cache-control when the module has no cache metadata', async () => {
getCurrentlyListeningAlbumsMock.mockResolvedValueOnce({ albums: [], cacheMaxAge: null });

const { event } = makeRequestEvent();
const response = await GET(event);
const body = await response.json();
const { event, capturedHeaders } = makeRequestEvent();
const response = await GET(event);
const body = await response.json();

expect(body).toEqual([]);
});
expect(body).toEqual([]);
expect(capturedHeaders).toEqual({});
});
});
Loading