diff --git a/.jules/bolt.md b/.jules/bolt.md index 9b7817f..edc00d2 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -14,3 +14,7 @@ ## 2026-04-24 - [OOM prevention through promise chunking] **Learning:** When fetching unbounded large collections (like photo albums and their assets), using concurrent `Promise.all` over the entire array can cause Node.js Out of Memory (OOM) crashes. **Action:** Instead of `Promise.all(array.map(...))`, use a chunked data fetching approach (e.g., `Promise.all` within a `for` loop processing chunks of e.g. 10 items) to balance memory constraints with concurrent network speed. + +## 2024-05-30 - [O(1) Memory Bucketing for Coordinate Aggregation] +**Learning:** When bucketing or grouping large datasets (e.g., thousands of map coordinates into geographical clusters), array allocations (`[].push()`) followed by `reduce` operations create an O(N) memory overhead per bucket and expensive post-processing loops. +**Action:** Replace memory-heavy array collections with running statistical accumulators (e.g., `latSum`, `lngSum`, `count`) directly on the bucket. This reduces bucket memory from O(N) to O(1). diff --git a/lib/mapService.ts b/lib/mapService.ts index 7e2d176..0d062c5 100644 --- a/lib/mapService.ts +++ b/lib/mapService.ts @@ -17,77 +17,101 @@ export interface MapLocation { * Aggregate all geotagged photos into location-level markers. * Returns one entry per unique city+country with averaged lat/lng. */ +let pendingMapDataPromise: Promise | null = null; + export async function getMapData(): Promise { const config = getConfig(); const cacheKey = 'map-data'; const cached = cache.get(cacheKey); if (cached) return cached; - const albums = await immich.getAlbums(); + if (pendingMapDataPromise) return pendingMapDataPromise; - // Build a lookup: album ID → { name, slug, subpageSlug? } - const albumMeta = new Map(); - for (const a of albums) { - // Check if this album belongs to a subpage - const sp = config.subpages.find((s) => s.albumIds.includes(a.id)); - albumMeta.set(a.id, { name: a.albumName, slug: a.slug, subpageSlug: sp?.slug }); - } + pendingMapDataPromise = (async () => { + try { + const albums = await immich.getAlbums(); - // Fetch full album data (with assets) for each - // Process in chunks of 10 to balance network speed and prevent Out of Memory (OOM) - // crashes from fetching thousands of assets simultaneously. - const fullAlbums: (Awaited> | null)[] = []; - const chunkSize = 10; - for (let i = 0; i < albums.length; i += chunkSize) { - const chunk = albums.slice(i, i + chunkSize); - const chunkResults = await Promise.all(chunk.map((a) => immich.getAlbum(a.id))); - fullAlbums.push(...chunkResults); - } - - // Bucket assets by city+country - const buckets = new Map< - string, - { lats: number[]; lngs: number[]; count: number; coverAssetId: string; albumIds: Set } - >(); - - for (const album of fullAlbums) { - if (!album) continue; - for (const asset of album.assets) { - const exif = asset.exifInfo; - if (!exif?.latitude || !exif?.longitude || !exif?.city || !exif?.country) continue; - - const key = `${exif.city}|${exif.country}`; - let bucket = buckets.get(key); - if (!bucket) { - bucket = { lats: [], lngs: [], count: 0, coverAssetId: asset.id, albumIds: new Set() }; - buckets.set(key, bucket); + // Build a lookup: album ID → { name, slug, subpageSlug? } + const albumMeta = new Map(); + for (const a of albums) { + // Check if this album belongs to a subpage + const sp = config.subpages.find((s) => s.albumIds.includes(a.id)); + albumMeta.set(a.id, { name: a.albumName, slug: a.slug, subpageSlug: sp?.slug }); } - bucket.lats.push(exif.latitude); - bucket.lngs.push(exif.longitude); - bucket.count++; - bucket.albumIds.add(album.id); + + // Fetch full album data (with assets) for each + // Process in chunks of 10 to balance network speed and prevent Out of Memory (OOM) + // crashes from fetching thousands of assets simultaneously. + const fullAlbums: (Awaited> | null)[] = []; + const chunkSize = 10; + for (let i = 0; i < albums.length; i += chunkSize) { + const chunk = albums.slice(i, i + chunkSize); + const chunkResults = await Promise.all(chunk.map((a) => immich.getAlbum(a.id))); + fullAlbums.push(...chunkResults); + } + + // Bucket assets by city+country + const buckets = new Map< + string, + { + latSum: number; + lngSum: number; + count: number; + coverAssetId: string; + albumIds: Set; + } + >(); + + for (const album of fullAlbums) { + if (!album) continue; + for (const asset of album.assets) { + const exif = asset.exifInfo; + if (!exif?.latitude || !exif?.longitude || !exif?.city || !exif?.country) continue; + + const key = `${exif.city}|${exif.country}`; + let bucket = buckets.get(key); + if (!bucket) { + bucket = { + latSum: 0, + lngSum: 0, + count: 0, + coverAssetId: asset.id, + albumIds: new Set(), + }; + buckets.set(key, bucket); + } + bucket.latSum += exif.latitude; + bucket.lngSum += exif.longitude; + bucket.count++; + bucket.albumIds.add(album.id); + } + } + + // Convert buckets to MapLocation[] + const locations: MapLocation[] = []; + for (const [key, bucket] of buckets) { + const [city, country] = key.split('|'); + const lat = bucket.latSum / bucket.count; + const lng = bucket.lngSum / bucket.count; + locations.push({ + city, + country, + lat, + lng, + photoCount: bucket.count, + coverAssetId: bucket.coverAssetId, + albums: [...bucket.albumIds] + .map((id) => albumMeta.get(id)) + .filter((m): m is NonNullable => !!m), + }); + } + + cache.set(cacheKey, locations, config.cacheTtl); + return locations; + } finally { + pendingMapDataPromise = null; } - } + })(); - // Convert buckets to MapLocation[] - const locations: MapLocation[] = []; - for (const [key, bucket] of buckets) { - const [city, country] = key.split('|'); - const lat = bucket.lats.reduce((a, b) => a + b, 0) / bucket.lats.length; - const lng = bucket.lngs.reduce((a, b) => a + b, 0) / bucket.lngs.length; - locations.push({ - city, - country, - lat, - lng, - photoCount: bucket.count, - coverAssetId: bucket.coverAssetId, - albums: [...bucket.albumIds] - .map((id) => albumMeta.get(id)) - .filter((m): m is NonNullable => !!m), - }); - } - - cache.set(cacheKey, locations, config.cacheTtl); - return locations; + return pendingMapDataPromise; }