Async non-blocking scanning, don't rerun when items haven't been modified.

This commit is contained in:
2026-02-09 03:22:20 +00:00
parent e2962953f2
commit e897dfcf9c
12 changed files with 366 additions and 269 deletions
+10 -7
View File
@@ -4,6 +4,8 @@ import httpx
from pathlib import Path
from typing import Optional
from aiopathlib import AsyncPath
from hivescan.utils import get_media_folder_path
@@ -32,15 +34,16 @@ async def _download_image(
url: str, output_path: Path, description: str
) -> Optional[str]:
"""Download an image from URL to output path."""
if output_path.exists():
ap = AsyncPath(output_path)
if await ap.exists():
return str(output_path)
try:
client = _get_image_client()
response = await client.get(url)
response.raise_for_status()
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(response.content)
await AsyncPath(output_path.parent).mkdir(parents=True, exist_ok=True)
await ap.write_bytes(response.content)
return str(output_path)
except Exception as e:
print(f" Failed to download {description}: {e}")
@@ -62,7 +65,7 @@ async def download_cover_image(
media_folder = get_media_folder_path(title, year, media_type, cover_dir)
cover_path = media_folder / "cover.jpg"
if cover_path.exists():
if await AsyncPath(cover_path).exists():
return str(cover_path)
url = f"{TMDB_IMAGE_BASE}/{size}{poster_path}"
@@ -85,7 +88,7 @@ async def download_backdrop_image(
media_folder = get_media_folder_path(title, year, media_type, cover_dir)
local_path = media_folder / "backdrop.jpg"
if local_path.exists():
if await AsyncPath(local_path).exists():
return str(local_path)
url = f"{TMDB_IMAGE_BASE}/{size}{backdrop_path}"
@@ -104,9 +107,9 @@ async def download_season_poster(
output_path = media_folder / f"season{season_num:02d}.jpg"
if output_path.exists():
if await AsyncPath(output_path).exists():
return str(output_path)
media_folder.mkdir(parents=True, exist_ok=True)
await AsyncPath(media_folder).mkdir(parents=True, exist_ok=True)
url = f"{TMDB_IMAGE_BASE}/{DEFAULT_POSTER_SIZE}{poster_path}"
return await _download_image(url, output_path, f"season {season_num} poster")
+36 -20
View File
@@ -9,11 +9,13 @@ debounced background task.
import asyncio
import logging
import os
from datetime import datetime
from pathlib import Path
from typing import Optional
import msgspec
from aiopathlib import AsyncPath
from fastapi import WebSocket
from hivescan.structs import (
@@ -57,14 +59,15 @@ class IndexStore:
# Persistence
# ------------------------------------------------------------------
def load_snapshot(self) -> None:
async def load_snapshot(self) -> None:
"""Load index from disk snapshot (recovery on startup)."""
if not self.snapshot_path.exists():
ap = AsyncPath(self.snapshot_path)
if not await ap.exists():
logger.info("No snapshot found at %s, starting fresh", self.snapshot_path)
return
try:
data = msgspec.json.decode(
self.snapshot_path.read_bytes(), type=IndexSnapshot
await ap.read_bytes(), type=IndexSnapshot
)
for m in data.movies:
self.movies[m.id] = m
@@ -78,8 +81,8 @@ class IndexStore:
except Exception:
logger.exception("Failed to load snapshot from %s", self.snapshot_path)
def _write_snapshot(self) -> None:
"""Write current index to disk (synchronous, called from debounce task)."""
async def _write_snapshot(self) -> None:
"""Write current index to disk (called from debounce task)."""
movies_list = sorted(
self.movies.values(), key=lambda x: (x.title.lower(), x.year or 0)
)
@@ -103,25 +106,28 @@ class IndexStore:
series=series_list,
)
self.snapshot_path.parent.mkdir(parents=True, exist_ok=True)
await AsyncPath(self.snapshot_path.parent).mkdir(parents=True, exist_ok=True)
tmp = self.snapshot_path.with_suffix(".tmp")
tmp.write_bytes(msgspec.json.format(msgspec.json.encode(snapshot), indent=2))
tmp.replace(self.snapshot_path)
await AsyncPath(tmp).write_bytes(msgspec.json.format(msgspec.json.encode(snapshot), indent=2))
# os.replace is atomic and overwrites on all platforms (unlike rename on Windows)
await asyncio.to_thread(os.replace, tmp, self.snapshot_path)
logger.debug("Snapshot written to %s", self.snapshot_path)
def _schedule_snapshot(self) -> None:
"""Schedule a debounced snapshot write."""
self._snapshot_dirty = True
if self._snapshot_task is None or self._snapshot_task.done():
self._snapshot_task = asyncio.create_task(self._debounced_snapshot())
self._snapshot_task = asyncio.create_task(self._snapshot_writer())
async def _debounced_snapshot(self) -> None:
"""Wait for debounce interval then write if still dirty."""
while self._snapshot_dirty:
self._snapshot_dirty = False
async def _snapshot_writer(self) -> None:
"""Flush to disk every SNAPSHOT_DEBOUNCE seconds while dirty."""
while True:
await asyncio.sleep(SNAPSHOT_DEBOUNCE)
# After the sleep, if no new mutations happened, write
self._write_snapshot()
if self._snapshot_dirty:
self._snapshot_dirty = False
await self._write_snapshot()
else:
break # No pending mutations — stop the loop
async def flush_snapshot(self) -> None:
"""Force-write a snapshot immediately (e.g. on shutdown)."""
@@ -131,23 +137,33 @@ class IndexStore:
await self._snapshot_task
except asyncio.CancelledError:
pass
self._write_snapshot()
await self._write_snapshot()
# ------------------------------------------------------------------
# Mutations
# ------------------------------------------------------------------
def upsert_movie(self, item: Movie) -> None:
"""Insert or update a movie in the index and broadcast."""
def upsert_movie(self, item: Movie) -> bool:
"""Insert or update a movie. Returns True if it was a real change."""
existing = self.movies.get(item.id)
if existing is not None:
if msgspec.json.encode(existing) == msgspec.json.encode(item):
return False
self.movies[item.id] = item
self._schedule_snapshot()
self._broadcast(WsUpsert(kind="movie", item=item))
return True
def upsert_series(self, item: Series) -> None:
"""Insert or update a series in the index and broadcast."""
def upsert_series(self, item: Series) -> bool:
"""Insert or update a series. Returns True if it was a real change."""
existing = self.series.get(item.id)
if existing is not None:
if msgspec.json.encode(existing) == msgspec.json.encode(item):
return False
self.series[item.id] = item
self._schedule_snapshot()
self._broadcast(WsUpsert(kind="series", item=item))
return True
def remove_movie(self, item_id: str) -> None:
"""Remove a movie from the index and broadcast."""
+120 -85
View File
@@ -1,5 +1,6 @@
"""Media index generation — async generators for continuous scanning."""
import asyncio
import hashlib
import logging
from pathlib import Path
@@ -35,24 +36,28 @@ from hivescan.images import (
)
from hivescan.utils import (
get_added_timestamp,
get_directory_size,
get_media_folder_path,
make_relative_path,
sort_by_quality,
RESOLUTION_PRIORITY,
)
logger = logging.getLogger("hivescan.indexer")
def _build_version_info(
async def _build_version_info(
item: ParsedContent, media_root: Optional[str] = None
) -> MovieVersion:
"""Build version/release info for a single torrent."""
playable_file = find_playable_file(item.path)
playable_file = await find_playable_file(item.path)
if item.content_hash and item.content_hash.size == 0:
item.content_hash.size = await get_directory_size(item.content_hash.path)
size = item.content_hash.size if item.content_hash else None
newest = get_added_timestamp(item.path)
newest = await get_added_timestamp(item.path)
return MovieVersion(
path=make_relative_path(str(item.path), media_root),
torrent_title=item.title,
playable_file=make_relative_path(playable_file, media_root),
resolution=item.resolution,
quality=item.quality,
@@ -64,7 +69,7 @@ def _build_version_info(
)
def _collect_episode_files(
async def _collect_episode_files(
items: List[ParsedContent],
) -> Dict[Tuple[int, int], List[Dict]]:
"""
@@ -75,7 +80,7 @@ def _collect_episode_files(
all_episode_files: Dict[Tuple[int, int], List[Dict]] = {}
for item in items:
episode_files = find_episode_files(item.path)
episode_files = await find_episode_files(item.path)
for (season_num, episode_num), files in episode_files.items():
key = (season_num, episode_num)
@@ -92,6 +97,7 @@ def _collect_episode_files(
"audio": item.audio,
"encoder": item.encoder,
"torrent_path": str(item.path),
"torrent_title": item.title,
}
)
@@ -104,7 +110,7 @@ def _collect_episode_files(
item.episode if isinstance(item.episode, list) else [item.episode]
)
playable = find_playable_file(item.path)
playable = await find_playable_file(item.path)
if playable:
for sn in season_nums:
for ep in episode_nums:
@@ -116,6 +122,8 @@ def _collect_episode_files(
for f in all_episode_files.get(key, [])
)
if not already_added:
if item.content_hash and item.content_hash.size == 0:
item.content_hash.size = await get_directory_size(item.content_hash.path)
size = item.content_hash.size if item.content_hash else 0
all_episode_files[key].append(
{
@@ -127,6 +135,7 @@ def _collect_episode_files(
"audio": item.audio,
"encoder": item.encoder,
"torrent_path": str(item.path),
"torrent_title": item.title,
}
)
@@ -166,19 +175,18 @@ def _build_episodes_data(
(best_file, series_folder, season_num, episode_num, series_title)
)
releases = []
releases = {}
for f in episode_files:
releases.append(
EpisodeRelease(
path=make_relative_path(f["torrent_path"], media_root),
playable_file=make_relative_path(f["path"], media_root),
resolution=f.get("resolution"),
quality=f.get("quality"),
codec=f.get("codec"),
audio=f.get("audio"),
encoder=f.get("encoder"),
size=f.get("size"),
)
relpath = make_relative_path(f["torrent_path"], media_root)
releases[relpath] = EpisodeRelease(
torrent_title=f["torrent_title"],
playable_file=make_relative_path(f["path"], media_root),
resolution=f.get("resolution"),
quality=f.get("quality"),
codec=f.get("codec"),
audio=f.get("audio"),
encoder=f.get("encoder"),
size=f.get("size"),
)
episode_data = Episode(
@@ -228,7 +236,7 @@ async def _build_seasons_data(
if tmdb_id:
cache_key = (tmdb_id, season_num)
if cache_key not in season_cache:
logger.info(
logger.debug(
" Fetching season %d details for %s", season_num, display_title
)
season_cache[cache_key] = await fetch_season_details(
@@ -297,16 +305,17 @@ async def _process_movies(
movie_tmdb_cache[cache_key] = tmdb_info
return tmdb_info
def has_playable(item: ParsedContent) -> bool:
return find_playable_file(item.path) is not None
async def has_playable(item: ParsedContent) -> bool:
return await find_playable_file(item.path) is not None
# Filter movies with playable files
valid_movies = [
item for item in categories[ContentType.MOVIE] if has_playable(item)
]
valid_movies = []
for item in categories[ContentType.MOVIE]:
if await has_playable(item):
valid_movies.append(item)
skipped = len(categories[ContentType.MOVIE]) - len(valid_movies)
if skipped > 0:
print(f" Skipped {skipped} movie torrents with no playable video files")
logger.debug(" Skipped %d movie torrents with no playable video files", skipped)
# Group by title+year
movie_groups: Dict[str, List[ParsedContent]] = {}
@@ -320,15 +329,16 @@ async def _process_movies(
tmdb_movie_groups: Dict[int, Dict] = {}
no_tmdb_movie_groups: Dict[str, Dict] = {}
logger.info(
" Processing %d unique movies (%d total versions)...",
len(movie_groups),
len(categories[ContentType.MOVIE]),
)
if movie_groups:
logger.info(
" Processing %d unique movies (%d total versions)...",
len(movie_groups),
len(categories[ContentType.MOVIE]),
) if movie_groups else None
for idx, (movie_key, items) in enumerate(movie_groups.items(), 1):
first_item = items[0]
logger.info(
logger.debug(
" [%d/%d] %s (%s)",
idx,
len(movie_groups),
@@ -337,6 +347,9 @@ async def _process_movies(
)
tmdb_info = await get_movie_tmdb(first_item.title, first_item.year)
# Yield to event loop so HTTP requests stay responsive
if idx % 20 == 0:
await asyncio.sleep(0)
if tmdb_info and tmdb_info.tmdb_id:
if tmdb_info.tmdb_id not in tmdb_movie_groups:
@@ -371,10 +384,10 @@ async def _process_movies(
# Find/download cover
cover_path = None
if fetch_covers:
cover_path = find_cover_image(display_title, year, "movie", cover_dir)
cover_path = await find_cover_image(display_title, year, "movie", cover_dir)
if not cover_path:
for tt in torrent_titles:
cover_path = find_cover_image(tt, year, "movie", cover_dir)
cover_path = await find_cover_image(tt, year, "movie", cover_dir)
if cover_path:
break
if not cover_path and tmdb_info.poster_path:
@@ -382,19 +395,30 @@ async def _process_movies(
tmdb_info.poster_path, display_title, year, "movie", cover_dir
)
versions = [_build_version_info(item, media_root) for item in items]
sort_by_quality(versions)
versions = {}
for item in items:
relpath = make_relative_path(str(item.path), media_root)
version = await _build_version_info(item, media_root)
versions[relpath] = version
sort_by_quality(list(versions.values()))
# Queue showreel generation
showreel_paths = []
showreel_task = None
if generate_showreels and versions:
best_playable = versions[0].playable_file
if best_playable:
# Find the best version for showreel (highest quality)
best_relpath = max(versions.keys(), key=lambda k: (
RESOLUTION_PRIORITY.get(versions[k].resolution or "", 0),
versions[k].size or 0,
k
))
best_version = versions[best_relpath]
if best_version.playable_file:
abs_playable = (
str(Path(media_root) / best_playable)
str(Path(media_root) / best_version.playable_file)
if media_root
else best_playable
else best_version.playable_file
)
media_folder = get_media_folder_path(
display_title, year, "movie", cover_dir
@@ -411,10 +435,7 @@ async def _process_movies(
tmdb_info.backdrop_path, display_title, year, "movie", cover_dir
)
different_titles = [
t for t in torrent_titles if t.lower() != display_title.lower()
]
version_timestamps = [v.newest for v in versions if v.newest]
version_timestamps = [v.newest for v in versions.values() if v.newest]
newest = max(version_timestamps) if version_timestamps else None
movie = Movie(
@@ -422,7 +443,6 @@ async def _process_movies(
title=display_title,
original_title=tmdb_info.original_title,
alternative_titles=tmdb_info.alternative_titles,
torrent_titles=different_titles if different_titles else None,
year=year,
newest=newest,
cover_path=make_relative_path(cover_path, media_root),
@@ -455,33 +475,43 @@ async def _process_movies(
item_id = hashlib.md5(f"movie:{title}:{year}".encode()).hexdigest()[:12]
cover_path = (
find_cover_image(title, year, "movie", cover_dir) if fetch_covers else None
await find_cover_image(title, year, "movie", cover_dir) if fetch_covers else None
)
versions = [_build_version_info(item, media_root) for item in items]
sort_by_quality(versions)
versions = {}
for item in items:
relpath = make_relative_path(str(item.path), media_root)
version = await _build_version_info(item, media_root)
versions[relpath] = version
sort_by_quality(list(versions.values()))
showreel_paths = []
showreel_task = None
if generate_showreels and versions:
best_playable = versions[0].playable_file
if best_playable:
# Find the best version for showreel (highest quality)
best_relpath = max(versions.keys(), key=lambda k: (
RESOLUTION_PRIORITY.get(versions[k].resolution or "", 0),
versions[k].size or 0,
k
))
best_version = versions[best_relpath]
if best_version.playable_file and not best_version.playable_file.endswith(".bdmv"):
abs_playable = (
str(Path(media_root) / best_playable)
str(Path(media_root) / best_version.playable_file)
if media_root
else best_playable
else best_version.playable_file
)
if not abs_playable.endswith(".bdmv"):
media_folder = get_media_folder_path(
title, year, "movie", cover_dir
)
showreel_paths = get_expected_showreel_paths(
media_folder,
media_root=Path(media_root) if media_root else None,
)
showreel_task = (abs_playable, media_folder, title)
media_folder = get_media_folder_path(
title, year, "movie", cover_dir
)
showreel_paths = get_expected_showreel_paths(
media_folder,
media_root=Path(media_root) if media_root else None,
)
showreel_task = (abs_playable, media_folder, title)
version_timestamps = [v.newest for v in versions if v.newest]
version_timestamps = [v.newest for v in versions.values() if v.newest]
newest = max(version_timestamps) if version_timestamps else None
movie = Movie(
@@ -520,15 +550,16 @@ async def _process_series(
series_tmdb_cache[cache_key] = tmdb_info
return tmdb_info
def has_video_content(item: ParsedContent) -> bool:
if find_playable_file(item.path):
async def has_video_content(item: ParsedContent) -> bool:
if await find_playable_file(item.path):
return True
return len(find_episode_files(item.path)) > 0
return len(await find_episode_files(item.path)) > 0
# Filter series with video content
valid_series = [
item for item in categories[ContentType.SERIES] if has_video_content(item)
]
valid_series = []
for item in categories[ContentType.SERIES]:
if await has_video_content(item):
valid_series.append(item)
skipped = len(categories[ContentType.SERIES]) - len(valid_series)
if skipped > 0:
logger.info(
@@ -547,17 +578,21 @@ async def _process_series(
tmdb_groups: Dict[int, Dict] = {}
no_tmdb_groups: Dict[str, Dict] = {}
logger.info(
" Processing %d unique series (%d total entries)...",
len(series_groups),
len(categories[ContentType.SERIES]),
)
if series_groups:
logger.info(
" Processing %d unique series (%d total entries)...",
len(series_groups),
len(categories[ContentType.SERIES]),
)
for idx, (series_key, items) in enumerate(series_groups.items(), 1):
first_item = items[0]
logger.info(" [%d/%d] %s", idx, len(series_groups), first_item.title)
logger.debug(" [%d/%d] %s", idx, len(series_groups), first_item.title)
tmdb_info = await get_series_tmdb(first_item.title)
# Yield to event loop so HTTP requests stay responsive
if idx % 20 == 0:
await asyncio.sleep(0)
if tmdb_info and tmdb_info.tmdb_id:
if tmdb_info.tmdb_id not in tmdb_groups:
@@ -583,17 +618,17 @@ async def _process_series(
display_title = tmdb_info.title
series_id = hashlib.md5(f"series:{tmdb_id}".encode()).hexdigest()[:12]
logger.info(" [%d/%d] %s", series_idx, len(tmdb_groups), display_title)
logger.debug(" [%d/%d] %s", series_idx, len(tmdb_groups), display_title)
series_folder = get_media_folder_path(display_title, None, "series", cover_dir)
# Find/download cover
cover_path = None
if fetch_covers:
cover_path = find_cover_image(display_title, None, "series", cover_dir)
cover_path = await find_cover_image(display_title, None, "series", cover_dir)
if not cover_path:
for tt in torrent_titles:
cover_path = find_cover_image(tt, None, "series", cover_dir)
cover_path = await find_cover_image(tt, None, "series", cover_dir)
if cover_path:
break
if not cover_path and tmdb_info.poster_path:
@@ -609,7 +644,7 @@ async def _process_series(
)
# Collect and build episode data
all_episode_files = _collect_episode_files(items)
all_episode_files = await _collect_episode_files(items)
ep_reel_tasks: List[Tuple[str, Path, int, int, str]] = []
seasons_data = await _build_seasons_data(
all_episode_files,
@@ -627,10 +662,10 @@ async def _process_series(
logger.info(" Skipping %s - no episodes found", display_title)
continue
different_titles = [
different_titles = sorted(
t for t in torrent_titles if t.lower() != display_title.lower()
]
item_timestamps = [get_added_timestamp(item.path) for item in items]
)
item_timestamps = [await get_added_timestamp(item.path) for item in items]
item_timestamps = [t for t in item_timestamps if t is not None]
newest = max(item_timestamps) if item_timestamps else None
@@ -638,7 +673,7 @@ async def _process_series(
id=series_id,
title=display_title,
original_title=tmdb_info.original_title,
torrent_titles=different_titles if different_titles else None,
alternative_titles=different_titles if different_titles else None,
newest=newest,
cover_path=make_relative_path(cover_path, media_root),
backdrop_path=make_relative_path(backdrop_path, media_root),
@@ -670,11 +705,11 @@ async def _process_series(
series_id = hashlib.md5(f"series:{title}".encode()).hexdigest()[:12]
cover_path = (
find_cover_image(title, None, "series", cover_dir) if fetch_covers else None
await find_cover_image(title, None, "series", cover_dir) if fetch_covers else None
)
series_folder = get_media_folder_path(title, None, "series", cover_dir)
all_episode_files = _collect_episode_files(items)
all_episode_files = await _collect_episode_files(items)
ep_reel_tasks: List[Tuple[str, Path, int, int, str]] = []
seasons_data = await _build_seasons_data(
all_episode_files,
@@ -692,7 +727,7 @@ async def _process_series(
logger.info(" Skipping %s - no episodes found", title)
continue
item_timestamps = [get_added_timestamp(item.path) for item in items]
item_timestamps = [await get_added_timestamp(item.path) for item in items]
item_timestamps = [t for t in item_timestamps if t is not None]
newest = max(item_timestamps) if item_timestamps else None
+1 -14
View File
@@ -21,20 +21,7 @@ class ContentHash:
path: Path
hash: str
_size: Optional[int] = None
@property
def size(self) -> int:
"""Get the size, computing it lazily if needed."""
if self._size is None:
from hivescan.utils import get_directory_size
self._size = get_directory_size(self.path)
return self._size
@size.setter
def size(self, value: int) -> None:
self._size = value
size: int = 0
@classmethod
def from_path(cls, path: Path) -> "ContentHash":
+3 -2
View File
@@ -5,6 +5,7 @@ from pathlib import Path
from typing import Optional, Tuple
import PTN
from aiopathlib import AsyncPath
from hivescan.models import ContentHash, ContentType, ParsedContent
@@ -22,7 +23,7 @@ def determine_content_type(parsed: dict) -> ContentType:
return ContentType.OTHER
def parse_download(path: Path) -> ParsedContent:
async def parse_download(path: Path) -> ParsedContent:
"""Parse a downloaded torrent directory/file name."""
name = path.name
parsed = PTN.parse(name)
@@ -44,7 +45,7 @@ def parse_download(path: Path) -> ParsedContent:
episode_name=parsed.get("episodeName"),
encoder=parsed.get("encoder"),
language=parsed.get("language"),
is_directory=path.is_dir(),
is_directory=await AsyncPath(path).is_dir(),
raw_parsed=parsed,
content_hash=content_hash,
)
+47 -34
View File
@@ -1,8 +1,11 @@
"""File system scanning functions."""
import asyncio
import glob
from pathlib import Path
from typing import Dict, Iterator, List, Optional, Tuple
from typing import Dict, List, Optional, Tuple
from aiopathlib import AsyncPath
from hivescan.models import ContentType, ParsedContent
from hivescan.parsing import parse_download, parse_episode_from_filename
@@ -28,20 +31,23 @@ _episode_files_cache: Dict[str, Dict[Tuple[int, int], List[Tuple[str, int]]]] =
_playable_file_cache: Dict[str, Optional[str]] = {}
def scan_downloads(base_pattern: str) -> Iterator[ParsedContent]:
async def scan_downloads(base_pattern: str) -> List[ParsedContent]:
"""
Scan download directories matching the pattern.
Args:
base_pattern: Glob pattern for finding download directories
Yields:
ParsedContent objects for each found download
Returns:
List of ParsedContent objects for each found download
"""
exclude_patterns = [".torrents", "incomplete", ".incomplete"]
results: List[ParsedContent] = []
for path_str in glob.glob(base_pattern):
paths = await asyncio.to_thread(glob.glob, base_pattern)
for path_str in paths:
path = Path(path_str)
ap = AsyncPath(path)
if path.name.startswith("."):
continue
@@ -49,10 +55,12 @@ def scan_downloads(base_pattern: str) -> Iterator[ParsedContent]:
if any(excl.lower() in path.name.lower() for excl in exclude_patterns):
continue
if not path.exists():
if not await ap.exists():
continue
yield parse_download(path)
results.append(await parse_download(path))
return results
def categorize_downloads(
@@ -71,7 +79,7 @@ def categorize_downloads(
return categories
def find_episode_files(path: Path) -> Dict[Tuple[int, int], List[Tuple[str, int]]]:
async def find_episode_files(path: Path) -> Dict[Tuple[int, int], List[Tuple[str, int]]]:
"""
Find all episode video files in a directory.
@@ -86,33 +94,35 @@ def find_episode_files(path: Path) -> Dict[Tuple[int, int], List[Tuple[str, int]
return _episode_files_cache[cache_key]
episodes: Dict[Tuple[int, int], List[Tuple[str, int]]] = {}
ap = AsyncPath(path)
if path.is_file():
if await ap.is_file():
if path.suffix.lower() in VIDEO_EXTENSIONS:
ep_info = parse_episode_from_filename(path.name)
if ep_info:
episodes[ep_info] = [(str(path), path.stat().st_size)]
episodes[ep_info] = [(str(path), (await ap.stat()).st_size)]
_episode_files_cache[cache_key] = episodes
return episodes
try:
for f in path.rglob("*"):
if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS:
if "sample" in f.name.lower():
for f in ap.rglob("*"):
af = AsyncPath(f)
if await af.is_file() and Path(f).suffix.lower() in VIDEO_EXTENSIONS:
if "sample" in Path(f).name.lower():
continue
ep_info = parse_episode_from_filename(f.name)
ep_info = parse_episode_from_filename(Path(f).name)
if ep_info:
if ep_info not in episodes:
episodes[ep_info] = []
episodes[ep_info].append((str(f), f.stat().st_size))
except OSError, PermissionError:
episodes[ep_info].append((str(f), (await af.stat()).st_size))
except (OSError, PermissionError):
pass
_episode_files_cache[cache_key] = episodes
return episodes
def find_playable_file(path: Path) -> Optional[str]:
async def find_playable_file(path: Path) -> Optional[str]:
"""
Find the main playable media file in a directory.
@@ -123,7 +133,9 @@ def find_playable_file(path: Path) -> Optional[str]:
if cache_key in _playable_file_cache:
return _playable_file_cache[cache_key]
if path.is_file():
ap = AsyncPath(path)
if await ap.is_file():
if path.suffix.lower() in VIDEO_EXTENSIONS:
result = str(path)
_playable_file_cache[cache_key] = result
@@ -133,32 +145,33 @@ def find_playable_file(path: Path) -> Optional[str]:
# Check for Blu-ray disc structure
bdmv_index = path / "BDMV" / "index.bdmv"
if bdmv_index.exists():
if await AsyncPath(bdmv_index).exists():
result = str(bdmv_index)
_playable_file_cache[cache_key] = result
return result
# Check nested Blu-ray structure (e.g., MovieName/DISC1/BDMV/)
try:
for subdir in path.iterdir():
if subdir.is_dir():
nested_bdmv = subdir / "BDMV" / "index.bdmv"
if nested_bdmv.exists():
for subdir in ap.iterdir():
if await AsyncPath(subdir).is_dir():
nested_bdmv = Path(subdir) / "BDMV" / "index.bdmv"
if await AsyncPath(nested_bdmv).exists():
result = str(nested_bdmv)
_playable_file_cache[cache_key] = result
return result
except OSError, PermissionError:
except (OSError, PermissionError):
pass
# Find largest video file
video_files = []
try:
for f in path.rglob("*"):
if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS:
if "sample" in f.name.lower():
for f in ap.rglob("*"):
af = AsyncPath(f)
if await af.is_file() and Path(f).suffix.lower() in VIDEO_EXTENSIONS:
if "sample" in Path(f).name.lower():
continue
video_files.append((f, f.stat().st_size))
except OSError, PermissionError:
video_files.append((str(f), (await af.stat()).st_size))
except (OSError, PermissionError):
pass
if not video_files:
@@ -166,29 +179,29 @@ def find_playable_file(path: Path) -> Optional[str]:
return None
video_files.sort(key=lambda x: x[1], reverse=True)
result = str(video_files[0][0])
result = video_files[0][0]
_playable_file_cache[cache_key] = result
return result
def find_cover_image(
async def find_cover_image(
title: str, year: Optional[int], media_type: str, cover_dir: Path
) -> Optional[str]:
"""Find a cover image for the given media item."""
media_folder = get_media_folder_path(title, year, media_type, cover_dir)
cover_path = media_folder / "cover.jpg"
if cover_path.exists():
if await AsyncPath(cover_path).exists():
return str(cover_path)
# Legacy structure fallback
subdir = "movies" if media_type == "movie" else "series"
if media_type == "movie" and year:
legacy_path = cover_dir / subdir / f"{sanitize_filename(title)} ({year}).jpg"
if legacy_path.exists():
if await AsyncPath(legacy_path).exists():
return str(legacy_path)
legacy_path = cover_dir / subdir / f"{sanitize_filename(title)}.jpg"
if legacy_path.exists():
if await AsyncPath(legacy_path).exists():
return str(legacy_path)
return None
+82 -47
View File
@@ -23,6 +23,7 @@ from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
import msgspec
from aiopathlib import AsyncPath
from hivescan.index_store import IndexStore
from hivescan.indexer import _process_movies, _process_series
from hivescan.structs import MsgspecResponse, ScanRequest, StatusResponse, TaskInfo
@@ -54,6 +55,7 @@ _scan_task: Optional[asyncio.Task] = None
_showreel_queue: asyncio.Queue = asyncio.Queue()
_showreel_worker_task: Optional[asyncio.Task] = None
_rescan_worker_task: Optional[asyncio.Task] = None
_seen_mtimes: dict[str, int] = {}
# ---------------------------------------------------------------------------
@@ -77,7 +79,7 @@ async def lifespan(app: FastAPI):
pattern = pattern.strip()
if not pattern:
continue
expanded = glob.glob(pattern)
expanded = await asyncio.to_thread(glob.glob, pattern)
if expanded:
all_paths.extend(Path(p) for p in expanded)
else:
@@ -89,18 +91,18 @@ async def lifespan(app: FastAPI):
OUTPUT_DIR = Path(os.environ["HIVESCAN_OUTPUT"])
MEDIA_ROOT = OUTPUT_DIR.parent
else:
MEDIA_ROOT = find_common_root(all_paths)
MEDIA_ROOT = await find_common_root(all_paths)
if MEDIA_ROOT is None:
logger.error("Cannot determine common root; set HIVESCAN_OUTPUT")
sys.exit(1)
OUTPUT_DIR = MEDIA_ROOT / DEFAULT_OUTPUT_FOLDER
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
await AsyncPath(OUTPUT_DIR).mkdir(parents=True, exist_ok=True)
set_cache_dir(OUTPUT_DIR / ".tmdb-cache")
# Initialise index store and load snapshot
store = IndexStore(OUTPUT_DIR / "index.json", media_root=str(MEDIA_ROOT))
store.load_snapshot()
await store.load_snapshot()
logger.info(
"Index store ready: %d movies, %d series (from snapshot)",
len(store.movies),
@@ -226,42 +228,65 @@ async def _rescan_loop():
logger.exception("Rescan loop error")
async def _discover_downloads(paths_to_scan: List[str]) -> List[ParsedContent]:
"""Walk the filesystem and parse all downloads, skipping unchanged torrents."""
downloads: List[ParsedContent] = []
media_root_str = str(MEDIA_ROOT) if MEDIA_ROOT else None
for pattern in paths_to_scan:
p = Path(pattern)
ap = AsyncPath(p)
if await ap.is_dir():
for item in ap.iterdir():
if not Path(item).name.startswith("."):
relpath = make_relative_path(str(item), media_root_str)
stat_info = await AsyncPath(item).stat()
mtime = int(stat_info.st_mtime)
if relpath in _seen_mtimes and _seen_mtimes[relpath] == mtime:
continue # skip unchanged
_seen_mtimes[relpath] = mtime
downloads.append(await parse_download(Path(item)))
elif await ap.exists():
relpath = make_relative_path(str(p), media_root_str)
stat_info = await ap.stat()
mtime = int(stat_info.st_mtime)
if relpath in _seen_mtimes and _seen_mtimes[relpath] == mtime:
continue
_seen_mtimes[relpath] = mtime
downloads.append(await parse_download(p))
return downloads
async def _run_scan(override_paths: Optional[List[str]] = None):
"""
Full scan pipeline:
1. Walk filesystem, parse torrents
1. Walk filesystem, parse torrents (in thread)
2. Categorise → movies / series
3. Iterate async generators, upsert each item into IndexStore
4. Queue showreel tasks
"""
task_id = f"scan-{uuid.uuid4().hex[:8]}"
logger.info("Scan started (%s)", task_id)
store.broadcast_task(
TaskInfo(
id=task_id, status="running", progress=0, detail="Scanning filesystem..."
)
)
paths_to_scan = override_paths or SCAN_PATHS
media_root_str = str(MEDIA_ROOT) if MEDIA_ROOT else None
try:
# 1. Discover downloads (sync filesystem walk — fast enough)
downloads: List[ParsedContent] = []
for pattern in paths_to_scan:
p = Path(pattern)
if p.is_dir():
for item in p.iterdir():
if not item.name.startswith("."):
downloads.append(parse_download(item))
elif p.exists():
downloads.append(parse_download(p))
# 1. Discover downloads (now async)
downloads = await _discover_downloads(paths_to_scan)
logger.info("Found %d items to process", len(downloads))
if downloads:
logger.info("Scan started (%s)", task_id)
store.broadcast_task(
TaskInfo(
id=task_id, status="running", progress=0, detail="Scanning filesystem..."
)
)
logger.info("Found %d items to process", len(downloads))
categories = categorize_downloads(downloads)
total = len(categories[ContentType.MOVIE]) + len(categories[ContentType.SERIES])
processed = 0
changed = 0
# 2. Process movies
store.broadcast_task(
@@ -276,19 +301,23 @@ async def _run_scan(override_paths: Optional[List[str]] = None):
generate_showreels=True,
media_root=media_root_str,
):
store.upsert_movie(movie)
was_changed = store.upsert_movie(movie)
if was_changed:
changed += 1
logger.info(" Updated movie: %s", movie.title)
if showreel_task:
await _showreel_queue.put(("movie", showreel_task, movie.id))
processed += 1
progress = processed / total if total else 1
store.broadcast_task(
TaskInfo(
id=task_id,
status="running",
progress=round(progress, 3),
detail=movie.title,
if was_changed:
store.broadcast_task(
TaskInfo(
id=task_id,
status="running",
progress=round(progress, 3),
detail=movie.title,
)
)
)
# 3. Process series
store.broadcast_task(
@@ -306,29 +335,35 @@ async def _run_scan(override_paths: Optional[List[str]] = None):
generate_showreels=True,
media_root=media_root_str,
):
store.upsert_series(series)
was_changed = store.upsert_series(series)
if was_changed:
changed += 1
logger.info(" Updated series: %s", series.title)
for task in ep_reel_tasks:
await _showreel_queue.put(("episode", task, series.id))
processed += 1
progress = processed / total if total else 1
store.broadcast_task(
TaskInfo(
id=task_id,
status="running",
progress=round(progress, 3),
detail=series.title,
if was_changed:
store.broadcast_task(
TaskInfo(
id=task_id,
status="running",
progress=round(progress, 3),
detail=series.title,
)
)
)
store.broadcast_task(
TaskInfo(id=task_id, status="completed", progress=1, detail="Scan complete")
)
logger.info(
"Scan complete (%s): %d movies, %d series",
task_id,
len(store.movies),
len(store.series),
)
if downloads:
logger.info(
"Scan complete (%s): %d movies, %d series (%d changed)",
task_id,
len(store.movies),
len(store.series),
changed,
)
except asyncio.CancelledError:
store.broadcast_task(
@@ -359,7 +394,7 @@ async def _showreel_worker():
if kind == "movie":
video_path, media_folder, title = task_data
if movie_showreels_exist(media_folder):
if await movie_showreels_exist(media_folder):
_showreel_queue.task_done()
continue
store.broadcast_task(
@@ -397,7 +432,7 @@ async def _showreel_worker():
video_path, media_folder, season_num, episode_num, series_title = (
task_data
)
if episode_reel_exists(media_folder, season_num, episode_num):
if await episode_reel_exists(media_folder, season_num, episode_num):
_showreel_queue.task_done()
continue
ep_code = f"S{season_num:02d}E{episode_num:02d}"
+19 -17
View File
@@ -15,6 +15,8 @@ from collections import Counter
from pathlib import Path
from typing import Optional
from aiopathlib import AsyncPath
logger = logging.getLogger("hivescan.showreel")
@@ -78,19 +80,19 @@ def get_expected_episode_reel_path(
return str(output_path)
def movie_showreels_exist(
async def movie_showreels_exist(
media_folder: Path, timestamps: list[int] = SHOWREEL_TIMESTAMPS
) -> bool:
"""Check if all showreel files for a movie already exist."""
for reel_num in range(1, len(timestamps) + 1):
if not (media_folder / f"reel{reel_num}.webm").exists():
if not await AsyncPath(media_folder / f"reel{reel_num}.webm").exists():
return False
return True
def episode_reel_exists(media_folder: Path, season_num: int, episode_num: int) -> bool:
async def episode_reel_exists(media_folder: Path, season_num: int, episode_num: int) -> bool:
"""Check if an episode reel file already exists."""
return (media_folder / f"S{season_num:02d}E{episode_num:02d}.webm").exists()
return await AsyncPath(media_folder / f"S{season_num:02d}E{episode_num:02d}.webm").exists()
def get_bluray_uri(video_path: str) -> Optional[str]:
@@ -542,7 +544,7 @@ async def generate_showreel_images(
if bluray_uri:
ffmpeg_input = bluray_uri
else:
if not Path(video_path).exists():
if not await AsyncPath(video_path).exists():
return []
ffmpeg_input = video_path
@@ -552,7 +554,7 @@ async def generate_showreel_images(
for reel_num in range(1, len(timestamps) + 1):
output_filename = f"reel{reel_num}.webm"
output_path = media_folder / output_filename
if output_path.exists():
if await AsyncPath(output_path).exists():
existing_paths.append(str(output_path))
else:
all_exist = False
@@ -561,7 +563,7 @@ async def generate_showreel_images(
if all_exist and existing_paths:
return existing_paths
media_folder.mkdir(parents=True, exist_ok=True)
await AsyncPath(media_folder).mkdir(parents=True, exist_ok=True)
# Check video duration to avoid seeking past the end
duration = await get_video_duration(ffmpeg_input)
@@ -598,7 +600,7 @@ async def generate_showreel_images(
output_path = media_folder / output_filename
# Skip if already exists
if output_path.exists():
if await AsyncPath(output_path).exists():
generated_paths.append(str(output_path))
if on_progress:
on_progress(reel_num)
@@ -655,16 +657,16 @@ async def generate_showreel_images(
)
await asyncio.wait_for(proc.communicate(), timeout=120)
if proc.returncode == 0 and output_path.exists():
if proc.returncode == 0 and await AsyncPath(output_path).exists():
generated_paths.append(str(output_path))
if on_progress:
on_progress(reel_num)
else:
output_path.unlink(missing_ok=True)
await AsyncPath(output_path).unlink(missing_ok=True)
# Abort remaining reels - if first one fails, others likely will too
break
except BaseException as e:
output_path.unlink(missing_ok=True)
await AsyncPath(output_path).unlink(missing_ok=True)
if isinstance(e, (KeyboardInterrupt, SystemExit, asyncio.CancelledError)):
raise
logger.error(
@@ -709,18 +711,18 @@ async def generate_episode_reel(
if bluray_uri:
ffmpeg_input = bluray_uri
else:
if not Path(video_path).exists():
if not await AsyncPath(video_path).exists():
return None
ffmpeg_input = video_path
media_folder.mkdir(parents=True, exist_ok=True)
await AsyncPath(media_folder).mkdir(parents=True, exist_ok=True)
# Normalize episode code to SxxExx format
output_filename = f"S{season_num:02d}E{episode_num:02d}.webm"
output_path = media_folder / output_filename
# Skip if already exists
if output_path.exists():
if await AsyncPath(output_path).exists():
return str(output_path)
# Check video duration
@@ -797,13 +799,13 @@ async def generate_episode_reel(
)
await asyncio.wait_for(proc.communicate(), timeout=120)
if proc.returncode == 0 and output_path.exists():
if proc.returncode == 0 and await AsyncPath(output_path).exists():
return str(output_path)
else:
output_path.unlink(missing_ok=True)
await AsyncPath(output_path).unlink(missing_ok=True)
return None
except BaseException as e:
output_path.unlink(missing_ok=True)
await AsyncPath(output_path).unlink(missing_ok=True)
if isinstance(e, (KeyboardInterrupt, SystemExit, asyncio.CancelledError)):
raise
episode_code = f"S{season_num:02d}E{episode_num:02d}"
+7 -9
View File
@@ -97,9 +97,9 @@ class TMDbInfo(msgspec.Struct):
class MovieVersion(msgspec.Struct):
"""One release/torrent of a movie."""
"""One release/torrent of a movie, keyed by relative torrent path."""
path: str | None = None
torrent_title: str | None = None
playable_file: str | None = None
resolution: str | None = None
quality: str | None = None
@@ -111,9 +111,9 @@ class MovieVersion(msgspec.Struct):
class EpisodeRelease(msgspec.Struct):
"""One release/torrent file of an episode."""
"""One release/torrent file of an episode, keyed by relative torrent path."""
path: str | None = None
torrent_title: str | None = None
playable_file: str | None = None
resolution: str | None = None
quality: str | None = None
@@ -135,7 +135,7 @@ class Episode(msgspec.Struct):
rating: float | None = None
director: str | None = None
reel_image: str | None = None
releases: list[EpisodeRelease] = []
releases: dict[str, EpisodeRelease] = {}
class Season(msgspec.Struct):
@@ -157,13 +157,12 @@ class Movie(msgspec.Struct):
title: str
original_title: str | None = None
alternative_titles: list[str] | None = None
torrent_titles: list[str] | None = None
year: int | None = None
newest: int | None = None
cover_path: str | None = None
backdrop_path: str | None = None
showreel_images: list[str] | None = None
versions: list[MovieVersion] = []
versions: dict[str, MovieVersion] = {}
tmdb_id: int | None = None
tmdb_title: str | None = None
rating: float | None = None
@@ -188,7 +187,6 @@ class Series(msgspec.Struct):
title: str
original_title: str | None = None
alternative_titles: list[str] | None = None
torrent_titles: list[str] | None = None
newest: int | None = None
cover_path: str | None = None
backdrop_path: str | None = None
@@ -229,7 +227,7 @@ class MediaStats(msgspec.Struct):
class IndexSnapshot(msgspec.Struct):
"""On-disk recovery snapshot of the full index."""
version: int = 5
version: int = 6
generated_at: str = ""
media_root: str | None = None
stats: MediaStats = msgspec.UNSET # type: ignore[assignment]
+19 -18
View File
@@ -13,6 +13,7 @@ from pathlib import Path
from typing import Dict, Optional
import httpx
from aiopathlib import AsyncPath
from hivescan.structs import (
CastMember,
@@ -72,30 +73,30 @@ def _get_cache_path(endpoint: str, params: Dict[str, str]) -> Path:
return _get_cache_dir() / f"{cache_hash}.json"
def _load_from_cache(cache_path: Path):
async def _load_from_cache(cache_path: Path):
"""Load cached response. Returns _NOT_FOUND if not cached."""
if not cache_path.exists():
if not await AsyncPath(cache_path).exists():
return _NOT_FOUND
try:
with open(cache_path, "r", encoding="utf-8") as f:
data = json.load(f)
# Handle cached "no results" / errors
if data.get("_cached_none"):
return None
return data
text = await AsyncPath(cache_path).read_text(encoding="utf-8")
data = json.loads(text)
# Handle cached "no results" / errors
if data.get("_cached_none"):
return None
return data
except Exception:
return _NOT_FOUND
def _save_to_cache(cache_path: Path, data: Optional[Dict]):
async def _save_to_cache(cache_path: Path, data: Optional[Dict]):
"""Save response to cache."""
try:
_get_cache_dir().mkdir(parents=True, exist_ok=True)
with open(cache_path, "w", encoding="utf-8") as f:
if data is None:
json.dump({"_cached_none": True}, f)
else:
json.dump(data, f)
await AsyncPath(_get_cache_dir()).mkdir(parents=True, exist_ok=True)
if data is None:
text = json.dumps({"_cached_none": True})
else:
text = json.dumps(data)
await AsyncPath(cache_path).write_text(text, encoding="utf-8")
except Exception:
pass # Cache write failures are not critical
@@ -111,7 +112,7 @@ async def tmdb_api_request(
# Check cache first (before adding API key to params for cache key)
cache_path = _get_cache_path(endpoint, params)
cached = _load_from_cache(cache_path)
cached = await _load_from_cache(cache_path)
if cached is not _NOT_FOUND:
return cached
@@ -132,11 +133,11 @@ async def tmdb_api_request(
response.raise_for_status()
data = response.json()
# Cache immediately after receiving response
_save_to_cache(cache_path, data)
await _save_to_cache(cache_path, data)
return data
except httpx.HTTPStatusError:
# Cache the failure (None) to avoid retrying
_save_to_cache(cache_path, None)
await _save_to_cache(cache_path, None)
return None
except Exception:
# Don't cache network errors - they may be transient
+21 -16
View File
@@ -5,6 +5,8 @@ import time
from pathlib import Path
from typing import Optional, List
from aiopathlib import AsyncPath
# Default output folder name (created at common root of scanned paths)
DEFAULT_OUTPUT_FOLDER = ".mediahive"
@@ -23,7 +25,7 @@ RESOLUTION_PRIORITY = {
}
def get_added_timestamp(path: Path) -> Optional[int]:
async def get_added_timestamp(path: Path) -> Optional[int]:
"""
Get the timestamp when a torrent was added to the collection.
@@ -35,12 +37,13 @@ def get_added_timestamp(path: Path) -> Optional[int]:
Returns:
Unix timestamp as int, or None if path doesn't exist
"""
ap = AsyncPath(path)
try:
stat_info = path.stat()
except OSError, PermissionError:
stat_info = await ap.stat()
except (OSError, PermissionError):
return None
if path.is_dir():
if await ap.is_dir():
return int(stat_info.st_ctime)
now = time.time()
@@ -52,16 +55,17 @@ def get_added_timestamp(path: Path) -> Optional[int]:
return int(atime)
def get_directory_size(path: Path) -> int:
async def get_directory_size(path: Path) -> int:
"""Calculate total size of a directory recursively."""
ap = AsyncPath(path)
total = 0
try:
if path.is_file():
return path.stat().st_size
for item in path.rglob("*"):
if item.is_file():
total += item.stat().st_size
except OSError, PermissionError:
if await ap.is_file():
return (await ap.stat()).st_size
for item in ap.rglob("*"):
if await AsyncPath(item).is_file():
total += (await AsyncPath(item).stat()).st_size
except (OSError, PermissionError):
pass
return total
@@ -75,7 +79,7 @@ def format_size(size_bytes: int) -> str:
return f"{size_bytes:.2f} PB"
def find_common_root(paths: List[Path]) -> Optional[Path]:
async def find_common_root(paths: List[Path]) -> Optional[Path]:
"""
Find the common root directory for a list of paths.
@@ -95,10 +99,10 @@ def find_common_root(paths: List[Path]) -> Optional[Path]:
for p in resolved:
# Find the first existing parent to get device info
check_path = p
while not check_path.exists() and check_path.parent != check_path:
while not await AsyncPath(check_path).exists() and check_path.parent != check_path:
check_path = check_path.parent
if check_path.exists():
devices.add(os.stat(check_path).st_dev)
if await AsyncPath(check_path).exists():
devices.add((await AsyncPath(check_path).stat()).st_dev)
if len(devices) > 1:
# Paths are on different devices/drives
@@ -109,7 +113,7 @@ def find_common_root(paths: List[Path]) -> Optional[Path]:
# Find common path prefix
if len(resolved) == 1:
# Single path - use its parent as root
return resolved[0].parent if resolved[0].is_file() else resolved[0]
return resolved[0].parent if await AsyncPath(resolved[0]).is_file() else resolved[0]
# Get parts of each path
all_parts = [p.parts for p in resolved]
@@ -186,6 +190,7 @@ def sort_by_quality(items: list, reverse: bool = True) -> None:
key=lambda v: (
RESOLUTION_PRIORITY.get(_val(v, "resolution", "") or "", 0),
_val(v, "size", 0) or 0,
_val(v, "path", "") or "",
),
reverse=reverse,
)
+1
View File
@@ -6,6 +6,7 @@ readme = "README.md"
requires-python = ">=3.14"
dependencies = [
"aiofiles>=25.1.0",
"aiopathlib>=0.6.0",
"bencodepy>=0.9.5",
"fastapi-vue>=0.5.2",
"fastapi[standard]>=0.128.0",