Async non-blocking scanning, don't rerun when items haven't been modified.
This commit is contained in:
+10
-7
@@ -4,6 +4,8 @@ import httpx
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from aiopathlib import AsyncPath
|
||||
|
||||
from hivescan.utils import get_media_folder_path
|
||||
|
||||
|
||||
@@ -32,15 +34,16 @@ async def _download_image(
|
||||
url: str, output_path: Path, description: str
|
||||
) -> Optional[str]:
|
||||
"""Download an image from URL to output path."""
|
||||
if output_path.exists():
|
||||
ap = AsyncPath(output_path)
|
||||
if await ap.exists():
|
||||
return str(output_path)
|
||||
|
||||
try:
|
||||
client = _get_image_client()
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_bytes(response.content)
|
||||
await AsyncPath(output_path.parent).mkdir(parents=True, exist_ok=True)
|
||||
await ap.write_bytes(response.content)
|
||||
return str(output_path)
|
||||
except Exception as e:
|
||||
print(f" Failed to download {description}: {e}")
|
||||
@@ -62,7 +65,7 @@ async def download_cover_image(
|
||||
media_folder = get_media_folder_path(title, year, media_type, cover_dir)
|
||||
cover_path = media_folder / "cover.jpg"
|
||||
|
||||
if cover_path.exists():
|
||||
if await AsyncPath(cover_path).exists():
|
||||
return str(cover_path)
|
||||
|
||||
url = f"{TMDB_IMAGE_BASE}/{size}{poster_path}"
|
||||
@@ -85,7 +88,7 @@ async def download_backdrop_image(
|
||||
media_folder = get_media_folder_path(title, year, media_type, cover_dir)
|
||||
local_path = media_folder / "backdrop.jpg"
|
||||
|
||||
if local_path.exists():
|
||||
if await AsyncPath(local_path).exists():
|
||||
return str(local_path)
|
||||
|
||||
url = f"{TMDB_IMAGE_BASE}/{size}{backdrop_path}"
|
||||
@@ -104,9 +107,9 @@ async def download_season_poster(
|
||||
|
||||
output_path = media_folder / f"season{season_num:02d}.jpg"
|
||||
|
||||
if output_path.exists():
|
||||
if await AsyncPath(output_path).exists():
|
||||
return str(output_path)
|
||||
|
||||
media_folder.mkdir(parents=True, exist_ok=True)
|
||||
await AsyncPath(media_folder).mkdir(parents=True, exist_ok=True)
|
||||
url = f"{TMDB_IMAGE_BASE}/{DEFAULT_POSTER_SIZE}{poster_path}"
|
||||
return await _download_image(url, output_path, f"season {season_num} poster")
|
||||
|
||||
+36
-20
@@ -9,11 +9,13 @@ debounced background task.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
from aiopathlib import AsyncPath
|
||||
from fastapi import WebSocket
|
||||
|
||||
from hivescan.structs import (
|
||||
@@ -57,14 +59,15 @@ class IndexStore:
|
||||
# Persistence
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_snapshot(self) -> None:
|
||||
async def load_snapshot(self) -> None:
|
||||
"""Load index from disk snapshot (recovery on startup)."""
|
||||
if not self.snapshot_path.exists():
|
||||
ap = AsyncPath(self.snapshot_path)
|
||||
if not await ap.exists():
|
||||
logger.info("No snapshot found at %s, starting fresh", self.snapshot_path)
|
||||
return
|
||||
try:
|
||||
data = msgspec.json.decode(
|
||||
self.snapshot_path.read_bytes(), type=IndexSnapshot
|
||||
await ap.read_bytes(), type=IndexSnapshot
|
||||
)
|
||||
for m in data.movies:
|
||||
self.movies[m.id] = m
|
||||
@@ -78,8 +81,8 @@ class IndexStore:
|
||||
except Exception:
|
||||
logger.exception("Failed to load snapshot from %s", self.snapshot_path)
|
||||
|
||||
def _write_snapshot(self) -> None:
|
||||
"""Write current index to disk (synchronous, called from debounce task)."""
|
||||
async def _write_snapshot(self) -> None:
|
||||
"""Write current index to disk (called from debounce task)."""
|
||||
movies_list = sorted(
|
||||
self.movies.values(), key=lambda x: (x.title.lower(), x.year or 0)
|
||||
)
|
||||
@@ -103,25 +106,28 @@ class IndexStore:
|
||||
series=series_list,
|
||||
)
|
||||
|
||||
self.snapshot_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
await AsyncPath(self.snapshot_path.parent).mkdir(parents=True, exist_ok=True)
|
||||
tmp = self.snapshot_path.with_suffix(".tmp")
|
||||
tmp.write_bytes(msgspec.json.format(msgspec.json.encode(snapshot), indent=2))
|
||||
tmp.replace(self.snapshot_path)
|
||||
await AsyncPath(tmp).write_bytes(msgspec.json.format(msgspec.json.encode(snapshot), indent=2))
|
||||
# os.replace is atomic and overwrites on all platforms (unlike rename on Windows)
|
||||
await asyncio.to_thread(os.replace, tmp, self.snapshot_path)
|
||||
logger.debug("Snapshot written to %s", self.snapshot_path)
|
||||
|
||||
def _schedule_snapshot(self) -> None:
|
||||
"""Schedule a debounced snapshot write."""
|
||||
self._snapshot_dirty = True
|
||||
if self._snapshot_task is None or self._snapshot_task.done():
|
||||
self._snapshot_task = asyncio.create_task(self._debounced_snapshot())
|
||||
self._snapshot_task = asyncio.create_task(self._snapshot_writer())
|
||||
|
||||
async def _debounced_snapshot(self) -> None:
|
||||
"""Wait for debounce interval then write if still dirty."""
|
||||
while self._snapshot_dirty:
|
||||
self._snapshot_dirty = False
|
||||
async def _snapshot_writer(self) -> None:
|
||||
"""Flush to disk every SNAPSHOT_DEBOUNCE seconds while dirty."""
|
||||
while True:
|
||||
await asyncio.sleep(SNAPSHOT_DEBOUNCE)
|
||||
# After the sleep, if no new mutations happened, write
|
||||
self._write_snapshot()
|
||||
if self._snapshot_dirty:
|
||||
self._snapshot_dirty = False
|
||||
await self._write_snapshot()
|
||||
else:
|
||||
break # No pending mutations — stop the loop
|
||||
|
||||
async def flush_snapshot(self) -> None:
|
||||
"""Force-write a snapshot immediately (e.g. on shutdown)."""
|
||||
@@ -131,23 +137,33 @@ class IndexStore:
|
||||
await self._snapshot_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._write_snapshot()
|
||||
await self._write_snapshot()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Mutations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def upsert_movie(self, item: Movie) -> None:
|
||||
"""Insert or update a movie in the index and broadcast."""
|
||||
def upsert_movie(self, item: Movie) -> bool:
|
||||
"""Insert or update a movie. Returns True if it was a real change."""
|
||||
existing = self.movies.get(item.id)
|
||||
if existing is not None:
|
||||
if msgspec.json.encode(existing) == msgspec.json.encode(item):
|
||||
return False
|
||||
self.movies[item.id] = item
|
||||
self._schedule_snapshot()
|
||||
self._broadcast(WsUpsert(kind="movie", item=item))
|
||||
return True
|
||||
|
||||
def upsert_series(self, item: Series) -> None:
|
||||
"""Insert or update a series in the index and broadcast."""
|
||||
def upsert_series(self, item: Series) -> bool:
|
||||
"""Insert or update a series. Returns True if it was a real change."""
|
||||
existing = self.series.get(item.id)
|
||||
if existing is not None:
|
||||
if msgspec.json.encode(existing) == msgspec.json.encode(item):
|
||||
return False
|
||||
self.series[item.id] = item
|
||||
self._schedule_snapshot()
|
||||
self._broadcast(WsUpsert(kind="series", item=item))
|
||||
return True
|
||||
|
||||
def remove_movie(self, item_id: str) -> None:
|
||||
"""Remove a movie from the index and broadcast."""
|
||||
|
||||
+120
-85
@@ -1,5 +1,6 @@
|
||||
"""Media index generation — async generators for continuous scanning."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
@@ -35,24 +36,28 @@ from hivescan.images import (
|
||||
)
|
||||
from hivescan.utils import (
|
||||
get_added_timestamp,
|
||||
get_directory_size,
|
||||
get_media_folder_path,
|
||||
make_relative_path,
|
||||
sort_by_quality,
|
||||
RESOLUTION_PRIORITY,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("hivescan.indexer")
|
||||
|
||||
|
||||
def _build_version_info(
|
||||
async def _build_version_info(
|
||||
item: ParsedContent, media_root: Optional[str] = None
|
||||
) -> MovieVersion:
|
||||
"""Build version/release info for a single torrent."""
|
||||
playable_file = find_playable_file(item.path)
|
||||
playable_file = await find_playable_file(item.path)
|
||||
if item.content_hash and item.content_hash.size == 0:
|
||||
item.content_hash.size = await get_directory_size(item.content_hash.path)
|
||||
size = item.content_hash.size if item.content_hash else None
|
||||
newest = get_added_timestamp(item.path)
|
||||
newest = await get_added_timestamp(item.path)
|
||||
|
||||
return MovieVersion(
|
||||
path=make_relative_path(str(item.path), media_root),
|
||||
torrent_title=item.title,
|
||||
playable_file=make_relative_path(playable_file, media_root),
|
||||
resolution=item.resolution,
|
||||
quality=item.quality,
|
||||
@@ -64,7 +69,7 @@ def _build_version_info(
|
||||
)
|
||||
|
||||
|
||||
def _collect_episode_files(
|
||||
async def _collect_episode_files(
|
||||
items: List[ParsedContent],
|
||||
) -> Dict[Tuple[int, int], List[Dict]]:
|
||||
"""
|
||||
@@ -75,7 +80,7 @@ def _collect_episode_files(
|
||||
all_episode_files: Dict[Tuple[int, int], List[Dict]] = {}
|
||||
|
||||
for item in items:
|
||||
episode_files = find_episode_files(item.path)
|
||||
episode_files = await find_episode_files(item.path)
|
||||
|
||||
for (season_num, episode_num), files in episode_files.items():
|
||||
key = (season_num, episode_num)
|
||||
@@ -92,6 +97,7 @@ def _collect_episode_files(
|
||||
"audio": item.audio,
|
||||
"encoder": item.encoder,
|
||||
"torrent_path": str(item.path),
|
||||
"torrent_title": item.title,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -104,7 +110,7 @@ def _collect_episode_files(
|
||||
item.episode if isinstance(item.episode, list) else [item.episode]
|
||||
)
|
||||
|
||||
playable = find_playable_file(item.path)
|
||||
playable = await find_playable_file(item.path)
|
||||
if playable:
|
||||
for sn in season_nums:
|
||||
for ep in episode_nums:
|
||||
@@ -116,6 +122,8 @@ def _collect_episode_files(
|
||||
for f in all_episode_files.get(key, [])
|
||||
)
|
||||
if not already_added:
|
||||
if item.content_hash and item.content_hash.size == 0:
|
||||
item.content_hash.size = await get_directory_size(item.content_hash.path)
|
||||
size = item.content_hash.size if item.content_hash else 0
|
||||
all_episode_files[key].append(
|
||||
{
|
||||
@@ -127,6 +135,7 @@ def _collect_episode_files(
|
||||
"audio": item.audio,
|
||||
"encoder": item.encoder,
|
||||
"torrent_path": str(item.path),
|
||||
"torrent_title": item.title,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -166,19 +175,18 @@ def _build_episodes_data(
|
||||
(best_file, series_folder, season_num, episode_num, series_title)
|
||||
)
|
||||
|
||||
releases = []
|
||||
releases = {}
|
||||
for f in episode_files:
|
||||
releases.append(
|
||||
EpisodeRelease(
|
||||
path=make_relative_path(f["torrent_path"], media_root),
|
||||
playable_file=make_relative_path(f["path"], media_root),
|
||||
resolution=f.get("resolution"),
|
||||
quality=f.get("quality"),
|
||||
codec=f.get("codec"),
|
||||
audio=f.get("audio"),
|
||||
encoder=f.get("encoder"),
|
||||
size=f.get("size"),
|
||||
)
|
||||
relpath = make_relative_path(f["torrent_path"], media_root)
|
||||
releases[relpath] = EpisodeRelease(
|
||||
torrent_title=f["torrent_title"],
|
||||
playable_file=make_relative_path(f["path"], media_root),
|
||||
resolution=f.get("resolution"),
|
||||
quality=f.get("quality"),
|
||||
codec=f.get("codec"),
|
||||
audio=f.get("audio"),
|
||||
encoder=f.get("encoder"),
|
||||
size=f.get("size"),
|
||||
)
|
||||
|
||||
episode_data = Episode(
|
||||
@@ -228,7 +236,7 @@ async def _build_seasons_data(
|
||||
if tmdb_id:
|
||||
cache_key = (tmdb_id, season_num)
|
||||
if cache_key not in season_cache:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
" Fetching season %d details for %s", season_num, display_title
|
||||
)
|
||||
season_cache[cache_key] = await fetch_season_details(
|
||||
@@ -297,16 +305,17 @@ async def _process_movies(
|
||||
movie_tmdb_cache[cache_key] = tmdb_info
|
||||
return tmdb_info
|
||||
|
||||
def has_playable(item: ParsedContent) -> bool:
|
||||
return find_playable_file(item.path) is not None
|
||||
async def has_playable(item: ParsedContent) -> bool:
|
||||
return await find_playable_file(item.path) is not None
|
||||
|
||||
# Filter movies with playable files
|
||||
valid_movies = [
|
||||
item for item in categories[ContentType.MOVIE] if has_playable(item)
|
||||
]
|
||||
valid_movies = []
|
||||
for item in categories[ContentType.MOVIE]:
|
||||
if await has_playable(item):
|
||||
valid_movies.append(item)
|
||||
skipped = len(categories[ContentType.MOVIE]) - len(valid_movies)
|
||||
if skipped > 0:
|
||||
print(f" Skipped {skipped} movie torrents with no playable video files")
|
||||
logger.debug(" Skipped %d movie torrents with no playable video files", skipped)
|
||||
|
||||
# Group by title+year
|
||||
movie_groups: Dict[str, List[ParsedContent]] = {}
|
||||
@@ -320,15 +329,16 @@ async def _process_movies(
|
||||
tmdb_movie_groups: Dict[int, Dict] = {}
|
||||
no_tmdb_movie_groups: Dict[str, Dict] = {}
|
||||
|
||||
logger.info(
|
||||
" Processing %d unique movies (%d total versions)...",
|
||||
len(movie_groups),
|
||||
len(categories[ContentType.MOVIE]),
|
||||
)
|
||||
if movie_groups:
|
||||
logger.info(
|
||||
" Processing %d unique movies (%d total versions)...",
|
||||
len(movie_groups),
|
||||
len(categories[ContentType.MOVIE]),
|
||||
) if movie_groups else None
|
||||
|
||||
for idx, (movie_key, items) in enumerate(movie_groups.items(), 1):
|
||||
first_item = items[0]
|
||||
logger.info(
|
||||
logger.debug(
|
||||
" [%d/%d] %s (%s)",
|
||||
idx,
|
||||
len(movie_groups),
|
||||
@@ -337,6 +347,9 @@ async def _process_movies(
|
||||
)
|
||||
|
||||
tmdb_info = await get_movie_tmdb(first_item.title, first_item.year)
|
||||
# Yield to event loop so HTTP requests stay responsive
|
||||
if idx % 20 == 0:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
if tmdb_info and tmdb_info.tmdb_id:
|
||||
if tmdb_info.tmdb_id not in tmdb_movie_groups:
|
||||
@@ -371,10 +384,10 @@ async def _process_movies(
|
||||
# Find/download cover
|
||||
cover_path = None
|
||||
if fetch_covers:
|
||||
cover_path = find_cover_image(display_title, year, "movie", cover_dir)
|
||||
cover_path = await find_cover_image(display_title, year, "movie", cover_dir)
|
||||
if not cover_path:
|
||||
for tt in torrent_titles:
|
||||
cover_path = find_cover_image(tt, year, "movie", cover_dir)
|
||||
cover_path = await find_cover_image(tt, year, "movie", cover_dir)
|
||||
if cover_path:
|
||||
break
|
||||
if not cover_path and tmdb_info.poster_path:
|
||||
@@ -382,19 +395,30 @@ async def _process_movies(
|
||||
tmdb_info.poster_path, display_title, year, "movie", cover_dir
|
||||
)
|
||||
|
||||
versions = [_build_version_info(item, media_root) for item in items]
|
||||
sort_by_quality(versions)
|
||||
versions = {}
|
||||
for item in items:
|
||||
relpath = make_relative_path(str(item.path), media_root)
|
||||
version = await _build_version_info(item, media_root)
|
||||
versions[relpath] = version
|
||||
|
||||
sort_by_quality(list(versions.values()))
|
||||
|
||||
# Queue showreel generation
|
||||
showreel_paths = []
|
||||
showreel_task = None
|
||||
if generate_showreels and versions:
|
||||
best_playable = versions[0].playable_file
|
||||
if best_playable:
|
||||
# Find the best version for showreel (highest quality)
|
||||
best_relpath = max(versions.keys(), key=lambda k: (
|
||||
RESOLUTION_PRIORITY.get(versions[k].resolution or "", 0),
|
||||
versions[k].size or 0,
|
||||
k
|
||||
))
|
||||
best_version = versions[best_relpath]
|
||||
if best_version.playable_file:
|
||||
abs_playable = (
|
||||
str(Path(media_root) / best_playable)
|
||||
str(Path(media_root) / best_version.playable_file)
|
||||
if media_root
|
||||
else best_playable
|
||||
else best_version.playable_file
|
||||
)
|
||||
media_folder = get_media_folder_path(
|
||||
display_title, year, "movie", cover_dir
|
||||
@@ -411,10 +435,7 @@ async def _process_movies(
|
||||
tmdb_info.backdrop_path, display_title, year, "movie", cover_dir
|
||||
)
|
||||
|
||||
different_titles = [
|
||||
t for t in torrent_titles if t.lower() != display_title.lower()
|
||||
]
|
||||
version_timestamps = [v.newest for v in versions if v.newest]
|
||||
version_timestamps = [v.newest for v in versions.values() if v.newest]
|
||||
newest = max(version_timestamps) if version_timestamps else None
|
||||
|
||||
movie = Movie(
|
||||
@@ -422,7 +443,6 @@ async def _process_movies(
|
||||
title=display_title,
|
||||
original_title=tmdb_info.original_title,
|
||||
alternative_titles=tmdb_info.alternative_titles,
|
||||
torrent_titles=different_titles if different_titles else None,
|
||||
year=year,
|
||||
newest=newest,
|
||||
cover_path=make_relative_path(cover_path, media_root),
|
||||
@@ -455,33 +475,43 @@ async def _process_movies(
|
||||
item_id = hashlib.md5(f"movie:{title}:{year}".encode()).hexdigest()[:12]
|
||||
|
||||
cover_path = (
|
||||
find_cover_image(title, year, "movie", cover_dir) if fetch_covers else None
|
||||
await find_cover_image(title, year, "movie", cover_dir) if fetch_covers else None
|
||||
)
|
||||
|
||||
versions = [_build_version_info(item, media_root) for item in items]
|
||||
sort_by_quality(versions)
|
||||
versions = {}
|
||||
for item in items:
|
||||
relpath = make_relative_path(str(item.path), media_root)
|
||||
version = await _build_version_info(item, media_root)
|
||||
versions[relpath] = version
|
||||
|
||||
sort_by_quality(list(versions.values()))
|
||||
|
||||
showreel_paths = []
|
||||
showreel_task = None
|
||||
if generate_showreels and versions:
|
||||
best_playable = versions[0].playable_file
|
||||
if best_playable:
|
||||
# Find the best version for showreel (highest quality)
|
||||
best_relpath = max(versions.keys(), key=lambda k: (
|
||||
RESOLUTION_PRIORITY.get(versions[k].resolution or "", 0),
|
||||
versions[k].size or 0,
|
||||
k
|
||||
))
|
||||
best_version = versions[best_relpath]
|
||||
if best_version.playable_file and not best_version.playable_file.endswith(".bdmv"):
|
||||
abs_playable = (
|
||||
str(Path(media_root) / best_playable)
|
||||
str(Path(media_root) / best_version.playable_file)
|
||||
if media_root
|
||||
else best_playable
|
||||
else best_version.playable_file
|
||||
)
|
||||
if not abs_playable.endswith(".bdmv"):
|
||||
media_folder = get_media_folder_path(
|
||||
title, year, "movie", cover_dir
|
||||
)
|
||||
showreel_paths = get_expected_showreel_paths(
|
||||
media_folder,
|
||||
media_root=Path(media_root) if media_root else None,
|
||||
)
|
||||
showreel_task = (abs_playable, media_folder, title)
|
||||
media_folder = get_media_folder_path(
|
||||
title, year, "movie", cover_dir
|
||||
)
|
||||
showreel_paths = get_expected_showreel_paths(
|
||||
media_folder,
|
||||
media_root=Path(media_root) if media_root else None,
|
||||
)
|
||||
showreel_task = (abs_playable, media_folder, title)
|
||||
|
||||
version_timestamps = [v.newest for v in versions if v.newest]
|
||||
version_timestamps = [v.newest for v in versions.values() if v.newest]
|
||||
newest = max(version_timestamps) if version_timestamps else None
|
||||
|
||||
movie = Movie(
|
||||
@@ -520,15 +550,16 @@ async def _process_series(
|
||||
series_tmdb_cache[cache_key] = tmdb_info
|
||||
return tmdb_info
|
||||
|
||||
def has_video_content(item: ParsedContent) -> bool:
|
||||
if find_playable_file(item.path):
|
||||
async def has_video_content(item: ParsedContent) -> bool:
|
||||
if await find_playable_file(item.path):
|
||||
return True
|
||||
return len(find_episode_files(item.path)) > 0
|
||||
return len(await find_episode_files(item.path)) > 0
|
||||
|
||||
# Filter series with video content
|
||||
valid_series = [
|
||||
item for item in categories[ContentType.SERIES] if has_video_content(item)
|
||||
]
|
||||
valid_series = []
|
||||
for item in categories[ContentType.SERIES]:
|
||||
if await has_video_content(item):
|
||||
valid_series.append(item)
|
||||
skipped = len(categories[ContentType.SERIES]) - len(valid_series)
|
||||
if skipped > 0:
|
||||
logger.info(
|
||||
@@ -547,17 +578,21 @@ async def _process_series(
|
||||
tmdb_groups: Dict[int, Dict] = {}
|
||||
no_tmdb_groups: Dict[str, Dict] = {}
|
||||
|
||||
logger.info(
|
||||
" Processing %d unique series (%d total entries)...",
|
||||
len(series_groups),
|
||||
len(categories[ContentType.SERIES]),
|
||||
)
|
||||
if series_groups:
|
||||
logger.info(
|
||||
" Processing %d unique series (%d total entries)...",
|
||||
len(series_groups),
|
||||
len(categories[ContentType.SERIES]),
|
||||
)
|
||||
|
||||
for idx, (series_key, items) in enumerate(series_groups.items(), 1):
|
||||
first_item = items[0]
|
||||
logger.info(" [%d/%d] %s", idx, len(series_groups), first_item.title)
|
||||
logger.debug(" [%d/%d] %s", idx, len(series_groups), first_item.title)
|
||||
|
||||
tmdb_info = await get_series_tmdb(first_item.title)
|
||||
# Yield to event loop so HTTP requests stay responsive
|
||||
if idx % 20 == 0:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
if tmdb_info and tmdb_info.tmdb_id:
|
||||
if tmdb_info.tmdb_id not in tmdb_groups:
|
||||
@@ -583,17 +618,17 @@ async def _process_series(
|
||||
display_title = tmdb_info.title
|
||||
series_id = hashlib.md5(f"series:{tmdb_id}".encode()).hexdigest()[:12]
|
||||
|
||||
logger.info(" [%d/%d] %s", series_idx, len(tmdb_groups), display_title)
|
||||
logger.debug(" [%d/%d] %s", series_idx, len(tmdb_groups), display_title)
|
||||
|
||||
series_folder = get_media_folder_path(display_title, None, "series", cover_dir)
|
||||
|
||||
# Find/download cover
|
||||
cover_path = None
|
||||
if fetch_covers:
|
||||
cover_path = find_cover_image(display_title, None, "series", cover_dir)
|
||||
cover_path = await find_cover_image(display_title, None, "series", cover_dir)
|
||||
if not cover_path:
|
||||
for tt in torrent_titles:
|
||||
cover_path = find_cover_image(tt, None, "series", cover_dir)
|
||||
cover_path = await find_cover_image(tt, None, "series", cover_dir)
|
||||
if cover_path:
|
||||
break
|
||||
if not cover_path and tmdb_info.poster_path:
|
||||
@@ -609,7 +644,7 @@ async def _process_series(
|
||||
)
|
||||
|
||||
# Collect and build episode data
|
||||
all_episode_files = _collect_episode_files(items)
|
||||
all_episode_files = await _collect_episode_files(items)
|
||||
ep_reel_tasks: List[Tuple[str, Path, int, int, str]] = []
|
||||
seasons_data = await _build_seasons_data(
|
||||
all_episode_files,
|
||||
@@ -627,10 +662,10 @@ async def _process_series(
|
||||
logger.info(" Skipping %s - no episodes found", display_title)
|
||||
continue
|
||||
|
||||
different_titles = [
|
||||
different_titles = sorted(
|
||||
t for t in torrent_titles if t.lower() != display_title.lower()
|
||||
]
|
||||
item_timestamps = [get_added_timestamp(item.path) for item in items]
|
||||
)
|
||||
item_timestamps = [await get_added_timestamp(item.path) for item in items]
|
||||
item_timestamps = [t for t in item_timestamps if t is not None]
|
||||
newest = max(item_timestamps) if item_timestamps else None
|
||||
|
||||
@@ -638,7 +673,7 @@ async def _process_series(
|
||||
id=series_id,
|
||||
title=display_title,
|
||||
original_title=tmdb_info.original_title,
|
||||
torrent_titles=different_titles if different_titles else None,
|
||||
alternative_titles=different_titles if different_titles else None,
|
||||
newest=newest,
|
||||
cover_path=make_relative_path(cover_path, media_root),
|
||||
backdrop_path=make_relative_path(backdrop_path, media_root),
|
||||
@@ -670,11 +705,11 @@ async def _process_series(
|
||||
series_id = hashlib.md5(f"series:{title}".encode()).hexdigest()[:12]
|
||||
|
||||
cover_path = (
|
||||
find_cover_image(title, None, "series", cover_dir) if fetch_covers else None
|
||||
await find_cover_image(title, None, "series", cover_dir) if fetch_covers else None
|
||||
)
|
||||
series_folder = get_media_folder_path(title, None, "series", cover_dir)
|
||||
|
||||
all_episode_files = _collect_episode_files(items)
|
||||
all_episode_files = await _collect_episode_files(items)
|
||||
ep_reel_tasks: List[Tuple[str, Path, int, int, str]] = []
|
||||
seasons_data = await _build_seasons_data(
|
||||
all_episode_files,
|
||||
@@ -692,7 +727,7 @@ async def _process_series(
|
||||
logger.info(" Skipping %s - no episodes found", title)
|
||||
continue
|
||||
|
||||
item_timestamps = [get_added_timestamp(item.path) for item in items]
|
||||
item_timestamps = [await get_added_timestamp(item.path) for item in items]
|
||||
item_timestamps = [t for t in item_timestamps if t is not None]
|
||||
newest = max(item_timestamps) if item_timestamps else None
|
||||
|
||||
|
||||
+1
-14
@@ -21,20 +21,7 @@ class ContentHash:
|
||||
|
||||
path: Path
|
||||
hash: str
|
||||
_size: Optional[int] = None
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
"""Get the size, computing it lazily if needed."""
|
||||
if self._size is None:
|
||||
from hivescan.utils import get_directory_size
|
||||
|
||||
self._size = get_directory_size(self.path)
|
||||
return self._size
|
||||
|
||||
@size.setter
|
||||
def size(self, value: int) -> None:
|
||||
self._size = value
|
||||
size: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_path(cls, path: Path) -> "ContentHash":
|
||||
|
||||
+3
-2
@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import PTN
|
||||
from aiopathlib import AsyncPath
|
||||
|
||||
from hivescan.models import ContentHash, ContentType, ParsedContent
|
||||
|
||||
@@ -22,7 +23,7 @@ def determine_content_type(parsed: dict) -> ContentType:
|
||||
return ContentType.OTHER
|
||||
|
||||
|
||||
def parse_download(path: Path) -> ParsedContent:
|
||||
async def parse_download(path: Path) -> ParsedContent:
|
||||
"""Parse a downloaded torrent directory/file name."""
|
||||
name = path.name
|
||||
parsed = PTN.parse(name)
|
||||
@@ -44,7 +45,7 @@ def parse_download(path: Path) -> ParsedContent:
|
||||
episode_name=parsed.get("episodeName"),
|
||||
encoder=parsed.get("encoder"),
|
||||
language=parsed.get("language"),
|
||||
is_directory=path.is_dir(),
|
||||
is_directory=await AsyncPath(path).is_dir(),
|
||||
raw_parsed=parsed,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
|
||||
+47
-34
@@ -1,8 +1,11 @@
|
||||
"""File system scanning functions."""
|
||||
|
||||
import asyncio
|
||||
import glob
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterator, List, Optional, Tuple
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from aiopathlib import AsyncPath
|
||||
|
||||
from hivescan.models import ContentType, ParsedContent
|
||||
from hivescan.parsing import parse_download, parse_episode_from_filename
|
||||
@@ -28,20 +31,23 @@ _episode_files_cache: Dict[str, Dict[Tuple[int, int], List[Tuple[str, int]]]] =
|
||||
_playable_file_cache: Dict[str, Optional[str]] = {}
|
||||
|
||||
|
||||
def scan_downloads(base_pattern: str) -> Iterator[ParsedContent]:
|
||||
async def scan_downloads(base_pattern: str) -> List[ParsedContent]:
|
||||
"""
|
||||
Scan download directories matching the pattern.
|
||||
|
||||
Args:
|
||||
base_pattern: Glob pattern for finding download directories
|
||||
|
||||
Yields:
|
||||
ParsedContent objects for each found download
|
||||
Returns:
|
||||
List of ParsedContent objects for each found download
|
||||
"""
|
||||
exclude_patterns = [".torrents", "incomplete", ".incomplete"]
|
||||
results: List[ParsedContent] = []
|
||||
|
||||
for path_str in glob.glob(base_pattern):
|
||||
paths = await asyncio.to_thread(glob.glob, base_pattern)
|
||||
for path_str in paths:
|
||||
path = Path(path_str)
|
||||
ap = AsyncPath(path)
|
||||
|
||||
if path.name.startswith("."):
|
||||
continue
|
||||
@@ -49,10 +55,12 @@ def scan_downloads(base_pattern: str) -> Iterator[ParsedContent]:
|
||||
if any(excl.lower() in path.name.lower() for excl in exclude_patterns):
|
||||
continue
|
||||
|
||||
if not path.exists():
|
||||
if not await ap.exists():
|
||||
continue
|
||||
|
||||
yield parse_download(path)
|
||||
results.append(await parse_download(path))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def categorize_downloads(
|
||||
@@ -71,7 +79,7 @@ def categorize_downloads(
|
||||
return categories
|
||||
|
||||
|
||||
def find_episode_files(path: Path) -> Dict[Tuple[int, int], List[Tuple[str, int]]]:
|
||||
async def find_episode_files(path: Path) -> Dict[Tuple[int, int], List[Tuple[str, int]]]:
|
||||
"""
|
||||
Find all episode video files in a directory.
|
||||
|
||||
@@ -86,33 +94,35 @@ def find_episode_files(path: Path) -> Dict[Tuple[int, int], List[Tuple[str, int]
|
||||
return _episode_files_cache[cache_key]
|
||||
|
||||
episodes: Dict[Tuple[int, int], List[Tuple[str, int]]] = {}
|
||||
ap = AsyncPath(path)
|
||||
|
||||
if path.is_file():
|
||||
if await ap.is_file():
|
||||
if path.suffix.lower() in VIDEO_EXTENSIONS:
|
||||
ep_info = parse_episode_from_filename(path.name)
|
||||
if ep_info:
|
||||
episodes[ep_info] = [(str(path), path.stat().st_size)]
|
||||
episodes[ep_info] = [(str(path), (await ap.stat()).st_size)]
|
||||
_episode_files_cache[cache_key] = episodes
|
||||
return episodes
|
||||
|
||||
try:
|
||||
for f in path.rglob("*"):
|
||||
if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS:
|
||||
if "sample" in f.name.lower():
|
||||
for f in ap.rglob("*"):
|
||||
af = AsyncPath(f)
|
||||
if await af.is_file() and Path(f).suffix.lower() in VIDEO_EXTENSIONS:
|
||||
if "sample" in Path(f).name.lower():
|
||||
continue
|
||||
ep_info = parse_episode_from_filename(f.name)
|
||||
ep_info = parse_episode_from_filename(Path(f).name)
|
||||
if ep_info:
|
||||
if ep_info not in episodes:
|
||||
episodes[ep_info] = []
|
||||
episodes[ep_info].append((str(f), f.stat().st_size))
|
||||
except OSError, PermissionError:
|
||||
episodes[ep_info].append((str(f), (await af.stat()).st_size))
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
_episode_files_cache[cache_key] = episodes
|
||||
return episodes
|
||||
|
||||
|
||||
def find_playable_file(path: Path) -> Optional[str]:
|
||||
async def find_playable_file(path: Path) -> Optional[str]:
|
||||
"""
|
||||
Find the main playable media file in a directory.
|
||||
|
||||
@@ -123,7 +133,9 @@ def find_playable_file(path: Path) -> Optional[str]:
|
||||
if cache_key in _playable_file_cache:
|
||||
return _playable_file_cache[cache_key]
|
||||
|
||||
if path.is_file():
|
||||
ap = AsyncPath(path)
|
||||
|
||||
if await ap.is_file():
|
||||
if path.suffix.lower() in VIDEO_EXTENSIONS:
|
||||
result = str(path)
|
||||
_playable_file_cache[cache_key] = result
|
||||
@@ -133,32 +145,33 @@ def find_playable_file(path: Path) -> Optional[str]:
|
||||
|
||||
# Check for Blu-ray disc structure
|
||||
bdmv_index = path / "BDMV" / "index.bdmv"
|
||||
if bdmv_index.exists():
|
||||
if await AsyncPath(bdmv_index).exists():
|
||||
result = str(bdmv_index)
|
||||
_playable_file_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
# Check nested Blu-ray structure (e.g., MovieName/DISC1/BDMV/)
|
||||
try:
|
||||
for subdir in path.iterdir():
|
||||
if subdir.is_dir():
|
||||
nested_bdmv = subdir / "BDMV" / "index.bdmv"
|
||||
if nested_bdmv.exists():
|
||||
for subdir in ap.iterdir():
|
||||
if await AsyncPath(subdir).is_dir():
|
||||
nested_bdmv = Path(subdir) / "BDMV" / "index.bdmv"
|
||||
if await AsyncPath(nested_bdmv).exists():
|
||||
result = str(nested_bdmv)
|
||||
_playable_file_cache[cache_key] = result
|
||||
return result
|
||||
except OSError, PermissionError:
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
# Find largest video file
|
||||
video_files = []
|
||||
try:
|
||||
for f in path.rglob("*"):
|
||||
if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS:
|
||||
if "sample" in f.name.lower():
|
||||
for f in ap.rglob("*"):
|
||||
af = AsyncPath(f)
|
||||
if await af.is_file() and Path(f).suffix.lower() in VIDEO_EXTENSIONS:
|
||||
if "sample" in Path(f).name.lower():
|
||||
continue
|
||||
video_files.append((f, f.stat().st_size))
|
||||
except OSError, PermissionError:
|
||||
video_files.append((str(f), (await af.stat()).st_size))
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
if not video_files:
|
||||
@@ -166,29 +179,29 @@ def find_playable_file(path: Path) -> Optional[str]:
|
||||
return None
|
||||
|
||||
video_files.sort(key=lambda x: x[1], reverse=True)
|
||||
result = str(video_files[0][0])
|
||||
result = video_files[0][0]
|
||||
_playable_file_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def find_cover_image(
|
||||
async def find_cover_image(
|
||||
title: str, year: Optional[int], media_type: str, cover_dir: Path
|
||||
) -> Optional[str]:
|
||||
"""Find a cover image for the given media item."""
|
||||
media_folder = get_media_folder_path(title, year, media_type, cover_dir)
|
||||
cover_path = media_folder / "cover.jpg"
|
||||
if cover_path.exists():
|
||||
if await AsyncPath(cover_path).exists():
|
||||
return str(cover_path)
|
||||
|
||||
# Legacy structure fallback
|
||||
subdir = "movies" if media_type == "movie" else "series"
|
||||
if media_type == "movie" and year:
|
||||
legacy_path = cover_dir / subdir / f"{sanitize_filename(title)} ({year}).jpg"
|
||||
if legacy_path.exists():
|
||||
if await AsyncPath(legacy_path).exists():
|
||||
return str(legacy_path)
|
||||
|
||||
legacy_path = cover_dir / subdir / f"{sanitize_filename(title)}.jpg"
|
||||
if legacy_path.exists():
|
||||
if await AsyncPath(legacy_path).exists():
|
||||
return str(legacy_path)
|
||||
|
||||
return None
|
||||
|
||||
+82
-47
@@ -23,6 +23,7 @@ from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
import msgspec
|
||||
from aiopathlib import AsyncPath
|
||||
from hivescan.index_store import IndexStore
|
||||
from hivescan.indexer import _process_movies, _process_series
|
||||
from hivescan.structs import MsgspecResponse, ScanRequest, StatusResponse, TaskInfo
|
||||
@@ -54,6 +55,7 @@ _scan_task: Optional[asyncio.Task] = None
|
||||
_showreel_queue: asyncio.Queue = asyncio.Queue()
|
||||
_showreel_worker_task: Optional[asyncio.Task] = None
|
||||
_rescan_worker_task: Optional[asyncio.Task] = None
|
||||
_seen_mtimes: dict[str, int] = {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -77,7 +79,7 @@ async def lifespan(app: FastAPI):
|
||||
pattern = pattern.strip()
|
||||
if not pattern:
|
||||
continue
|
||||
expanded = glob.glob(pattern)
|
||||
expanded = await asyncio.to_thread(glob.glob, pattern)
|
||||
if expanded:
|
||||
all_paths.extend(Path(p) for p in expanded)
|
||||
else:
|
||||
@@ -89,18 +91,18 @@ async def lifespan(app: FastAPI):
|
||||
OUTPUT_DIR = Path(os.environ["HIVESCAN_OUTPUT"])
|
||||
MEDIA_ROOT = OUTPUT_DIR.parent
|
||||
else:
|
||||
MEDIA_ROOT = find_common_root(all_paths)
|
||||
MEDIA_ROOT = await find_common_root(all_paths)
|
||||
if MEDIA_ROOT is None:
|
||||
logger.error("Cannot determine common root; set HIVESCAN_OUTPUT")
|
||||
sys.exit(1)
|
||||
OUTPUT_DIR = MEDIA_ROOT / DEFAULT_OUTPUT_FOLDER
|
||||
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
await AsyncPath(OUTPUT_DIR).mkdir(parents=True, exist_ok=True)
|
||||
set_cache_dir(OUTPUT_DIR / ".tmdb-cache")
|
||||
|
||||
# Initialise index store and load snapshot
|
||||
store = IndexStore(OUTPUT_DIR / "index.json", media_root=str(MEDIA_ROOT))
|
||||
store.load_snapshot()
|
||||
await store.load_snapshot()
|
||||
logger.info(
|
||||
"Index store ready: %d movies, %d series (from snapshot)",
|
||||
len(store.movies),
|
||||
@@ -226,42 +228,65 @@ async def _rescan_loop():
|
||||
logger.exception("Rescan loop error")
|
||||
|
||||
|
||||
async def _discover_downloads(paths_to_scan: List[str]) -> List[ParsedContent]:
|
||||
"""Walk the filesystem and parse all downloads, skipping unchanged torrents."""
|
||||
downloads: List[ParsedContent] = []
|
||||
media_root_str = str(MEDIA_ROOT) if MEDIA_ROOT else None
|
||||
for pattern in paths_to_scan:
|
||||
p = Path(pattern)
|
||||
ap = AsyncPath(p)
|
||||
if await ap.is_dir():
|
||||
for item in ap.iterdir():
|
||||
if not Path(item).name.startswith("."):
|
||||
relpath = make_relative_path(str(item), media_root_str)
|
||||
stat_info = await AsyncPath(item).stat()
|
||||
mtime = int(stat_info.st_mtime)
|
||||
if relpath in _seen_mtimes and _seen_mtimes[relpath] == mtime:
|
||||
continue # skip unchanged
|
||||
_seen_mtimes[relpath] = mtime
|
||||
downloads.append(await parse_download(Path(item)))
|
||||
elif await ap.exists():
|
||||
relpath = make_relative_path(str(p), media_root_str)
|
||||
stat_info = await ap.stat()
|
||||
mtime = int(stat_info.st_mtime)
|
||||
if relpath in _seen_mtimes and _seen_mtimes[relpath] == mtime:
|
||||
continue
|
||||
_seen_mtimes[relpath] = mtime
|
||||
downloads.append(await parse_download(p))
|
||||
return downloads
|
||||
|
||||
|
||||
async def _run_scan(override_paths: Optional[List[str]] = None):
|
||||
"""
|
||||
Full scan pipeline:
|
||||
1. Walk filesystem, parse torrents
|
||||
1. Walk filesystem, parse torrents (in thread)
|
||||
2. Categorise → movies / series
|
||||
3. Iterate async generators, upsert each item into IndexStore
|
||||
4. Queue showreel tasks
|
||||
"""
|
||||
task_id = f"scan-{uuid.uuid4().hex[:8]}"
|
||||
logger.info("Scan started (%s)", task_id)
|
||||
|
||||
store.broadcast_task(
|
||||
TaskInfo(
|
||||
id=task_id, status="running", progress=0, detail="Scanning filesystem..."
|
||||
)
|
||||
)
|
||||
|
||||
paths_to_scan = override_paths or SCAN_PATHS
|
||||
media_root_str = str(MEDIA_ROOT) if MEDIA_ROOT else None
|
||||
|
||||
try:
|
||||
# 1. Discover downloads (sync filesystem walk — fast enough)
|
||||
downloads: List[ParsedContent] = []
|
||||
for pattern in paths_to_scan:
|
||||
p = Path(pattern)
|
||||
if p.is_dir():
|
||||
for item in p.iterdir():
|
||||
if not item.name.startswith("."):
|
||||
downloads.append(parse_download(item))
|
||||
elif p.exists():
|
||||
downloads.append(parse_download(p))
|
||||
# 1. Discover downloads (now async)
|
||||
downloads = await _discover_downloads(paths_to_scan)
|
||||
|
||||
logger.info("Found %d items to process", len(downloads))
|
||||
if downloads:
|
||||
logger.info("Scan started (%s)", task_id)
|
||||
|
||||
store.broadcast_task(
|
||||
TaskInfo(
|
||||
id=task_id, status="running", progress=0, detail="Scanning filesystem..."
|
||||
)
|
||||
)
|
||||
|
||||
logger.info("Found %d items to process", len(downloads))
|
||||
categories = categorize_downloads(downloads)
|
||||
total = len(categories[ContentType.MOVIE]) + len(categories[ContentType.SERIES])
|
||||
processed = 0
|
||||
changed = 0
|
||||
|
||||
# 2. Process movies
|
||||
store.broadcast_task(
|
||||
@@ -276,19 +301,23 @@ async def _run_scan(override_paths: Optional[List[str]] = None):
|
||||
generate_showreels=True,
|
||||
media_root=media_root_str,
|
||||
):
|
||||
store.upsert_movie(movie)
|
||||
was_changed = store.upsert_movie(movie)
|
||||
if was_changed:
|
||||
changed += 1
|
||||
logger.info(" Updated movie: %s", movie.title)
|
||||
if showreel_task:
|
||||
await _showreel_queue.put(("movie", showreel_task, movie.id))
|
||||
processed += 1
|
||||
progress = processed / total if total else 1
|
||||
store.broadcast_task(
|
||||
TaskInfo(
|
||||
id=task_id,
|
||||
status="running",
|
||||
progress=round(progress, 3),
|
||||
detail=movie.title,
|
||||
if was_changed:
|
||||
store.broadcast_task(
|
||||
TaskInfo(
|
||||
id=task_id,
|
||||
status="running",
|
||||
progress=round(progress, 3),
|
||||
detail=movie.title,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# 3. Process series
|
||||
store.broadcast_task(
|
||||
@@ -306,29 +335,35 @@ async def _run_scan(override_paths: Optional[List[str]] = None):
|
||||
generate_showreels=True,
|
||||
media_root=media_root_str,
|
||||
):
|
||||
store.upsert_series(series)
|
||||
was_changed = store.upsert_series(series)
|
||||
if was_changed:
|
||||
changed += 1
|
||||
logger.info(" Updated series: %s", series.title)
|
||||
for task in ep_reel_tasks:
|
||||
await _showreel_queue.put(("episode", task, series.id))
|
||||
processed += 1
|
||||
progress = processed / total if total else 1
|
||||
store.broadcast_task(
|
||||
TaskInfo(
|
||||
id=task_id,
|
||||
status="running",
|
||||
progress=round(progress, 3),
|
||||
detail=series.title,
|
||||
if was_changed:
|
||||
store.broadcast_task(
|
||||
TaskInfo(
|
||||
id=task_id,
|
||||
status="running",
|
||||
progress=round(progress, 3),
|
||||
detail=series.title,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
store.broadcast_task(
|
||||
TaskInfo(id=task_id, status="completed", progress=1, detail="Scan complete")
|
||||
)
|
||||
logger.info(
|
||||
"Scan complete (%s): %d movies, %d series",
|
||||
task_id,
|
||||
len(store.movies),
|
||||
len(store.series),
|
||||
)
|
||||
if downloads:
|
||||
logger.info(
|
||||
"Scan complete (%s): %d movies, %d series (%d changed)",
|
||||
task_id,
|
||||
len(store.movies),
|
||||
len(store.series),
|
||||
changed,
|
||||
)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
store.broadcast_task(
|
||||
@@ -359,7 +394,7 @@ async def _showreel_worker():
|
||||
|
||||
if kind == "movie":
|
||||
video_path, media_folder, title = task_data
|
||||
if movie_showreels_exist(media_folder):
|
||||
if await movie_showreels_exist(media_folder):
|
||||
_showreel_queue.task_done()
|
||||
continue
|
||||
store.broadcast_task(
|
||||
@@ -397,7 +432,7 @@ async def _showreel_worker():
|
||||
video_path, media_folder, season_num, episode_num, series_title = (
|
||||
task_data
|
||||
)
|
||||
if episode_reel_exists(media_folder, season_num, episode_num):
|
||||
if await episode_reel_exists(media_folder, season_num, episode_num):
|
||||
_showreel_queue.task_done()
|
||||
continue
|
||||
ep_code = f"S{season_num:02d}E{episode_num:02d}"
|
||||
|
||||
+19
-17
@@ -15,6 +15,8 @@ from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from aiopathlib import AsyncPath
|
||||
|
||||
logger = logging.getLogger("hivescan.showreel")
|
||||
|
||||
|
||||
@@ -78,19 +80,19 @@ def get_expected_episode_reel_path(
|
||||
return str(output_path)
|
||||
|
||||
|
||||
def movie_showreels_exist(
|
||||
async def movie_showreels_exist(
|
||||
media_folder: Path, timestamps: list[int] = SHOWREEL_TIMESTAMPS
|
||||
) -> bool:
|
||||
"""Check if all showreel files for a movie already exist."""
|
||||
for reel_num in range(1, len(timestamps) + 1):
|
||||
if not (media_folder / f"reel{reel_num}.webm").exists():
|
||||
if not await AsyncPath(media_folder / f"reel{reel_num}.webm").exists():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def episode_reel_exists(media_folder: Path, season_num: int, episode_num: int) -> bool:
|
||||
async def episode_reel_exists(media_folder: Path, season_num: int, episode_num: int) -> bool:
|
||||
"""Check if an episode reel file already exists."""
|
||||
return (media_folder / f"S{season_num:02d}E{episode_num:02d}.webm").exists()
|
||||
return await AsyncPath(media_folder / f"S{season_num:02d}E{episode_num:02d}.webm").exists()
|
||||
|
||||
|
||||
def get_bluray_uri(video_path: str) -> Optional[str]:
|
||||
@@ -542,7 +544,7 @@ async def generate_showreel_images(
|
||||
if bluray_uri:
|
||||
ffmpeg_input = bluray_uri
|
||||
else:
|
||||
if not Path(video_path).exists():
|
||||
if not await AsyncPath(video_path).exists():
|
||||
return []
|
||||
ffmpeg_input = video_path
|
||||
|
||||
@@ -552,7 +554,7 @@ async def generate_showreel_images(
|
||||
for reel_num in range(1, len(timestamps) + 1):
|
||||
output_filename = f"reel{reel_num}.webm"
|
||||
output_path = media_folder / output_filename
|
||||
if output_path.exists():
|
||||
if await AsyncPath(output_path).exists():
|
||||
existing_paths.append(str(output_path))
|
||||
else:
|
||||
all_exist = False
|
||||
@@ -561,7 +563,7 @@ async def generate_showreel_images(
|
||||
if all_exist and existing_paths:
|
||||
return existing_paths
|
||||
|
||||
media_folder.mkdir(parents=True, exist_ok=True)
|
||||
await AsyncPath(media_folder).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Check video duration to avoid seeking past the end
|
||||
duration = await get_video_duration(ffmpeg_input)
|
||||
@@ -598,7 +600,7 @@ async def generate_showreel_images(
|
||||
output_path = media_folder / output_filename
|
||||
|
||||
# Skip if already exists
|
||||
if output_path.exists():
|
||||
if await AsyncPath(output_path).exists():
|
||||
generated_paths.append(str(output_path))
|
||||
if on_progress:
|
||||
on_progress(reel_num)
|
||||
@@ -655,16 +657,16 @@ async def generate_showreel_images(
|
||||
)
|
||||
await asyncio.wait_for(proc.communicate(), timeout=120)
|
||||
|
||||
if proc.returncode == 0 and output_path.exists():
|
||||
if proc.returncode == 0 and await AsyncPath(output_path).exists():
|
||||
generated_paths.append(str(output_path))
|
||||
if on_progress:
|
||||
on_progress(reel_num)
|
||||
else:
|
||||
output_path.unlink(missing_ok=True)
|
||||
await AsyncPath(output_path).unlink(missing_ok=True)
|
||||
# Abort remaining reels - if first one fails, others likely will too
|
||||
break
|
||||
except BaseException as e:
|
||||
output_path.unlink(missing_ok=True)
|
||||
await AsyncPath(output_path).unlink(missing_ok=True)
|
||||
if isinstance(e, (KeyboardInterrupt, SystemExit, asyncio.CancelledError)):
|
||||
raise
|
||||
logger.error(
|
||||
@@ -709,18 +711,18 @@ async def generate_episode_reel(
|
||||
if bluray_uri:
|
||||
ffmpeg_input = bluray_uri
|
||||
else:
|
||||
if not Path(video_path).exists():
|
||||
if not await AsyncPath(video_path).exists():
|
||||
return None
|
||||
ffmpeg_input = video_path
|
||||
|
||||
media_folder.mkdir(parents=True, exist_ok=True)
|
||||
await AsyncPath(media_folder).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Normalize episode code to SxxExx format
|
||||
output_filename = f"S{season_num:02d}E{episode_num:02d}.webm"
|
||||
output_path = media_folder / output_filename
|
||||
|
||||
# Skip if already exists
|
||||
if output_path.exists():
|
||||
if await AsyncPath(output_path).exists():
|
||||
return str(output_path)
|
||||
|
||||
# Check video duration
|
||||
@@ -797,13 +799,13 @@ async def generate_episode_reel(
|
||||
)
|
||||
await asyncio.wait_for(proc.communicate(), timeout=120)
|
||||
|
||||
if proc.returncode == 0 and output_path.exists():
|
||||
if proc.returncode == 0 and await AsyncPath(output_path).exists():
|
||||
return str(output_path)
|
||||
else:
|
||||
output_path.unlink(missing_ok=True)
|
||||
await AsyncPath(output_path).unlink(missing_ok=True)
|
||||
return None
|
||||
except BaseException as e:
|
||||
output_path.unlink(missing_ok=True)
|
||||
await AsyncPath(output_path).unlink(missing_ok=True)
|
||||
if isinstance(e, (KeyboardInterrupt, SystemExit, asyncio.CancelledError)):
|
||||
raise
|
||||
episode_code = f"S{season_num:02d}E{episode_num:02d}"
|
||||
|
||||
+7
-9
@@ -97,9 +97,9 @@ class TMDbInfo(msgspec.Struct):
|
||||
|
||||
|
||||
class MovieVersion(msgspec.Struct):
|
||||
"""One release/torrent of a movie."""
|
||||
"""One release/torrent of a movie, keyed by relative torrent path."""
|
||||
|
||||
path: str | None = None
|
||||
torrent_title: str | None = None
|
||||
playable_file: str | None = None
|
||||
resolution: str | None = None
|
||||
quality: str | None = None
|
||||
@@ -111,9 +111,9 @@ class MovieVersion(msgspec.Struct):
|
||||
|
||||
|
||||
class EpisodeRelease(msgspec.Struct):
|
||||
"""One release/torrent file of an episode."""
|
||||
"""One release/torrent file of an episode, keyed by relative torrent path."""
|
||||
|
||||
path: str | None = None
|
||||
torrent_title: str | None = None
|
||||
playable_file: str | None = None
|
||||
resolution: str | None = None
|
||||
quality: str | None = None
|
||||
@@ -135,7 +135,7 @@ class Episode(msgspec.Struct):
|
||||
rating: float | None = None
|
||||
director: str | None = None
|
||||
reel_image: str | None = None
|
||||
releases: list[EpisodeRelease] = []
|
||||
releases: dict[str, EpisodeRelease] = {}
|
||||
|
||||
|
||||
class Season(msgspec.Struct):
|
||||
@@ -157,13 +157,12 @@ class Movie(msgspec.Struct):
|
||||
title: str
|
||||
original_title: str | None = None
|
||||
alternative_titles: list[str] | None = None
|
||||
torrent_titles: list[str] | None = None
|
||||
year: int | None = None
|
||||
newest: int | None = None
|
||||
cover_path: str | None = None
|
||||
backdrop_path: str | None = None
|
||||
showreel_images: list[str] | None = None
|
||||
versions: list[MovieVersion] = []
|
||||
versions: dict[str, MovieVersion] = {}
|
||||
tmdb_id: int | None = None
|
||||
tmdb_title: str | None = None
|
||||
rating: float | None = None
|
||||
@@ -188,7 +187,6 @@ class Series(msgspec.Struct):
|
||||
title: str
|
||||
original_title: str | None = None
|
||||
alternative_titles: list[str] | None = None
|
||||
torrent_titles: list[str] | None = None
|
||||
newest: int | None = None
|
||||
cover_path: str | None = None
|
||||
backdrop_path: str | None = None
|
||||
@@ -229,7 +227,7 @@ class MediaStats(msgspec.Struct):
|
||||
class IndexSnapshot(msgspec.Struct):
|
||||
"""On-disk recovery snapshot of the full index."""
|
||||
|
||||
version: int = 5
|
||||
version: int = 6
|
||||
generated_at: str = ""
|
||||
media_root: str | None = None
|
||||
stats: MediaStats = msgspec.UNSET # type: ignore[assignment]
|
||||
|
||||
+19
-18
@@ -13,6 +13,7 @@ from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import httpx
|
||||
from aiopathlib import AsyncPath
|
||||
|
||||
from hivescan.structs import (
|
||||
CastMember,
|
||||
@@ -72,30 +73,30 @@ def _get_cache_path(endpoint: str, params: Dict[str, str]) -> Path:
|
||||
return _get_cache_dir() / f"{cache_hash}.json"
|
||||
|
||||
|
||||
def _load_from_cache(cache_path: Path):
|
||||
async def _load_from_cache(cache_path: Path):
|
||||
"""Load cached response. Returns _NOT_FOUND if not cached."""
|
||||
if not cache_path.exists():
|
||||
if not await AsyncPath(cache_path).exists():
|
||||
return _NOT_FOUND
|
||||
try:
|
||||
with open(cache_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
# Handle cached "no results" / errors
|
||||
if data.get("_cached_none"):
|
||||
return None
|
||||
return data
|
||||
text = await AsyncPath(cache_path).read_text(encoding="utf-8")
|
||||
data = json.loads(text)
|
||||
# Handle cached "no results" / errors
|
||||
if data.get("_cached_none"):
|
||||
return None
|
||||
return data
|
||||
except Exception:
|
||||
return _NOT_FOUND
|
||||
|
||||
|
||||
def _save_to_cache(cache_path: Path, data: Optional[Dict]):
|
||||
async def _save_to_cache(cache_path: Path, data: Optional[Dict]):
|
||||
"""Save response to cache."""
|
||||
try:
|
||||
_get_cache_dir().mkdir(parents=True, exist_ok=True)
|
||||
with open(cache_path, "w", encoding="utf-8") as f:
|
||||
if data is None:
|
||||
json.dump({"_cached_none": True}, f)
|
||||
else:
|
||||
json.dump(data, f)
|
||||
await AsyncPath(_get_cache_dir()).mkdir(parents=True, exist_ok=True)
|
||||
if data is None:
|
||||
text = json.dumps({"_cached_none": True})
|
||||
else:
|
||||
text = json.dumps(data)
|
||||
await AsyncPath(cache_path).write_text(text, encoding="utf-8")
|
||||
except Exception:
|
||||
pass # Cache write failures are not critical
|
||||
|
||||
@@ -111,7 +112,7 @@ async def tmdb_api_request(
|
||||
|
||||
# Check cache first (before adding API key to params for cache key)
|
||||
cache_path = _get_cache_path(endpoint, params)
|
||||
cached = _load_from_cache(cache_path)
|
||||
cached = await _load_from_cache(cache_path)
|
||||
if cached is not _NOT_FOUND:
|
||||
return cached
|
||||
|
||||
@@ -132,11 +133,11 @@ async def tmdb_api_request(
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
# Cache immediately after receiving response
|
||||
_save_to_cache(cache_path, data)
|
||||
await _save_to_cache(cache_path, data)
|
||||
return data
|
||||
except httpx.HTTPStatusError:
|
||||
# Cache the failure (None) to avoid retrying
|
||||
_save_to_cache(cache_path, None)
|
||||
await _save_to_cache(cache_path, None)
|
||||
return None
|
||||
except Exception:
|
||||
# Don't cache network errors - they may be transient
|
||||
|
||||
+21
-16
@@ -5,6 +5,8 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
from aiopathlib import AsyncPath
|
||||
|
||||
|
||||
# Default output folder name (created at common root of scanned paths)
|
||||
DEFAULT_OUTPUT_FOLDER = ".mediahive"
|
||||
@@ -23,7 +25,7 @@ RESOLUTION_PRIORITY = {
|
||||
}
|
||||
|
||||
|
||||
def get_added_timestamp(path: Path) -> Optional[int]:
|
||||
async def get_added_timestamp(path: Path) -> Optional[int]:
|
||||
"""
|
||||
Get the timestamp when a torrent was added to the collection.
|
||||
|
||||
@@ -35,12 +37,13 @@ def get_added_timestamp(path: Path) -> Optional[int]:
|
||||
Returns:
|
||||
Unix timestamp as int, or None if path doesn't exist
|
||||
"""
|
||||
ap = AsyncPath(path)
|
||||
try:
|
||||
stat_info = path.stat()
|
||||
except OSError, PermissionError:
|
||||
stat_info = await ap.stat()
|
||||
except (OSError, PermissionError):
|
||||
return None
|
||||
|
||||
if path.is_dir():
|
||||
if await ap.is_dir():
|
||||
return int(stat_info.st_ctime)
|
||||
|
||||
now = time.time()
|
||||
@@ -52,16 +55,17 @@ def get_added_timestamp(path: Path) -> Optional[int]:
|
||||
return int(atime)
|
||||
|
||||
|
||||
def get_directory_size(path: Path) -> int:
|
||||
async def get_directory_size(path: Path) -> int:
|
||||
"""Calculate total size of a directory recursively."""
|
||||
ap = AsyncPath(path)
|
||||
total = 0
|
||||
try:
|
||||
if path.is_file():
|
||||
return path.stat().st_size
|
||||
for item in path.rglob("*"):
|
||||
if item.is_file():
|
||||
total += item.stat().st_size
|
||||
except OSError, PermissionError:
|
||||
if await ap.is_file():
|
||||
return (await ap.stat()).st_size
|
||||
for item in ap.rglob("*"):
|
||||
if await AsyncPath(item).is_file():
|
||||
total += (await AsyncPath(item).stat()).st_size
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
return total
|
||||
|
||||
@@ -75,7 +79,7 @@ def format_size(size_bytes: int) -> str:
|
||||
return f"{size_bytes:.2f} PB"
|
||||
|
||||
|
||||
def find_common_root(paths: List[Path]) -> Optional[Path]:
|
||||
async def find_common_root(paths: List[Path]) -> Optional[Path]:
|
||||
"""
|
||||
Find the common root directory for a list of paths.
|
||||
|
||||
@@ -95,10 +99,10 @@ def find_common_root(paths: List[Path]) -> Optional[Path]:
|
||||
for p in resolved:
|
||||
# Find the first existing parent to get device info
|
||||
check_path = p
|
||||
while not check_path.exists() and check_path.parent != check_path:
|
||||
while not await AsyncPath(check_path).exists() and check_path.parent != check_path:
|
||||
check_path = check_path.parent
|
||||
if check_path.exists():
|
||||
devices.add(os.stat(check_path).st_dev)
|
||||
if await AsyncPath(check_path).exists():
|
||||
devices.add((await AsyncPath(check_path).stat()).st_dev)
|
||||
|
||||
if len(devices) > 1:
|
||||
# Paths are on different devices/drives
|
||||
@@ -109,7 +113,7 @@ def find_common_root(paths: List[Path]) -> Optional[Path]:
|
||||
# Find common path prefix
|
||||
if len(resolved) == 1:
|
||||
# Single path - use its parent as root
|
||||
return resolved[0].parent if resolved[0].is_file() else resolved[0]
|
||||
return resolved[0].parent if await AsyncPath(resolved[0]).is_file() else resolved[0]
|
||||
|
||||
# Get parts of each path
|
||||
all_parts = [p.parts for p in resolved]
|
||||
@@ -186,6 +190,7 @@ def sort_by_quality(items: list, reverse: bool = True) -> None:
|
||||
key=lambda v: (
|
||||
RESOLUTION_PRIORITY.get(_val(v, "resolution", "") or "", 0),
|
||||
_val(v, "size", 0) or 0,
|
||||
_val(v, "path", "") or "",
|
||||
),
|
||||
reverse=reverse,
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ readme = "README.md"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
"aiofiles>=25.1.0",
|
||||
"aiopathlib>=0.6.0",
|
||||
"bencodepy>=0.9.5",
|
||||
"fastapi-vue>=0.5.2",
|
||||
"fastapi[standard]>=0.128.0",
|
||||
|
||||
Reference in New Issue
Block a user