filter books by scraper and issues

backend/src/cms_backend/api/routes/books.py

@@ -1,4 +1,3 @@
-import datetime
 from http import HTTPStatus
 from typing import Annotated, Literal
 from uuid import UUID
@@ -35,6 +34,7 @@
 from cms_backend.schemas.models import (
     BookLanguagesSchema,
     BookUpdateSchema,
+    GetBooksSchema,
     ZimUrlsSchema,
 )
 from cms_backend.schemas.orms import (
@@ -50,51 +50,18 @@ class BookMoveSchema(BaseModel):
     destination: Literal["prod", "staging"]
 
 
-class BooksGetSchema(BaseModel):
-    skip: SkipField = 0
-    limit: LimitFieldMax200 = 20
-    id: NotEmptyString | None = None
-    has_title: bool | None = None
-    needs_processing: bool | None = None
-    has_error: bool | None = None
-    needs_file_operation: bool | None = None
-    location_kinds: list[NotEmptyString] | None = None
-    needs_attention: bool | None = None
-    has_backup: bool | None = None
-    updated_before: datetime.datetime | None = None
-    updated_after: datetime.datetime | None = None
-    name: NotEmptyString | None = None
-    flavour: NotEmptyString | None = None
-
-
 class RevertBookSchema(BaseModel):
     comment: NotEmptyString | None = None
 
 
 @router.get("")
 def get_books(
-    params: Annotated[BooksGetSchema, Query()],
+    params: Annotated[GetBooksSchema, Query()],
     session: Annotated[OrmSession, Depends(gen_dbsession)],
 ) -> ListResponse[BookLightSchema]:
     """Get a list of books"""
 
-    results = db_get_books(
-        session,
-        skip=params.skip,
-        limit=params.limit,
-        book_id=params.id,
-        has_title=params.has_title,
-        needs_processing=params.needs_processing,
-        has_error=params.has_error,
-        needs_file_operation=params.needs_file_operation,
-        location_kinds=params.location_kinds,
-        needs_attention=params.needs_attention,
-        updated_before=params.updated_before,
-        updated_after=params.updated_after,
-        name=params.name,
-        flavour=params.flavour,
-        has_backup=params.has_backup,
-    )
+    results = db_get_books(session, params=params)
 
     return ListResponse[BookLightSchema](
         meta=calculate_pagination_metadata(

backend/src/cms_backend/db/book.py

@@ -139,6 +139,7 @@ def create_book_full_schema(book: Book) -> BookFullSchema:
         zimcheck_summary=ZimcheckSummarySchema.model_validate(book.zimcheck_summary)
         if book.zimcheck_summary
         else None,
+        scraper=book.zim_metadata.get("Scraper"),
     )
 
 

backend/src/cms_backend/db/books.py

@@ -1,4 +1,3 @@
-import datetime
 from pathlib import Path
 from uuid import UUID
 
@@ -10,30 +9,19 @@
 from cms_backend.db import count_from_stmt
 from cms_backend.db.models import Book, BookLocation, Collection, CollectionTitle, Title
 from cms_backend.db.rules import has_flavour_mismatch
-from cms_backend.schemas.models import BookLanguagesSchema, ZimUrlSchema, ZimUrlsSchema
+from cms_backend.schemas.models import (
+    BookLanguagesSchema,
+    GetBooksSchema,
+    ZimUrlSchema,
+    ZimUrlsSchema,
+)
 from cms_backend.schemas.orms import BookLightSchema, ListResult
 from cms_backend.utils.filename import construct_download_url
 
 
 def get_books(
     session: OrmSession,
-    *,
-    skip: int,
-    limit: int,
-    book_id: str | None = None,
-    name: str | None = None,
-    flavour: str | None = None,
-    has_title: bool | None = None,
-    needs_processing: bool | None = None,
-    has_error: bool | None = None,
-    needs_file_operation: bool | None = None,
-    location_kinds: list[str] | None = None,
-    needs_attention: bool | None = None,
-    has_backup: bool | None = None,
-    updated_before: datetime.datetime | None = None,
-    updated_after: datetime.datetime | None = None,
-    created_before: datetime.datetime | None = None,
-    omit_book_ids: list[UUID] | None = None,
+    params: GetBooksSchema,
 ) -> ListResult[BookLightSchema]:
     """Get a list of books"""
 
@@ -52,48 +40,58 @@ def get_books(
         Book.flavour,
         Book.issues,
         Title.flavours,
+        Book.zim_metadata["Scraper"].astext.label("scraper"),
     ).join(Title, Book.title_id == Title.id, isouter=True)
 
-    if book_id is not None:
-        stmt = stmt.where(Book.id.cast(String).ilike(f"%{book_id}%"))
+    if params.id is not None:
+        stmt = stmt.where(Book.id.cast(String).ilike(f"%{params.id}%"))
 
-    if name is not None:
-        stmt = stmt.where(Book.name.ilike(f"%{name}%"))
+    if params.name is not None:
+        stmt = stmt.where(Book.name.ilike(f"%{params.name}%"))
 
-    if flavour is not None:
-        stmt = stmt.where(Book.flavour == flavour)
+    if params.flavour is not None:
+        stmt = stmt.where(Book.flavour == params.flavour)
 
-    if has_title is not None:
-        if has_title:
+    if params.has_title is not None:
+        if params.has_title:
             stmt = stmt.where(Book.title_id.is_not(None))
         else:
             stmt = stmt.where(Book.title_id.is_(None))
 
-    if needs_processing is not None:
-        stmt = stmt.where(Book.needs_processing == needs_processing)
+    if params.needs_processing is not None:
+        stmt = stmt.where(Book.needs_processing == params.needs_processing)
 
-    if has_error is not None:
-        stmt = stmt.where(Book.has_error == has_error)
+    if params.has_error is not None:
+        stmt = stmt.where(Book.has_error == params.has_error)
 
-    if needs_file_operation is not None:
-        stmt = stmt.where(Book.needs_file_operation == needs_file_operation)
+    if params.needs_file_operation is not None:
+        stmt = stmt.where(Book.needs_file_operation == params.needs_file_operation)
 
-    if location_kinds is not None:
-        stmt = stmt.where(Book.location_kind.in_(location_kinds))
+    if params.location_kinds is not None:
+        stmt = stmt.where(Book.location_kind.in_(params.location_kinds))
 
-    if updated_before is not None:
-        stmt = stmt.where(Book.updated_at < updated_before)
+    if params.updated_before is not None:
+        stmt = stmt.where(Book.updated_at < params.updated_before)
 
-    if updated_after is not None:
-        stmt = stmt.where(Book.updated_at > updated_after)
+    if params.updated_after is not None:
+        stmt = stmt.where(Book.updated_at > params.updated_after)
 
-    if created_before is not None:
-        stmt = stmt.where(Book.created_at < created_before)
+    if params.created_before is not None:
+        stmt = stmt.where(Book.created_at < params.created_before)
 
-    if omit_book_ids is not None:
-        stmt.where(Book.id.not_in(omit_book_ids))
+    if params.omit_book_ids is not None:
+        stmt = stmt.where(Book.id.not_in(params.omit_book_ids))
 
-    if needs_attention is True:
+    if params.scraper is not None:
+        stmt = stmt.where(
+            Book.zim_metadata.has_key("Scraper"),
+            Book.zim_metadata["Scraper"].astext.ilike(f"%{params.scraper}%"),
+        )
+
+    if params.issue is not None:
+        stmt = stmt.where(Book.issues.contains([params.issue]))
+
+    if params.needs_attention is True:
         stmt = stmt.where(
             or_(
                 Book.location_kind.in_(["quarantine", "staging"]),
@@ -103,7 +101,7 @@ def get_books(
                 Book.has_error.is_(True),
             )
         )
-    elif needs_attention is False:
+    elif params.needs_attention is False:
         stmt = stmt.where(
             and_(
                 Book.location_kind.not_in(["quarantine", "staging", "deleted"]),
@@ -114,15 +112,15 @@ def get_books(
             )
         )
 
-    if has_backup:
+    if params.has_backup:
         backup_books = (
             select(BookLocation.book_id)
             .where(BookLocation.status == "current", BookLocation.is_backup.is_(True))
             .distinct()
         )
         stmt = stmt.where(Book.id.in_(backup_books))
 
-    if needs_attention is True:
+    if params.needs_attention is True:
         order_clauses = [
             Book.has_error,
             Book.location_kind.in_(["staging", "to_delete"]).desc(),
@@ -167,6 +165,7 @@ def get_books(
                 has_flavour_mismatch=has_flavour_mismatch(flavour, title_flavours)
                 if title_flavours is not None
                 else False,
+                scraper=scraper,
             )
             for (
                 book_id_result,
@@ -183,8 +182,9 @@ def get_books(
                 flavour,
                 book_issues,
                 title_flavours,
+                scraper,
             ) in session.execute(
-                stmt.offset(skip).limit(limit).order_by(*order_clauses)
+                stmt.offset(params.skip).limit(params.limit).order_by(*order_clauses)
             ).all()
         ],
     )

backend/src/cms_backend/db/title.py

@@ -80,6 +80,7 @@ def create_title_full_schema(title: Title) -> TitleFullSchema:
                 flavour=book.flavour,
                 issues=book.issues,
                 has_flavour_mismatch=has_flavour_mismatch(book.flavour, title.flavours),
+                scraper=book.zim_metadata.get("Scraper"),
             )
             for book in sorted(
                 title.books,

backend/src/cms_backend/mill/mark_staging_books_for_deletion.py

@@ -6,6 +6,7 @@
 from cms_backend.db.book import delete_book
 from cms_backend.db.books import get_books
 from cms_backend.mill.context import Context as MillContext
+from cms_backend.schemas.models import GetBooksSchema
 from cms_backend.utils.datetime import getnow
 
 
@@ -16,13 +17,15 @@ def mark_staging_books_for_deletion(session: OrmSession):
     while True:
         results = get_books(
             session,
-            needs_file_operation=False,
-            needs_processing=False,
-            created_before=getnow() - MillContext.staging_books_lifespan,
-            skip=0,
-            limit=50,
-            location_kinds=["staging"],
-            omit_book_ids=omit_book_ids,
+            GetBooksSchema(
+                needs_file_operation=False,
+                needs_processing=False,
+                created_before=getnow() - MillContext.staging_books_lifespan,
+                skip=0,
+                limit=50,
+                location_kinds=["staging"],
+                omit_book_ids=omit_book_ids,
+            ),
         )
         if not results.records:
             break

backend/src/cms_backend/schemas/models.py

@@ -1,3 +1,4 @@
+import datetime
 import re
 from dataclasses import dataclass
 from pathlib import Path
@@ -10,7 +11,9 @@
     Base64Str,
     GraphemeLength,
     LangCode,
+    LimitFieldMax200,
     NotEmptyString,
+    SkipField,
     ZimFlavour,
 )
 from cms_backend.context import Context
@@ -35,6 +38,27 @@ class ZimUrlsSchema(BaseModel):
     urls: dict[UUID, list[ZimUrlSchema]]
 
 
+class GetBooksSchema(BaseModel):
+    skip: SkipField = 0
+    limit: LimitFieldMax200 = 20
+    id: NotEmptyString | None = None
+    name: NotEmptyString | None = None
+    flavour: NotEmptyString | None = None
+    has_title: bool | None = None
+    needs_processing: bool | None = None
+    has_error: bool | None = None
+    needs_file_operation: bool | None = None
+    location_kinds: list[NotEmptyString] | None = None
+    needs_attention: bool | None = None
+    has_backup: bool | None = None
+    updated_before: datetime.datetime | None = None
+    updated_after: datetime.datetime | None = None
+    created_before: datetime.datetime | None = None
+    omit_book_ids: list[UUID] | None = None
+    scraper: NotEmptyString | None = None
+    issue: NotEmptyString | None = None
+
+
 class BookLanguagesSchema(BaseModel):
     languages: list[str]
 

backend/src/cms_backend/schemas/orms.py

@@ -133,6 +133,7 @@ class BookLightSchema(BaseModel):
     date: str | None
     flavour: str | None
     issues: list[str]
+    scraper: str | None
     has_flavour_mismatch: bool
 
 

backend/tests/api/routes/test_books.py

@@ -277,6 +277,57 @@ def test_get_books_filter_by_id(
     assert response_doc["items"][0]["id"] == str(book1.id)
 
 
+def test_get_books_filter_by_scraper(
+    client: TestClient,
+    create_book: Callable[..., Book],
+):
+    """Test get books endpoint passes scraper filter to database layer"""
+
+    book1 = create_book(
+        _id=UUID("12345678-1234-5678-1234-567812345678"),
+        zim_metadata={"test": "book1", "Scraper": "mwoffliner 1.17.5"},
+    )
+    create_book(
+        _id=UUID("87654321-4321-8765-4321-876543218765"),
+        zim_metadata={"test": "book2", "Scraper": "sotoki 1.14"},
+    )
+
+    # Test that scraper parameter is passed through and filters correctly
+    response = client.get("/v1/books?scraper=mwoffliner")
+    assert response.status_code == HTTPStatus.OK
+    response_doc = response.json()
+    assert response_doc["meta"]["count"] == 1
+    assert response_doc["items"][0]["id"] == str(book1.id)
+
+
+def test_get_books_filter_by_issues(
+    dbsession: OrmSession,
+    client: TestClient,
+    create_book: Callable[..., Book],
+):
+    """Test get books endpoint passes scraper filter to database layer"""
+
+    book1 = create_book(
+        _id=UUID("12345678-1234-5678-1234-567812345678"),
+        zim_metadata={"test": "book1", "Scraper": "mwoffliner 1.17.5"},
+    )
+    book1.issues = ["flavour mismatch"]
+    dbsession.add(book1)
+    dbsession.flush()
+
+    create_book(
+        _id=UUID("87654321-4321-8765-4321-876543218765"),
+        zim_metadata={"test": "book2", "Scraper": "sotoki 1.14"},
+    )
+
+    # Test that scraper parameter is passed through and filters correctly
+    response = client.get("/v1/books?issue=flavour mismatch")
+    assert response.status_code == HTTPStatus.OK
+    response_doc = response.json()
+    assert response_doc["meta"]["count"] == 1
+    assert response_doc["items"][0]["id"] == str(book1.id)
+
+
 def test_get_book_languages(
     client: TestClient,
     dbsession: OrmSession,