From 0cc2c566ab5fbbfea6ed47bb9b8da326d0b7b586 Mon Sep 17 00:00:00 2001 From: Uchechukwu Orji Date: Wed, 1 Jul 2026 21:17:31 +0100 Subject: [PATCH] filter books by offliner or issues and show books as cards --- backend/src/cms_backend/api/routes/books.py | 39 +--- backend/src/cms_backend/db/book.py | 1 + backend/src/cms_backend/db/books.py | 96 ++++----- backend/src/cms_backend/db/title.py | 1 + .../mill/mark_staging_books_for_deletion.py | 17 +- backend/src/cms_backend/schemas/models.py | 24 +++ backend/src/cms_backend/schemas/orms.py | 1 + backend/tests/api/routes/test_books.py | 51 +++++ backend/tests/api/routes/test_titles.py | 1 + backend/tests/db/test_books.py | 67 ++++--- dev/frontend-dev/config.json | 1 + frontend/public/config.json | 1 + frontend/src/components/BookCard.vue | 174 ++++++++++++++++ .../src/components/BookIssuesIndicator.vue | 70 +++++++ frontend/src/components/BookLocationChip.vue | 48 +++++ frontend/src/components/BookStatus.vue | 185 ------------------ .../src/components/BookStatusIndicator.vue | 59 ++++++ frontend/src/components/BookTable.vue | 106 ++++++++-- frontend/src/components/BooksViewFilters.vue | 6 +- frontend/src/components/EventFilters.vue | 2 +- frontend/src/components/InboxBookFilters.vue | 64 +++++- .../components/ZimfarmNotificationFilters.vue | 2 +- frontend/src/config.ts | 1 + frontend/src/stores/book.ts | 4 + frontend/src/stores/zimfarm/offliner.ts | 46 +++++ frontend/src/types/book.ts | 1 + frontend/src/utils/format.ts | 9 + frontend/src/views/BookView.vue | 30 ++- frontend/src/views/InboxView.vue | 35 +++- frontend/src/views/TitleView.vue | 38 ++-- 30 files changed, 831 insertions(+), 349 deletions(-) create mode 100644 frontend/src/components/BookCard.vue create mode 100644 frontend/src/components/BookIssuesIndicator.vue create mode 100644 frontend/src/components/BookLocationChip.vue delete mode 100644 frontend/src/components/BookStatus.vue create mode 100644 frontend/src/components/BookStatusIndicator.vue create mode 100644 frontend/src/stores/zimfarm/offliner.ts diff --git a/backend/src/cms_backend/api/routes/books.py b/backend/src/cms_backend/api/routes/books.py index cf929e2e..59464d62 100644 --- a/backend/src/cms_backend/api/routes/books.py +++ b/backend/src/cms_backend/api/routes/books.py @@ -1,4 +1,3 @@ -import datetime from http import HTTPStatus from typing import Annotated, Literal from uuid import UUID @@ -35,6 +34,7 @@ from cms_backend.schemas.models import ( BookLanguagesSchema, BookUpdateSchema, + GetBooksSchema, ZimUrlsSchema, ) from cms_backend.schemas.orms import ( @@ -50,51 +50,18 @@ class BookMoveSchema(BaseModel): destination: Literal["prod", "staging"] -class BooksGetSchema(BaseModel): - skip: SkipField = 0 - limit: LimitFieldMax200 = 20 - id: NotEmptyString | None = None - has_title: bool | None = None - needs_processing: bool | None = None - has_error: bool | None = None - needs_file_operation: bool | None = None - location_kinds: list[NotEmptyString] | None = None - needs_attention: bool | None = None - has_backup: bool | None = None - updated_before: datetime.datetime | None = None - updated_after: datetime.datetime | None = None - name: NotEmptyString | None = None - flavour: NotEmptyString | None = None - - class RevertBookSchema(BaseModel): comment: NotEmptyString | None = None @router.get("") def get_books( - params: Annotated[BooksGetSchema, Query()], + params: Annotated[GetBooksSchema, Query()], session: Annotated[OrmSession, Depends(gen_dbsession)], ) -> ListResponse[BookLightSchema]: """Get a list of books""" - results = db_get_books( - session, - skip=params.skip, - limit=params.limit, - book_id=params.id, - has_title=params.has_title, - needs_processing=params.needs_processing, - has_error=params.has_error, - needs_file_operation=params.needs_file_operation, - location_kinds=params.location_kinds, - needs_attention=params.needs_attention, - updated_before=params.updated_before, - updated_after=params.updated_after, - name=params.name, - flavour=params.flavour, - has_backup=params.has_backup, - ) + results = db_get_books(session, params=params) return ListResponse[BookLightSchema]( meta=calculate_pagination_metadata( diff --git a/backend/src/cms_backend/db/book.py b/backend/src/cms_backend/db/book.py index d975d5fe..f5c03804 100644 --- a/backend/src/cms_backend/db/book.py +++ b/backend/src/cms_backend/db/book.py @@ -139,6 +139,7 @@ def create_book_full_schema(book: Book) -> BookFullSchema: zimcheck_summary=ZimcheckSummarySchema.model_validate(book.zimcheck_summary) if book.zimcheck_summary else None, + offliner=book.zim_metadata.get("Scraper"), ) diff --git a/backend/src/cms_backend/db/books.py b/backend/src/cms_backend/db/books.py index f6e46bdc..77cfdb04 100644 --- a/backend/src/cms_backend/db/books.py +++ b/backend/src/cms_backend/db/books.py @@ -1,4 +1,3 @@ -import datetime from pathlib import Path from uuid import UUID @@ -10,30 +9,19 @@ from cms_backend.db import count_from_stmt from cms_backend.db.models import Book, BookLocation, Collection, CollectionTitle, Title from cms_backend.db.rules import has_flavour_mismatch -from cms_backend.schemas.models import BookLanguagesSchema, ZimUrlSchema, ZimUrlsSchema +from cms_backend.schemas.models import ( + BookLanguagesSchema, + GetBooksSchema, + ZimUrlSchema, + ZimUrlsSchema, +) from cms_backend.schemas.orms import BookLightSchema, ListResult from cms_backend.utils.filename import construct_download_url def get_books( session: OrmSession, - *, - skip: int, - limit: int, - book_id: str | None = None, - name: str | None = None, - flavour: str | None = None, - has_title: bool | None = None, - needs_processing: bool | None = None, - has_error: bool | None = None, - needs_file_operation: bool | None = None, - location_kinds: list[str] | None = None, - needs_attention: bool | None = None, - has_backup: bool | None = None, - updated_before: datetime.datetime | None = None, - updated_after: datetime.datetime | None = None, - created_before: datetime.datetime | None = None, - omit_book_ids: list[UUID] | None = None, + params: GetBooksSchema, ) -> ListResult[BookLightSchema]: """Get a list of books""" @@ -52,48 +40,58 @@ def get_books( Book.flavour, Book.issues, Title.flavours, + Book.zim_metadata["Scraper"].astext.label("scraper"), ).join(Title, Book.title_id == Title.id, isouter=True) - if book_id is not None: - stmt = stmt.where(Book.id.cast(String).ilike(f"%{book_id}%")) + if params.id is not None: + stmt = stmt.where(Book.id.cast(String).ilike(f"%{params.id}%")) - if name is not None: - stmt = stmt.where(Book.name.ilike(f"%{name}%")) + if params.name is not None: + stmt = stmt.where(Book.name.ilike(f"%{params.name}%")) - if flavour is not None: - stmt = stmt.where(Book.flavour == flavour) + if params.flavour is not None: + stmt = stmt.where(Book.flavour == params.flavour) - if has_title is not None: - if has_title: + if params.has_title is not None: + if params.has_title: stmt = stmt.where(Book.title_id.is_not(None)) else: stmt = stmt.where(Book.title_id.is_(None)) - if needs_processing is not None: - stmt = stmt.where(Book.needs_processing == needs_processing) + if params.needs_processing is not None: + stmt = stmt.where(Book.needs_processing == params.needs_processing) - if has_error is not None: - stmt = stmt.where(Book.has_error == has_error) + if params.has_error is not None: + stmt = stmt.where(Book.has_error == params.has_error) - if needs_file_operation is not None: - stmt = stmt.where(Book.needs_file_operation == needs_file_operation) + if params.needs_file_operation is not None: + stmt = stmt.where(Book.needs_file_operation == params.needs_file_operation) - if location_kinds is not None: - stmt = stmt.where(Book.location_kind.in_(location_kinds)) + if params.location_kinds is not None: + stmt = stmt.where(Book.location_kind.in_(params.location_kinds)) - if updated_before is not None: - stmt = stmt.where(Book.updated_at < updated_before) + if params.updated_before is not None: + stmt = stmt.where(Book.updated_at < params.updated_before) - if updated_after is not None: - stmt = stmt.where(Book.updated_at > updated_after) + if params.updated_after is not None: + stmt = stmt.where(Book.updated_at > params.updated_after) - if created_before is not None: - stmt = stmt.where(Book.created_at < created_before) + if params.created_before is not None: + stmt = stmt.where(Book.created_at < params.created_before) - if omit_book_ids is not None: - stmt.where(Book.id.not_in(omit_book_ids)) + if params.omit_book_ids is not None: + stmt = stmt.where(Book.id.not_in(params.omit_book_ids)) - if needs_attention is True: + if params.offliner is not None: + stmt = stmt.where( + Book.zim_metadata.has_key("Scraper"), + Book.zim_metadata["Scraper"].astext.ilike(f"%{params.offliner}%"), + ) + + if params.issue is not None: + stmt = stmt.where(Book.issues.contains([params.issue])) + + if params.needs_attention is True: stmt = stmt.where( or_( Book.location_kind.in_(["quarantine", "staging"]), @@ -103,7 +101,7 @@ def get_books( Book.has_error.is_(True), ) ) - elif needs_attention is False: + elif params.needs_attention is False: stmt = stmt.where( and_( Book.location_kind.not_in(["quarantine", "staging", "deleted"]), @@ -114,7 +112,7 @@ def get_books( ) ) - if has_backup: + if params.has_backup: backup_books = ( select(BookLocation.book_id) .where(BookLocation.status == "current", BookLocation.is_backup.is_(True)) @@ -122,7 +120,7 @@ def get_books( ) stmt = stmt.where(Book.id.in_(backup_books)) - if needs_attention is True: + if params.needs_attention is True: order_clauses = [ Book.has_error, Book.location_kind.in_(["staging", "to_delete"]).desc(), @@ -167,6 +165,7 @@ def get_books( has_flavour_mismatch=has_flavour_mismatch(flavour, title_flavours) if title_flavours is not None else False, + offliner=scraper, ) for ( book_id_result, @@ -183,8 +182,9 @@ def get_books( flavour, book_issues, title_flavours, + scraper, ) in session.execute( - stmt.offset(skip).limit(limit).order_by(*order_clauses) + stmt.offset(params.skip).limit(params.limit).order_by(*order_clauses) ).all() ], ) diff --git a/backend/src/cms_backend/db/title.py b/backend/src/cms_backend/db/title.py index 70304dcb..3fca1c5a 100644 --- a/backend/src/cms_backend/db/title.py +++ b/backend/src/cms_backend/db/title.py @@ -80,6 +80,7 @@ def create_title_full_schema(title: Title) -> TitleFullSchema: flavour=book.flavour, issues=book.issues, has_flavour_mismatch=has_flavour_mismatch(book.flavour, title.flavours), + offliner=book.zim_metadata.get("Scraper"), ) for book in sorted( title.books, diff --git a/backend/src/cms_backend/mill/mark_staging_books_for_deletion.py b/backend/src/cms_backend/mill/mark_staging_books_for_deletion.py index 5046c170..0a993fd3 100644 --- a/backend/src/cms_backend/mill/mark_staging_books_for_deletion.py +++ b/backend/src/cms_backend/mill/mark_staging_books_for_deletion.py @@ -6,6 +6,7 @@ from cms_backend.db.book import delete_book from cms_backend.db.books import get_books from cms_backend.mill.context import Context as MillContext +from cms_backend.schemas.models import GetBooksSchema from cms_backend.utils.datetime import getnow @@ -16,13 +17,15 @@ def mark_staging_books_for_deletion(session: OrmSession): while True: results = get_books( session, - needs_file_operation=False, - needs_processing=False, - created_before=getnow() - MillContext.staging_books_lifespan, - skip=0, - limit=50, - location_kinds=["staging"], - omit_book_ids=omit_book_ids, + GetBooksSchema( + needs_file_operation=False, + needs_processing=False, + created_before=getnow() - MillContext.staging_books_lifespan, + skip=0, + limit=50, + location_kinds=["staging"], + omit_book_ids=omit_book_ids, + ), ) if not results.records: break diff --git a/backend/src/cms_backend/schemas/models.py b/backend/src/cms_backend/schemas/models.py index 8ecba0ff..8cfa5002 100644 --- a/backend/src/cms_backend/schemas/models.py +++ b/backend/src/cms_backend/schemas/models.py @@ -1,3 +1,4 @@ +import datetime import re from dataclasses import dataclass from pathlib import Path @@ -10,7 +11,9 @@ Base64Str, GraphemeLength, LangCode, + LimitFieldMax200, NotEmptyString, + SkipField, ZimFlavour, ) from cms_backend.context import Context @@ -37,6 +40,27 @@ class ZimUrlsSchema(BaseModel): urls: dict[UUID, list[ZimUrlSchema]] +class GetBooksSchema(BaseModel): + skip: SkipField = 0 + limit: LimitFieldMax200 = 20 + id: NotEmptyString | None = None + name: NotEmptyString | None = None + flavour: NotEmptyString | None = None + has_title: bool | None = None + needs_processing: bool | None = None + has_error: bool | None = None + needs_file_operation: bool | None = None + location_kinds: list[NotEmptyString] | None = None + needs_attention: bool | None = None + has_backup: bool | None = None + updated_before: datetime.datetime | None = None + updated_after: datetime.datetime | None = None + created_before: datetime.datetime | None = None + omit_book_ids: list[UUID] | None = None + offliner: NotEmptyString | None = None + issue: NotEmptyString | None = None + + class BookLanguagesSchema(BaseModel): languages: list[str] diff --git a/backend/src/cms_backend/schemas/orms.py b/backend/src/cms_backend/schemas/orms.py index f8e6af1c..f813c210 100644 --- a/backend/src/cms_backend/schemas/orms.py +++ b/backend/src/cms_backend/schemas/orms.py @@ -135,6 +135,7 @@ class BookLightSchema(BaseModel): date: str | None flavour: str | None issues: list[str] + offliner: str | None has_flavour_mismatch: bool diff --git a/backend/tests/api/routes/test_books.py b/backend/tests/api/routes/test_books.py index bf18f413..88d04f12 100644 --- a/backend/tests/api/routes/test_books.py +++ b/backend/tests/api/routes/test_books.py @@ -277,6 +277,57 @@ def test_get_books_filter_by_id( assert response_doc["items"][0]["id"] == str(book1.id) +def test_get_books_filter_by_offliner( + client: TestClient, + create_book: Callable[..., Book], +): + """Test get books endpoint passes offliner filter to database layer""" + + book1 = create_book( + _id=UUID("12345678-1234-5678-1234-567812345678"), + zim_metadata={"test": "book1", "Scraper": "mwoffliner 1.17.5"}, + ) + create_book( + _id=UUID("87654321-4321-8765-4321-876543218765"), + zim_metadata={"test": "book2", "Scraper": "sotoki 1.14"}, + ) + + # Test that offliner parameter is passed through and filters correctly + response = client.get("/v1/books?offliner=mwoffliner") + assert response.status_code == HTTPStatus.OK + response_doc = response.json() + assert response_doc["meta"]["count"] == 1 + assert response_doc["items"][0]["id"] == str(book1.id) + + +def test_get_books_filter_by_issues( + dbsession: OrmSession, + client: TestClient, + create_book: Callable[..., Book], +): + """Test get books endpoint passes scraper filter to database layer""" + + book1 = create_book( + _id=UUID("12345678-1234-5678-1234-567812345678"), + zim_metadata={"test": "book1", "Scraper": "mwoffliner 1.17.5"}, + ) + book1.issues = ["flavour mismatch"] + dbsession.add(book1) + dbsession.flush() + + create_book( + _id=UUID("87654321-4321-8765-4321-876543218765"), + zim_metadata={"test": "book2", "Scraper": "sotoki 1.14"}, + ) + + # Test that scraper parameter is passed through and filters correctly + response = client.get("/v1/books?issue=flavour mismatch") + assert response.status_code == HTTPStatus.OK + response_doc = response.json() + assert response_doc["meta"]["count"] == 1 + assert response_doc["items"][0]["id"] == str(book1.id) + + def test_get_book_languages( client: TestClient, dbsession: OrmSession, diff --git a/backend/tests/api/routes/test_titles.py b/backend/tests/api/routes/test_titles.py index 901d5799..fd4519b8 100644 --- a/backend/tests/api/routes/test_titles.py +++ b/backend/tests/api/routes/test_titles.py @@ -395,6 +395,7 @@ def test_get_title_by_id_with_books( "deletion_date", "issues", "has_flavour_mismatch", + "offliner", } assert data["books"][0]["title_id"] == str(title.id) assert data["books"][1]["title_id"] == str(title.id) diff --git a/backend/tests/db/test_books.py b/backend/tests/db/test_books.py index 006424b5..0ceac1ff 100644 --- a/backend/tests/db/test_books.py +++ b/backend/tests/db/test_books.py @@ -24,6 +24,7 @@ Title, Warehouse, ) +from cms_backend.schemas.models import GetBooksSchema from cms_backend.utils.datetime import getnow @@ -54,7 +55,7 @@ def test_get_book_exist(dbsession: OrmSession, book: Book): def test_get_books_empty(dbsession: OrmSession): """Returns empty list when no books exist""" - results = get_books(dbsession, skip=0, limit=20) + results = get_books(dbsession, GetBooksSchema(skip=0, limit=20)) assert results.nb_records == 0 assert len(results.records) == 0 @@ -70,7 +71,7 @@ def test_get_books_with_data( book = create_book() books.append(book) - results = get_books(dbsession, skip=0, limit=20) + results = get_books(dbsession, GetBooksSchema(skip=0, limit=20)) assert results.nb_records == 5 assert len(results.records) == 5 @@ -103,8 +104,10 @@ def test_get_books_pagination( results = get_books( dbsession, - skip=skip, - limit=limit, + GetBooksSchema( + skip=skip, + limit=limit, + ), ) assert results.nb_records == 8 assert len(results.records) <= limit @@ -144,9 +147,11 @@ def test_get_books_filter_by_has_title( results = get_books( dbsession, - skip=0, - limit=20, - has_title=has_title, + GetBooksSchema( + skip=0, + limit=20, + has_title=has_title, + ), ) assert results.nb_records == expected_count assert len(results.records) == expected_count @@ -202,9 +207,11 @@ def test_get_books_book_id_filter( results = get_books( dbsession, - skip=0, - limit=20, - book_id=book_id_filter, + GetBooksSchema( + skip=0, + limit=20, + id=book_id_filter, + ), ) assert results.nb_records == expected_count @@ -245,21 +252,27 @@ def test_get_books_book_id_combined_with_other_filters( dbsession.flush() # Filter by ID pattern "aaaa" - should match book1 and book2 - results = get_books(dbsession, skip=0, limit=20, book_id="aaaa") + results = get_books(dbsession, GetBooksSchema(skip=0, limit=20, id="aaaa")) assert results.nb_records == 2 # Filter by ID pattern "aaaa" AND has_title=true - should only match book1 - results = get_books(dbsession, skip=0, limit=20, book_id="aaaa", has_title=True) + results = get_books( + dbsession, GetBooksSchema(skip=0, limit=20, id="aaaa", has_title=True) + ) assert results.nb_records == 1 assert results.records[0].id == book1.id # Filter by ID pattern "aaaa" AND has_title=false - should only match book2 - results = get_books(dbsession, skip=0, limit=20, book_id="aaaa", has_title=False) + results = get_books( + dbsession, GetBooksSchema(skip=0, limit=20, id="aaaa", has_title=False) + ) assert results.nb_records == 1 assert results.records[0].id == book2.id # Filter by ID pattern "bbbb" AND has_title=true - should only match book3 - results = get_books(dbsession, skip=0, limit=20, book_id="bbbb", has_title=True) + results = get_books( + dbsession, GetBooksSchema(skip=0, limit=20, id="bbbb", has_title=True) + ) assert results.nb_records == 1 assert results.records[0].id == book3.id @@ -307,9 +320,11 @@ def test_get_books_filter_by_needs_attention( results = get_books( dbsession, - skip=0, - limit=20, - needs_attention=needs_attention, + GetBooksSchema( + skip=0, + limit=20, + needs_attention=needs_attention, + ), ) assert results.nb_records == expected_count @@ -1226,10 +1241,12 @@ def test_get_books_time_filters( results = get_books( dbsession, - skip=0, - limit=20, - updated_after=updated_after, - updated_before=updated_before, + GetBooksSchema( + skip=0, + limit=20, + updated_after=updated_after, + updated_before=updated_before, + ), ) assert results.nb_records == expected_count assert len(results.records) == expected_count @@ -1270,9 +1287,11 @@ def test_get_books_created_before_filters( results = get_books( dbsession, - skip=0, - limit=20, - created_before=created_before, + GetBooksSchema( + skip=0, + limit=20, + created_before=created_before, + ), ) assert results.nb_records == expected_count assert len(results.records) == expected_count diff --git a/dev/frontend-dev/config.json b/dev/frontend-dev/config.json index b8aafe84..2909dfe5 100644 --- a/dev/frontend-dev/config.json +++ b/dev/frontend-dev/config.json @@ -1,5 +1,6 @@ { "CMS_API": "http://localhost:37601/v1", + "ZIMFARM_API": "http://localhost:8000/v2", "OAUTH_BASE_URL": "https://ory.login-staging.kiwix.org", "LOGIN_MODES": ["local", "oauth"] } diff --git a/frontend/public/config.json b/frontend/public/config.json index 2d5e951c..683097ad 100644 --- a/frontend/public/config.json +++ b/frontend/public/config.json @@ -1,5 +1,6 @@ { "CMS_API": "https://api.cms.openzim.org/v1", + "ZIMFARM_API": "https://api.farm.openzim.org/v2", "MATOMO_ENABLED": false, "MATOMO_HOST": "https://stats.kiwix.org", "MATOMO_SITE_ID": 12, diff --git a/frontend/src/components/BookCard.vue b/frontend/src/components/BookCard.vue new file mode 100644 index 00000000..a79f18df --- /dev/null +++ b/frontend/src/components/BookCard.vue @@ -0,0 +1,174 @@ + + + diff --git a/frontend/src/components/BookIssuesIndicator.vue b/frontend/src/components/BookIssuesIndicator.vue new file mode 100644 index 00000000..18c12068 --- /dev/null +++ b/frontend/src/components/BookIssuesIndicator.vue @@ -0,0 +1,70 @@ + + + diff --git a/frontend/src/components/BookLocationChip.vue b/frontend/src/components/BookLocationChip.vue new file mode 100644 index 00000000..a0971a7c --- /dev/null +++ b/frontend/src/components/BookLocationChip.vue @@ -0,0 +1,48 @@ + + + diff --git a/frontend/src/components/BookStatus.vue b/frontend/src/components/BookStatus.vue deleted file mode 100644 index 70ec3e20..00000000 --- a/frontend/src/components/BookStatus.vue +++ /dev/null @@ -1,185 +0,0 @@ - - - diff --git a/frontend/src/components/BookStatusIndicator.vue b/frontend/src/components/BookStatusIndicator.vue new file mode 100644 index 00000000..0a7c0585 --- /dev/null +++ b/frontend/src/components/BookStatusIndicator.vue @@ -0,0 +1,59 @@ + + + diff --git a/frontend/src/components/BookTable.vue b/frontend/src/components/BookTable.vue index b000e1b1..e9d68a66 100644 --- a/frontend/src/components/BookTable.vue +++ b/frontend/src/components/BookTable.vue @@ -12,7 +12,7 @@ :items-per-page-options="isServerSide ? limits : undefined" :mobile="smAndDown" :density="smAndDown ? 'compact' : 'comfortable'" - class="elevation-1 book-table" + :class="['elevation-1', 'book-table', { 'card-mode': displayMode === 'card' }]" item-value="id" hover @update:options="isServerSide ? onUpdateOptions($event) : undefined" @@ -29,6 +29,19 @@ + + + + + +