Allow uploading unsuccessful dataset, and export failed rows (For conversions)

specifyweb/backend/workbench/upload/upload.py

@@ -175,6 +175,7 @@
         unupload_record(record, agent)
 
 
+FORCE_UPLOAD = True
 def do_upload_dataset(
     collection,
     uploading_agent_id: int,
@@ -198,7 +199,7 @@
     batch_edit_packs = [get_batch_edit_pack_from_row(ncols, row) for row in ds.data]
     base_table, upload_plan, batchEditPrefs = get_raw_ds_upload_plan(ds)
 
-    results = do_upload(
+    results, failed_rows = do_upload(
         collection,
         rows,
         upload_plan,
@@ -218,6 +219,19 @@
         ),
     )
     success = not any(r.contains_failure() for r in results)
+
+    # Export failed rows
+    if FORCE_UPLOAD:
+        # upload counts as success if at least one row uploaded
+        success = any(not r.contains_failure() for r in results)
+
+        agent = models.Agent.objects.get(id=uploading_agent_id)
+        user = agent.specifyuser
+
+        logger.debug(failed_rows)
+        if len(failed_rows) > 0:
+            export_failed_rows(failed_rows, user)
+
     if not no_commit:
         ds.uploadresult = {
             "success": success,
@@ -314,6 +328,7 @@
     base_table, plan, _ = get_raw_ds_upload_plan(ds)
     return base_table, plan.apply_scoping(collection)
 
+
 def do_upload(
     collection,
     rows: Rows,
@@ -341,6 +356,10 @@
 
     scope_context = ScopeContext()
 
+    if FORCE_UPLOAD:
+        allow_partial = True
+
+    failed_rows = []
     with savepoint("main upload"):
         tic = time.perf_counter()
         results: list[UploadResult] = []
@@ -414,8 +433,11 @@
                     f"finished row {len(results)}, cache size: {cache and len(cache)}"
                 )
                 if result.contains_failure():
-                    cache = _cache
-                    raise Rollback("failed row")
+                    if FORCE_UPLOAD:
+                        failed_rows.append(row)
+                    else:
+                        cache = _cache
+                        raise Rollback("failed row")
 
                 autonum_dispatcher.commit_highest()
 
@@ -427,11 +449,42 @@
         else:
             fixup_trees(scoped_table, results)
 
-    return results
-
+    return results, failed_rows
 
+from django.conf import settings
+import csv
+import re
+import os
+from specifyweb.notifications.models import Message
+import uuid
 do_upload_csv = do_upload
 
+def export_failed_rows(failed_rows, user):
+    message_type = "workbench-failed-rows"
+
+
+    filename = f"failed_rows_{uuid.uuid4().hex}.csv"
+    path = os.path.join(settings.DEPOSITORY_DIR, filename)
+    bom = True
+    delimiter = ','
+
+    encoding = 'utf-8-sig' if bom else 'utf-8'
+
+    column_order = None
+    if column_order is None:
+        column_order = list(failed_rows[0].keys())
+
+    with open(path, 'w', newline='', encoding=encoding) as f:
+        csv_writer = csv.DictWriter(f, fieldnames=column_order, delimiter=delimiter)
+        csv_writer.writeheader()
+        for row in failed_rows:
+            csv_writer.writerow(row)
+
+    Message.objects.create(user=user, content=json.dumps({
+        'type': message_type,
+        'file': filename,
+        'datasetname': 'PLACEHOLDER',
+    }))
 
 def validate_row(
     collection,

specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx

@@ -99,26 +99,40 @@
       </>
     );
   },
   'query-export-to-csv-complete'(notification) {
     return (
       <>
         {notificationsText.queryExportToCsvCompleted()}
         <Link.Success
           className="w-fit"
           download
           href={`/static/depository/${encodeURIComponent(
             notification.payload.file
           )}`}
         >
           {notificationsText.download()}
         </Link.Success>
       </>
     );
   },
-  'query-export-to-kml-complete'(notification) {
+  'query-export-to-csv-complete'(notification) {
+    return (
+      <>
+        {notificationsText.queryExportToCsvCompleted()}
+        <Link.Success
+          className="w-fit"
+          download
+          href={`/static/depository/${notification.payload.file}`}
+        >
+          {notificationsText.download()}
+        </Link.Success>
+      </>
+    );
+  },
+  'workbench-failed-rows'(notification) {
     return (
       <>
-        {notificationsText.queryExportToKmlCompleted()}
+        {notificationsText.workbenchFailedRows({name:notification.payload.datasetname})}
         <Link.Success
           className="w-fit"
           download

specifyweb/frontend/js_src/lib/components/WbActions/WbUpload.tsx

@@ -13,6 +13,8 @@ import type { WbCellCounts } from '../WorkBench/CellMeta';
 import type { WbMapping } from '../WorkBench/mapping';
 import type { WbStatus } from '../WorkBench/WbView';
 
+const FORCE_UPLOAD = true
+
 export function WbUpload({
   hasUnsavedChanges,
   mappings,
@@ -63,7 +65,7 @@ export function WbUpload({
         {noShowWarning || !isFromBatchEdit ? (
           <Button.Small
             aria-haspopup="dialog"
-            disabled={hasUnsavedChanges || cellCounts.invalidCells > 0}
+            disabled={hasUnsavedChanges || (cellCounts.invalidCells > 0 && FORCE_UPLOAD !== true)}
             title={
               hasUnsavedChanges
                 ? wbText.unavailableWhileEditing()

specifyweb/frontend/js_src/lib/localization/notifications.ts

@@ -124,6 +124,9 @@ export const notificationsText = createDictionary({
     'pt-br': 'Exportação da consulta para CSV concluída.',
     'hr-hr': 'Izvoz upita u CSV je završen.',
   },
+  workbenchFailedRows: {
+    'en-us': 'Csv with failed rows for dataset {name:string}.',
+  },
   queryExportToKmlCompleted: {
     'en-us': 'Query export to KML completed.',
     'ru-ru': 'Экспорт запроса в KML завершен.',