download data in workflow

Author: wvictor14

.github/workflows/publish.yml

@@ -17,7 +17,6 @@ jobs:
       - name: Set up Quarto
         uses: quarto-dev/quarto-actions/setup@v2
 
-
       - name: "Set up Python"
         uses: actions/setup-python@v5
         with:
@@ -31,6 +30,15 @@ jobs:
 
       - name: Install the project
         run: uv sync --locked --all-extras --dev
+
+      - name: Install curl
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y curl jq unzip
+
+      - name: Download data
+        run: |
+          sh data/downloadLabourData.sh
       
       - name: Render and Publish
         uses: quarto-dev/quarto-actions/publish@v2

data/downloadLabourData.sh

@@ -1,13 +1,24 @@
-# download unadjusted 
-cd data
+#!/bin/bash
+
+# Set target directory (modify this path as needed)
+TARGET_DIR=data/
+
+# Create target directory if it doesn't exist
+mkdir -p "$TARGET_DIR"
 
-URL=$(curl -s 'https://www150.statcan.gc.ca/t1/wds/rest/getFullTableDownloadCSV/14100022/en' | grep -o 'https://[^"]*')
-curl -o 14100022-eng.zip "$URL"
-unzip 14100022-eng.zip
-rm 14100022-eng.zip
+# Change to target directory
+cd "$TARGET_DIR"
+
+# download unadjusted 
+# URL=$(curl -s 'https://www150.statcan.gc.ca/t1/wds/rest/getFullTableDownloadCSV/14100022/en' | grep -o 'https://[^"]*')
+# curl -o 14100022-eng.zip "$URL"
+# unzip 14100022-eng.zip
+# rm 14100022-eng.zip
 
 # download seasonally adjusted
+echo "Downloading to: $(pwd)"
 URL=$(curl -s 'https://www150.statcan.gc.ca/t1/wds/rest/getFullTableDownloadCSV/14100355/en' | jq -r '.object')
 curl -o 14100355-eng.zip "$URL"
 unzip 14100355-eng.zip
-rm 14100355-eng.zip
+rm 14100355-eng.zip
+echo "Download completed in: $TARGET_DIR"