Merge pull request #55 from ASTRAOS-de/dev

Dev

Author: maldwg

.github/workflows/build_test_linux.yml

@@ -5,8 +5,6 @@ on:
       - main
       - dev
   pull_request:
-    branches:
-      - main
 jobs:
   build_test_linux:
     name: ${{ matrix.python-version }} on ${{ matrix.os }}
@@ -15,8 +13,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu-latest" ]
-        python-version: [ "3.11",
-                          "3.10",
+        python-version: [
+                          "3.12",
+                          "3.13",
+                          "3.14"
         ]
 
     steps:

.github/workflows/build_test_macos.yml

@@ -5,8 +5,6 @@ on:
       - main
       - dev
   pull_request:
-    branches:
-      - main
 jobs:
   build_test_macos:
     name: ${{ matrix.python-version }} on ${{ matrix.os }}
@@ -15,8 +13,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "macos-14" ]
-        python-version: [ "3.11",
-                          "3.10",
+        python-version: [ "3.12",
+                          "3.13",
+                          "3.14"
         ]
 
     steps:

.github/workflows/formatting.yml

@@ -6,8 +6,6 @@ on:
         - main
         - dev
     pull_request:
-        branches:
-        - main
 
 jobs:
   build:
@@ -16,10 +14,10 @@ jobs:
     steps:
     - uses: actions/checkout@v4.2.0
 
-    - name: Setup py-3.11
+    - name: Setup py-3.14
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: '3.14'
         cache: 'pip'
         cache-dependency-path: |
           requirements/requirements.*.txt

.github/workflows/publish_dev.yml

@@ -124,7 +124,7 @@ jobs:
           ref: ${{ github.event.pull_request.base.ref }}
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log into registry ${{ env.REGISTRY }}
         uses: docker/login-action@v3
@@ -143,12 +143,10 @@ jobs:
             type=raw,value=latest-dev
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           file: ./docker/dockerfiles/Dockerfile.${{ matrix.container }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/publish_main.yml

@@ -149,7 +149,7 @@ jobs:
           echo "major_minor=v${MAJOR}.${MINOR}" >> "$GITHUB_OUTPUT"
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log into registry ${{ env.REGISTRY }}
         uses: docker/login-action@v3
@@ -170,12 +170,10 @@ jobs:
             type=raw,value=latest
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           file: ./docker/dockerfiles/Dockerfile.${{ matrix.container }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/test_coverage.yml

@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu-latest" ]
-        python-version: [ "3.11" ]
+        python-version: [ "3.14" ]
     permissions:
       issues: write
       pull-requests: write
@@ -55,7 +55,7 @@ jobs:
         if: steps.pip-cache.outputs.cache-hit != 'true'
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements/requirements.dev.txt -r requirements/requirements.detector.txt -r requirements/requirements.logcollector.txt -r requirements/requirements.prefilter.txt -r requirements/requirements.inspector.txt -r requirements/requirements.logserver.txt
+          python -m pip install -r requirements/requirements.dev.txt -r requirements/requirements.detector.txt -r requirements/requirements.logcollector.txt -r requirements/requirements.prefilter.txt -r requirements/requirements.inspector.txt -r requirements/requirements.logserver.txt -r requirements/requirements.alerter.txt
 
       - name: Test
         run: |

README.md

@@ -53,6 +53,10 @@
 
 ## Getting Started
 
+### Requirements
+
+HAMSTRING currently supports Python 3.12 through 3.14.
+
 #### Run **HAMSTRING** using Docker Compose:
 ```sh
 HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile prod up

config.yaml

@@ -46,33 +46,33 @@ pipeline:
         max_workers: 1
         instances:
           dga_collector:
-            max_workers: 2
+            max_workers: 1
           domainator_collector:
-            max_workers: 2
+            max_workers: 1
       log_filtering.prefilter:
         executor: thread
-        max_workers: 2
+        max_workers: 1
         instances:
           dga_filter:
-            max_workers: 2
+            max_workers: 1
           no_filter:
-            max_workers: 2
+            max_workers: 1
       data_inspection.inspector:
         executor: thread
         max_workers: 2
         instances:
           dga_inspector:
-            max_workers: 2
+            max_workers: 1
           no_inspector:
-            max_workers: 2
+            max_workers: 1
       data_analysis.detector:
         executor: thread
-        max_workers: 2
+        max_workers: 1
         instances:
           RF-dga_detector:
-            max_workers: 2
+            max_workers: 1
           domainator:
-            max_workers: 3
+            max_workers: 2
           domainator_attributor:
             max_workers: 1
           domainator_attributor_behaviour:
@@ -83,12 +83,10 @@ pipeline:
             max_workers: 1
       pipeline.alerter:
         executor: thread
-        max_workers: 2
+        max_workers: 1
         instances:
           generic:
-            max_workers: 2
-          attributor:
-            max_workers: 2
+            max_workers: 1
       monitoring.agent:
         executor: thread
         max_workers: 2
@@ -197,7 +195,7 @@ pipeline:
       use_scaler: false
       checksum: a4aac4c585f1e614c3cf0d737e80b960c5de6e87b253f7cdd07125d9ce486476
       base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/
-      threshold: 0.05
+      threshold: 0.5
       consume_from: inspector
       inspector_name: "domainator_inspector"
       next_detectors:
@@ -218,7 +216,7 @@ pipeline:
       use_scaler: false
       checksum: d8f302edc166ecc80985838a30b5dff16ccc83480ea3c2480652f49c8f6b5e9b
       base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/
-      threshold: 0.05
+      threshold: 0.5
       consume_from: detector
       detector_name: "domainator"
       next_detectors: ""
@@ -236,7 +234,7 @@ pipeline:
       use_scaler: false
       checksum: 9a0970b4160b22f4c3c5ac99760f0ace5500dd25c5a195ff13254ad3c11d5dcd
       base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/
-      threshold: 0.05
+      threshold: 0.5
       consume_from: detector
       detector_name: "domainator"
       next_detectors: ""
@@ -254,7 +252,7 @@ pipeline:
       use_scaler: false
       checksum: 360bd26881beabce7e7581963240915de807c48b5e4a3501a657139f2ecb8a8b
       base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/
-      threshold: 0.05
+      threshold: 0.5
       consume_from: detector
       detector_name: "domainator"
       next_detectors: ""
@@ -288,7 +286,7 @@ pipeline:
         protocols:
           - dns
         interfaces:
-          - eth0
+          - wlp3s0
 
 environment:
   kafka_brokers:

docker/data/test_pcaps/ctu-sample.pcap

@@ -1,33 +1,19 @@
 services:
-  zookeeper:
-    extends:
-      file: "docker-compose/base/docker-compose.kafka.yml"
-      service: zookeeper
-
   kafka1:
     extends:
       file: "docker-compose/base/docker-compose.kafka.yml"
       service: kafka1
-    depends_on:
-      zookeeper:
-        condition: service_healthy
 
   kafka2:
     extends:
       file: "docker-compose/base/docker-compose.kafka.yml"
       service: kafka2
-    depends_on:
-      zookeeper:
-        condition: service_healthy
 
 
   kafka3:
     extends:
       file: "docker-compose/base/docker-compose.kafka.yml"
       service: kafka3
-    depends_on:
-      zookeeper:
-        condition: service_healthy
   clickhouse-server:
     extends:
       file: "docker-compose/base/docker-compose.clickhouse.yml"
@@ -305,8 +291,6 @@ networks:
 volumes:
   ch_data:
   ch_logs:
-  kafka-data1:
-  kafka-data2:
-  kafka-data3:
-  zk-data:
-  zk-txn-logs:
+  kafka-broker-data1:
+  kafka-broker-data2:
+  kafka-broker-data3: