Small changes

Author: stela2502

docs/11-performance.md

@@ -45,25 +45,42 @@ def timed(func, *args, **kwargs):
 
 # A slow approach: looping over rows
 
-This function computes row means and variances using a loop.
+This function computes the eucledian distance for two arrays.
 
 ```python
-def calc_mean_and_var_slow(mat):
-    means = []
-    variances = []
+import math
 
-    for i in range(mat.shape[0]):
-        row = mat.iloc[i, :]
-        means.append(np.mean(row))
-        variances.append(np.var(row, ddof=1))
+def euclid_distance_by_hand(v1, v2):
+    if len(v1) != len(v2):
+        raise ValueError(f"Length mismatch: {len(v1)} != {len(v2)}")
 
-    return {"means": means, "variances": variances}
+    s = 0.0
+    for k in range(len(v1)):
+        diff = float(v1[k]) - float(v2[k])
+        s += diff * diff
+
+    return math.sqrt(s)
+
+
+def distance_matrix_df(df, func):
+    X = np.asarray(df)
+    names = df.index
+    n = X.shape[0]
+
+    D = np.zeros((n, n))
+
+    for i in range(n):
+        for j in range(i,n):
+            D[i, j] = func(X[i], X[j])
+            D[j, i] = D[i, j]
+
+    return pd.DataFrame(D, index=names, columns=names)
 ```
 
 Time it:
 
 ```python
-(_, t_slow) = timed(calc_mean_and_var_slow, gmp_data)
+(_, t_slow) = timed(distance_matrix_df, hspc_data.iloc[range(200)], euclid_distance_by_hand )# first 200 genes of the whole data only
 print("slow:", t_slow)
 ```
 
@@ -74,77 +91,93 @@ print("slow:", t_slow)
 Pandas can do these operations in compiled code (fast).
 
 ```python
-def calc_mean_and_var_fast(mat):
-    return {
-        "means": mat.mean(axis=1),
-        "variances": mat.var(axis=1, ddof=1)
-    }
+def euclid_distance_vec(v1, v2):
+    v1 = np.asarray(v1, dtype=float)
+    v2 = np.asarray(v2, dtype=float)
+
+    return np.sqrt(np.sum((v1 - v2) ** 2))
+
 ```
 
 Time it:
 
 ```python
-(_, t_fast) = timed(calc_mean_and_var_fast, gmp_data)
-print("fast:", t_fast)
+(_, t_faster) = timed(distance_matrix_df, hspc_data.iloc[range(200)], euclid_distance_vec )
+print("faster:", t_faster)
 ```
 
 You should usually see a big speedup.
 
 ---
 
+But there are also specialized function that operate on the full matrix:
+
+```python
+from scipy.spatial.distance import pdist, squareform
+
+def vectorized_dist_mat(mat):
+    """
+    mat: pandas DataFrame or numpy array (rows = observations, cols = features)
+    returns: full (n x n) Euclidean distance matrix as numpy array
+    """
+    X = np.asarray(mat, dtype=float)
+
+    # upper triangle (condensed form)
+    upper = pdist(X, metric="euclidean")
+
+    # convert to full symmetric matrix
+    D = squareform(upper)
+
+    return D
+```
+
+I am sure by now you can check the run time without my help.
+
+
 # The key lesson
 
 For numerical work:
 
 ✅ Prefer **built-in** pandas/NumPy methods  
 ❌ Avoid Python loops over rows/columns
 
+If not 100% sure you know all about the possible libraries you could use I recommend asking an AI tool for the max speed up in your function.
+
 ---
 
 # What about pandas apply?
 
 In R, `apply()` can be a speed trick.  
 In Python, `DataFrame.apply()` usually still runs a Python function once per row/column.
 
-That means it often behaves like a loop (and can be slow).
-
-Example task:
+That means it often behaves like a loop (and can be slow). But eucledian distance is probably the worst example for apply as it works on one row/column only.
+So for this we switch to a simple mean calculation.
 
-For each row, compute:
-
-```
-(mean * std) / sum
+```python
+def mean_vec(vec1):
+    return vec1.mean() # very fast here
 ```
 
----
-
-# Fast (vectorized) solution
-
 ```python
-means = hspc_data.mean(axis=1)
-stds  = hspc_data.std(axis=1, ddof=1)
-sums  = hspc_data.sum(axis=1)
-
-result_vectorized = (means * stds) / sums
+(_, t_vec) = timed( hspc_data.apply, mean_vec , axis=1)
+print("apply time:", t_vec)
 ```
 
-Time it:
+This is rather fast, but using the inbuild pandas mean function is even faster:
+
 
 ```python
-(_, t_vec) = timed(lambda: (hspc_data.mean(axis=1) * hspc_data.std(axis=1, ddof=1)) / hspc_data.sum(axis=1))
-print("vectorized:", t_vec)
+(_, t_vec) = timed( hspc_data.mean,  axis=1)
+print("pandas time:", t_vec)
 ```
 
----
+If we would have our data as a numpy array instead we could get this even faster using numpy's vectorization:
 
-# Slower solution using apply
 
 ```python
-def example_func(v):
-    return (np.mean(v) * np.std(v, ddof=1)) / np.sum(v)
-
-(_, t_apply) = timed(hspc_data.apply, example_func, axis=1)
-print("apply:", t_apply)
+arr = np.array(hspc_data)
+(c, t_fast) = timed(arr.mean, axis=1 )
+print("numpy:", t_fast)
 ```
 
 ---

docs/15-kmeans-project.md

@@ -168,7 +168,7 @@ One simple idea is the average within-cluster pairwise distance.
 For clusters \(C_1, \ldots, C_K\):
 
 \[
-E = \frac{1}{K}\sum_{k=1}^{K} \left( \frac{1}{|C_k|(|C_k|-1)} \sum_{i \in C_k} \sum_{j \in C_k, j \neq i} d_{ij} \right)
+E = \frac{1}{K}\sum_{k=1}^{K} \left( \sum_{i \in C_k} \sum_{j \in C_k, j \neq i} d_{ij} \right)
 \]
 
 Lower energy means tighter clusters.
@@ -243,3 +243,12 @@ Before you start:
 - Remember: the goal is learning, not competing with libraries
 
 You now have the tools to tackle a real algorithmic problem in Python.
+Use Google, but no AI tool to learn e.g. how to accept user input in a python script.
+
+# Performance
+
+Once you have your script up and runnig and the results look accetable (plots!) and you still have some juce to program more:
+
+ 1. Store the disnatnce matrix and do nor re-calculate
+ 2. Store the per cluster energies and only re-caulaulte the ones affected by the move
+ 3. Only calculate a delta energy and not touch the stable genes in the clusters

scartchbook/how_to_parser_user_input.py

@@ -0,0 +1,49 @@
+#how_to_parser_user_input.py
+# Source - https://stackoverflow.com/a
+# Posted by JoErNanO, modified by community. See post 'Timeline' for change history
+# Retrieved 2026-01-28, License - CC BY-SA 3.0
+
+#!/usr/bin/python
+# coding: utf-8
+
+import argparse
+
+def parseArguments():
+    # Create argument parser
+    parser = argparse.ArgumentParser()
+
+    # Positional mandatory arguments
+    parser.add_argument("creditMom", help="Credit mom.", type=float)
+    parser.add_argument("creditDad", help="Credit dad.", type=float)
+    parser.add_argument("debtMom", help="Debt mom.", type=float)
+
+    # Optional arguments
+    parser.add_argument("-dD", "--debtDad", help="Debt dad.", type=float, default=1000.)
+    parser.add_argument("-s", "--salary", help="Debt dad.", type=float, default=2000.)
+    parser.add_argument("-b", "--bonus", help="Debt dad.", type=float, default=0.)
+
+    # Print version
+    parser.add_argument("--version", action="version", version='%(prog)s - Version 1.0')
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    return args
+
+def example(credit_mom, credit_dad, debt_mom, debt_dad = 1000, salary = 2000, bonus = 0):
+    total_gain = salary + credit_dad + credit_mom + bonus
+    total_loss = debt_dad + debt_mom
+
+    return total_gain - total_loss
+
+if __name__ == '__main__':
+    # Parse the arguments
+    args = parseArguments()
+
+    # Raw print arguments
+    print("You are running the script with arguments: ")
+    for a in args.__dict__:
+        print(str(a) + ": " + str(args.__dict__[a]))
+
+    # Run function
+    print(example(args.creditMom, args.creditDad, args.debtMom, args.debtDad, args.salary, args.bonus))