Skip to content

Commit b227aa2

Browse files
samjacobs1020samjacobs1020
committed
add datatree demo notebook and numeric conversion fix
1 parent 81deed5 commit b227aa2

4 files changed

Lines changed: 161 additions & 36 deletions

File tree

‎demo/demo_datatree.ipynb‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "77d8504a",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"from pathlib import Path\n",
11+
"from hallmark.eht_datatree import build_tree, validate"
12+
]
13+
},
14+
{
15+
"cell_type": "markdown",
16+
"id": "bc6c59ff",
17+
"metadata": {},
18+
"source": [
19+
"Will build a branch for drives, a branch for each fmt, and a branch for all other files. hardcoded for one dataset right now"
20+
]
21+
},
22+
{
23+
"cell_type": "code",
24+
"execution_count": null,
25+
"id": "91c2e301",
26+
"metadata": {},
27+
"outputs": [],
28+
"source": [
29+
"root = Path(\"~/eht_m87_2019/EHTC_FirstM87Results_Apr2019\").expanduser()\n",
30+
"tree = build_tree(root)\n",
31+
"print(\"tree keys:\", list(tree.keys()))"
32+
]
33+
},
34+
{
35+
"cell_type": "code",
36+
"execution_count": null,
37+
"id": "0cb26601",
38+
"metadata": {},
39+
"outputs": [],
40+
"source": [
41+
"print(\"=== META ===\")\n",
42+
"print(tree[\"meta\"])"
43+
]
44+
},
45+
{
46+
"cell_type": "code",
47+
"execution_count": null,
48+
"id": "c1b7eb90",
49+
"metadata": {},
50+
"outputs": [],
51+
"source": [
52+
"print(\"=== DRIVES ===\")\n",
53+
"print(tree[\"drives\"])"
54+
]
55+
},
56+
{
57+
"cell_type": "code",
58+
"execution_count": null,
59+
"id": "dbc50055",
60+
"metadata": {},
61+
"outputs": [],
62+
"source": [
63+
"print(\"=== DATA STEMS ===\")\n",
64+
"for stem in tree[\"data\"].keys():\n",
65+
" print(f\" {stem}\")"
66+
]
67+
},
68+
{
69+
"cell_type": "code",
70+
"execution_count": null,
71+
"id": "d3747445",
72+
"metadata": {},
73+
"outputs": [],
74+
"source": [
75+
"print(\"=== ONE STEM ===\")\n",
76+
"print(tree[\"data\"][\"SR1_M87_2017_095_hi_hops_netcal_StokesI\"])"
77+
]
78+
}
79+
],
80+
"metadata": {
81+
"kernelspec": {
82+
"display_name": "base",
83+
"language": "python",
84+
"name": "python3"
85+
},
86+
"language_info": {
87+
"codemirror_mode": {
88+
"name": "ipython",
89+
"version": 3
90+
},
91+
"file_extension": ".py",
92+
"mimetype": "text/x-python",
93+
"name": "python",
94+
"nbconvert_exporter": "python",
95+
"pygments_lexer": "ipython3",
96+
"version": "3.13.13"
97+
}
98+
},
99+
"nbformat": 4,
100+
"nbformat_minor": 5
101+
}

‎mod/hallmark/eht_datatree.py‎

Lines changed: 31 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,19 @@
22
from pathlib import Path
33
from hallmark import ParaFrame
44

5-
"""
6-
Read INVENTORY.txt and return expected relative file paths.
5+
def read_inventory(root: Path) -> list[str]:
6+
"""
7+
Read INVENTORY.txt and return expected relative file paths.
78
8-
Args:
9-
root: Path to the dataset root directory containing INVENTORY.txt.
9+
Args:
10+
root: Path to the dataset root directory containing INVENTORY.txt.
1011
11-
Returns:
12-
List of relative file path strings expected to exist under root.
12+
Returns:
13+
List of relative file path strings expected to exist under root.
1314
14-
Raises:
15-
FileNotFoundError: If INVENTORY.txt does not exist under root.
16-
"""
17-
def read_inventory(root: Path) -> list[str]:
15+
Raises:
16+
FileNotFoundError: If INVENTORY.txt does not exist under root.
17+
"""
1818
inventory_path = Path(root) / "INVENTORY.txt"
1919
if not inventory_path.exists():
2020
raise FileNotFoundError(f"INVENTORY.txt not found in {root}")
@@ -29,17 +29,17 @@ def read_inventory(root: Path) -> list[str]:
2929
files_list.append(line)
3030
return files_list
3131

32-
"""
33-
Cross-check INVENTORY.txt against a set of tracked files in tree.
32+
def validate(root: Path, tracked: set[str]) -> bool:
33+
"""
34+
Cross-check INVENTORY.txt against a set of tracked files in tree.
3435
35-
Args:
36-
root: Path to the dataset root containing INVENTORY.txt.
37-
tracked: Set of relative file path strings already in the tree.
36+
Args:
37+
root: Path to the dataset root containing INVENTORY.txt.
38+
tracked: Set of relative file path strings already in the tree.
3839
39-
Returns:
40-
True if all inventory files are accounted for, False otherwise.
41-
"""
42-
def validate(root: Path, tracked: set[str]) -> bool:
40+
Returns:
41+
True if all inventory files are accounted for, False otherwise.
42+
"""
4343
files_list = read_inventory(root)
4444
missing = []
4545
# Check that every file in the inventory is present in the tracked set
@@ -54,24 +54,23 @@ def validate(root: Path, tracked: set[str]) -> bool:
5454
else:
5555
print(" ✓ all inventory files are present in the tree")
5656
return True
57-
58-
"""
59-
Build an in-memory pytree for an EHT dataset directory.
60-
61-
Args:
62-
root: Path to the EHT dataset root directory.
63-
64-
Returns:
65-
A dictionary with keys:
66-
- "meta" : ParaFrame of housekeeping files
67-
- "drives" : ParaFrame of compressed archives
68-
- "data" : dict of {stem -> ParaFrame}
69-
"""
57+
7058
# hardcoded formats for this dataset, based on inventory and globbing
7159
FMT_DRIVES = "{name}.tgz"
7260
FMT_DATA = "{ext}/SR1_M87_{year}_{day}_{band}_hops_netcal_StokesI.{ext}"
7361
def build_tree(root: Path) -> dict:
62+
"""
63+
Build an in-memory pytree for an EHT dataset directory.
7464
65+
Args:
66+
root: Path to the EHT dataset root directory.
67+
68+
Returns:
69+
A dictionary with keys:
70+
- "meta" : ParaFrame of housekeeping files
71+
- "drives" : ParaFrame of compressed archives
72+
- "data" : dict of {stem -> ParaFrame}
73+
"""
7574
root = Path(root).expanduser().resolve()
7675
# track files that are included in the tree, to cross-check against inventory
7776
tracked = set()

‎mod/hallmark/helper_functions.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414

1515
import re
16+
import pandas as pd
1617

1718

1819
def find_spec_by_fmt(fmt, encodings):
@@ -59,3 +60,29 @@ def regex_sub(value, yaml_encodings):
5960
result = re.sub(match.group(0), "-" + str(match.group(1)), result)
6061

6162
return result
63+
64+
def try_numeric_conversion(series):
65+
"""
66+
Attempt to convert a pandas Series to numeric.
67+
68+
Converts the series to numeric iff:
69+
1. All values are numeric
70+
2. Converting back to string matches the original values to avoid
71+
unintended conversions (e.g., "001" -> 1)
72+
73+
Args:
74+
series: A pandas Series of strings to attempt conversion on.
75+
76+
Returns:
77+
The converted numeric Series if both conditions are met,
78+
otherwise returns original series.
79+
"""
80+
converted = pd.to_numeric(series, errors="coerce")
81+
if converted.isna().any():
82+
return series
83+
if not all(str(int(numeric_val)) == str(original_val)
84+
or str(numeric_val) == str(original_val)
85+
for numeric_val, original_val in zip(converted, series)):
86+
return series
87+
return converted
88+

‎mod/hallmark/paraframe.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import pandas as pd
2222
import numpy as np
2323

24-
from .helper_functions import find_spec_by_fmt, regex_sub
24+
from .helper_functions import find_spec_by_fmt, regex_sub, try_numeric_conversion
2525

2626

2727
class ParaFrame(pd.DataFrame):
@@ -276,7 +276,5 @@ def parse(
276276
# if conversion fails the column stays as string
277277
result = cls(frame, encodings=encodings, base_path=base_path)
278278
for col in result.columns:
279-
converted = pd.to_numeric(result[col], errors="coerce")
280-
if not converted.isna().any():
281-
result[col] = converted
279+
result[col] = try_numeric_conversion(result[col])
282280
return result

0 commit comments

Comments
 (0)