From 6e8e6c638639f2a41d3e49b2bcf843ca8065d8b1 Mon Sep 17 00:00:00 2001 From: Matt Richards Date: Wed, 15 Jun 2022 09:26:22 +1000 Subject: [PATCH 1/2] debug --- sharrow/aster.py | 9 +++++++++ sharrow/flows.py | 39 +++++++++++++++++++++++++++++++++------ sharrow/relationships.py | 6 ++++-- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/sharrow/aster.py b/sharrow/aster.py index e746a83..b5c9acf 100755 --- a/sharrow/aster.py +++ b/sharrow/aster.py @@ -4,6 +4,8 @@ import sys import tokenize +from .debug import trunc_repr + try: from ast import unparse except ImportError: @@ -394,6 +396,8 @@ def _replacement( # return original_node dim_slots = self.dim_slots + logger.info(f"VLC: _replacement: topname={trunc_repr(topname)}, attr={attr}, dim_slots={trunc_repr(dim_slots)} spacevars={trunc_repr(self.spacevars)}" + f", transpose_lead={trunc_repr(transpose_lead)}, ") if isinstance(self.spacevars, dict): dim_slots = self.spacevars[attr] @@ -413,6 +417,11 @@ def _replacement( if isinstance(n, int): elts.append(ast.Name(id=f"_arg{n:02}", ctx=ast.Load())) elif isinstance(n, dict): + try: + n_val = n[missing_dim_value] + except KeyError: + tmp = repr(n) + print(f"missing dim failing, bubbling up\n {missing_dim_value}, '{tmp[:50]}...{tmp[-50:]}'") if sys.version_info >= (3, 8): elts.append( ast.Constant(n=n[missing_dim_value], ctx=ast.Load()) diff --git a/sharrow/flows.py b/sharrow/flows.py index e43d5a3..d13222c 100644 --- a/sharrow/flows.py +++ b/sharrow/flows.py @@ -21,6 +21,7 @@ from .filewrite import blacken, rewrite from .relationships import DataTree from .table import Table +from .debug import trunc_repr logger = logging.getLogger("sharrow") @@ -553,15 +554,17 @@ def __new__( cache_dir=None, name=None, dtype="float32", - boundscheck=False, + boundscheck=True, nopython=True, + # nopython=False, fastmath=True, parallel=True, readme=None, flow_library=None, extra_hash_data=(), write_hash_audit=True, - hashing_level=1, + # hashing_level=1, + hashing_level=100, dim_order=None, dim_exclude=None, ): @@ -803,8 +806,14 @@ def init_sub_funcs( for n, (k, expr) in enumerate(defs.items()): expr = str(expr).lstrip() init_expr = expr + print("expr:", expr) for spacename, spacearrays in self.tree.subspaces.items(): + print("\tspace", spacename) dim_slots, digital_encodings = meta_data[spacename] + # dim_slots: Dict[data_cols_in_xr or digitisedOffset of a relation, astNode] + # if skim, somtimes is a Dict[str, Dict[label ->index]] + # display(dim_slots) + # display(digital_encodings) try: expr = expression_for_numba( expr, @@ -814,6 +823,17 @@ def init_sub_funcs( digital_encodings=digital_encodings, ) except KeyError as key_err: + # print(key_err, type(key_err), type(repr(key_err))) + # print(vars(key_err)) + if str(key_err) == "None": + # apparently keyerrors can't have newlines + raise ValueError("VLCKeyError: expression_for_numba raised a null keyerror\n" + f"args were \n\t'{expr=}'\n\t'{spacename=}','{trunc_repr(dim_slots)=}...', '{digital_encodings=}'" + ) + print(key_err) + # if key_err.args is None: + # print("Custom:args none", key_err) + # raise if ".." in key_err.args[0]: topkey, attrkey = key_err.args[0].split("..") else: @@ -1279,9 +1299,9 @@ def iload_raw( ): assert isinstance(rg, DataTree) with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", category=nb.NumbaExperimentalFeatureWarning - ) + # warnings.filterwarnings( + # "ignore", category=nb.NumbaExperimentalFeatureWarning + # ) try: if runner is None: if mnl is not None: @@ -1320,10 +1340,16 @@ def iload_raw( tree_root_dims[i] for i in presorted(tree_root_dims, self.dim_order, self.dim_exclude) ] + # for c in argument: + # print(np.dtype(c)) + # logger.info(f"VLC: rg={rg}, {argshape=}, {arguments}, {kwargs}") return runner_(np.asarray(argshape), *arguments, **kwargs) - except nb.TypingError as err: + except nb.core.errors.TypingError as err: _raw_functions = getattr(self, "_raw_functions", {}) logger.error(f"nb.TypingError in {len(_raw_functions)} functions") + for c in arguments: + if c.dtype.kind in {'U', 'S'}: + logging.error(f"VLC: string array detected! {c}:shape={c.shape}") for k, v in _raw_functions.items(): logger.error(f"{k} = {v[0]} = {v[1]}") if "NameError:" in err.args[0]: @@ -1408,6 +1434,7 @@ def _load( if not source.relationships_are_digitized: source = source.digitize_relationships() if source.relationships_are_digitized: + # logger.info(f"VLC: {source=}, {runner=}, {dtype=}, {dot=}") if mnl_draws is None: result = self.iload_raw(source, runner=runner, dtype=dtype, dot=dot) else: diff --git a/sharrow/relationships.py b/sharrow/relationships.py index e6b0ed0..e9ee686 100644 --- a/sharrow/relationships.py +++ b/sharrow/relationships.py @@ -1033,6 +1033,7 @@ def digitize_relationships(self, inplace=False, redigitize=True): for e in obj._graph.edges: r = obj._get_relationship(e) + logger.info(f"processing relationship {r}, {r.parent_name}") if redigitize and r.analog: p_dataset = obj._graph.nodes[r.parent_data].get("dataset", None) if p_dataset is not None: @@ -1042,7 +1043,7 @@ def digitize_relationships(self, inplace=False, redigitize=True): if r.indexing == "label": p_dataset = obj._graph.nodes[r.parent_data].get("dataset", None) c_dataset = obj._graph.nodes[r.child_data].get("dataset", None) - + upstream = p_dataset[r.parent_name] downstream = c_dataset[r.child_name] @@ -1146,7 +1147,8 @@ def _arg_tokenizer(self, spacename, spacearray, exclude_dims=None): n_missing_tokens += 1 if n_missing_tokens > 1: - raise ValueError("at most one missing dimension is allowed") + raise ValueError("at most one missing dimension is allowed. VLC: probably a disconnected relationship " + "graph / unused variable") result = [] for t in tokens: if isinstance(t, str): From d4c1c4e6cb2991896443df44c6a12c5d0b1a82e6 Mon Sep 17 00:00:00 2001 From: Matt Richards Date: Wed, 15 Jun 2022 10:14:11 +1000 Subject: [PATCH 2/2] missing --- sharrow/debug.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 sharrow/debug.py diff --git a/sharrow/debug.py b/sharrow/debug.py new file mode 100644 index 0000000..d5f8818 --- /dev/null +++ b/sharrow/debug.py @@ -0,0 +1,6 @@ +def trunc_repr(thing, n=160): + f = n//2 + a = repr(thing) + if len(a) < n+20: + return a + return a[:f] + "..." + a[-f:] \ No newline at end of file