Introduce method to convert a simulation array's dtype

[apontzen]

Make KDTree construction automatically change dtypes if needed (and warn appropriately)

[Martin-Rey]

Thanks a lot @apontzen, this almost works.

I can now run the KD tree (although I am not seeing the warning about implicit conversion). Could it be because you are doing some type conversion during array creation? My mass array that used to be f4 is now f8 when I load it from disk?

Also, this creates a new issue when connecting arrays across families, e.g.

s.g['rho']
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In [13], line 1
----> 1 s.g['rho']

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/simsnap.py:261, in SimSnap.__getitem__(self, i)
    258 from . import subsnap
    260 if isinstance(i, str):
--> 261     return self._get_array_with_lazy_actions(i)
    262 elif isinstance(i, slice):
    263     return subsnap.SubSnap(self, i)

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/simsnap.py:359, in SimSnap._get_array_with_lazy_actions(self, name)
    357 if not self.lazy_off:
    358     if not self.lazy_load_off:
--> 359         self.__load_if_required(name)
    360     if not self.lazy_derive_off:
    361         self.__derive_if_required(name)

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/simsnap.py:370, in SimSnap.__load_if_required(self, name)
    368 if name not in list(self.keys()):
    369     try:
--> 370         self.__load_array_and_perform_postprocessing(name)
    371     except OSError:
    372         pass

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/simsnap.py:869, in SimSnap.__load_array_and_perform_postprocessing(self, array_name, fam)
    867 else:
    868     try:
--> 869         self._load_array(array_name, fam)
    870     except OSError:
    871         for fam_x in self.families():

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/subsnap.py:368, in FamilySubSnap._load_array(self, array_name, fam, **kwargs)
    366 def _load_array(self, array_name, fam=None, **kwargs):
    367     if fam is self._unifamily or fam is None:
--> 368         self._subsnap_base._load_array(array_name, self._unifamily)

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/gadgethdf.py:488, in GadgetHDFSnap._load_array(self, array_name, fam)
    485     target = self[fam]
    486     all_fams_to_load = [fam]
--> 488 target._create_array(array_name, dy, dtype=dtype)
    490 if units is not None:
    491     target[array_name].units = units

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/subsnap.py:349, in FamilySubSnap._create_array(self, array_name, ndim, dtype, zeros, derived, shared)
    347 def _create_array(self, array_name, ndim=1, dtype=None, zeros=True, derived=False, shared=None):
    348     # Array creation now maps into family-array creation in the parent
--> 349     self._subsnap_base._create_family_array(
    350         array_name, self._unifamily, ndim, dtype, derived, shared)

File ~/.python3_base/lib/python3.9/site-packages/pynbody/snapshot/simsnap.py:1199, in SimSnap._create_family_array(self, array_name, family, ndim, dtype, derived, shared, source_array)
   1193 fams.append(family)
   1195 if dtype is not None and dtx is not None and dtype != dtx:
   1196 
   1197     # We insist on the data types being the same for, e.g. sim.gas['my_prop'] and sim.star['my_prop']
   1198     # This makes promotion to simulation-level arrays possible.
-> 1199     raise ValueError("Requested data type {!r} is not consistent with existing data type {!r} for family array {!r}".format(
   1200         str(dtype), str(dtx), array_name))
   1202 if all([x in fams for x in self_families]):
   1203     # If, once we created this array, *all* families would have
   1204     # this array, just create a simulation-level array
   1205     if self._promote_family_array(array_name, ndim=ndim, derived=derived, shared=shared) is not None:

ValueError: Requested data type 'float32' is not consistent with existing data type 'float64' for family array 'rho'

[apontzen]

I don't think this snippet shows it is reading it as a different dtype, just that it is being converted to that dtype when the kdtree gets built. (Please avoid snapshots in favour of pasting text btw)

But that's a good point about what then happens when other bits of a family array get loaded. This is a good argument for the more technically complex, but more sensible, solution of just increasing the number of template parameters. I will give it some more thought, to see if there is a quicker fix. If not it might just have to wait until someone has time to do the proper solution.