From f5c3a4513e4fa03dd4eeb4788cfea4a99c9fcf90 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 31 Oct 2017 12:43:36 -0400 Subject: [PATCH 01/42] add sfhistory properties function (to new Michael folder) with property to calculate inner SFHistory of galaxies. Change bins to 2000 (10 Myr). --- michaels_properties/sfhistory.py | 30 ++++++++++++++++++++++++++++++ tangos/properties/__init__.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 michaels_properties/sfhistory.py diff --git a/michaels_properties/sfhistory.py b/michaels_properties/sfhistory.py new file mode 100644 index 00000000..fa4aa9e7 --- /dev/null +++ b/michaels_properties/sfhistory.py @@ -0,0 +1,30 @@ +#from __future__ import absolute_import + +from tangos.properties import HaloProperties, TimeChunkedProperty +import numpy as np +import pynbody + +class InnerStarFormHistogram(TimeChunkedProperty): + + _maxr_frac = 0.1 + + @classmethod + def name(self): + return "inner_SFR_histogram" + + def requires_property(self): + return ["SSC", "Rvir"] + + def calculate(self, halo, existing_properties): + filter = pynbody.filt.Sphere(self._maxr_frac*existing_properties['Rvir'],cen=existing_properties['SSC']) + M,_ = np.histogram(halo.st[filter]['tform'].in_units("Gyr"),weights=halo.st[filter]['massform'].in_units("Msol"),bins=self.nbins,range=(0,self.tmax_Gyr)) + t_now = halo.properties['time'].in_units("Gyr") + M/=self.delta_t + M = M[self.store_slice(t_now)] + + return M + + @classmethod + def reassemble(cls, *options): + reassembled = TimeChunkedProperty.reassemble(*options) + return reassembled/1e9 # Msol per Gyr -> Msol per yr diff --git a/tangos/properties/__init__.py b/tangos/properties/__init__.py index 5c89818b..081fae3b 100644 --- a/tangos/properties/__init__.py +++ b/tangos/properties/__init__.py @@ -197,7 +197,7 @@ class TimeChunkedProperty(HaloProperties): """TimeChunkedProperty implements a special type of halo property where chunks of a histogram are stored at each time step, then appropriately reassembled when the histogram is retrieved.""" - nbins = 1000 + nbins = 2000 tmax_Gyr = 20.0 minimum_store_Gyr = 1.0 From b92e831d600b71f4c8933816c180232ab679a7af Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 1 Nov 2017 10:28:46 -0400 Subject: [PATCH 02/42] add appropriate __init__ file to michaels_properties and include current property module --- .idea/vcs.xml | 6 ++++++ michaels_properties/__init__.py | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 .idea/vcs.xml create mode 100644 michaels_properties/__init__.py diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/michaels_properties/__init__.py b/michaels_properties/__init__.py new file mode 100644 index 00000000..c8eec057 --- /dev/null +++ b/michaels_properties/__init__.py @@ -0,0 +1,3 @@ +from tangos.properties import * + +from . import sfhistory \ No newline at end of file From 1f17c4db3d85210c160704ff879f4bcef6e9e19d Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 1 Nov 2017 10:35:20 -0400 Subject: [PATCH 03/42] add appropriate environment files to source for bash and cshell --- environment.csh | 5 +++++ environment.sh | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 environment.csh create mode 100644 environment.sh diff --git a/environment.csh b/environment.csh new file mode 100644 index 00000000..c264bd4d --- /dev/null +++ b/environment.csh @@ -0,0 +1,5 @@ +setenv TANGOS_PROPERTY_MODULES mytangosproperty +if [[ -e $DIR/environment_local.sh ]] +then + source $DIR/environment_local.sh +fi diff --git a/environment.sh b/environment.sh new file mode 100644 index 00000000..1b255fcd --- /dev/null +++ b/environment.sh @@ -0,0 +1,5 @@ +export TANGOS_PROPERTY_MODULES=mytangosproperty +if [[ -e $DIR/environment_local.sh ]] +then + source $DIR/environment_local.sh +fi From ea2720172093159866e28ed265cad8afc1fd5949 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 1 Nov 2017 10:38:31 -0400 Subject: [PATCH 04/42] attempt fix to cshell script --- environment.csh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.csh b/environment.csh index c264bd4d..99e075cf 100644 --- a/environment.csh +++ b/environment.csh @@ -1,5 +1,5 @@ setenv TANGOS_PROPERTY_MODULES mytangosproperty -if [[ -e $DIR/environment_local.sh ]] +if [[ -e environment_local.sh ]] then - source $DIR/environment_local.sh + source environment_local.sh fi From cb4daa2b3f387ca2c61c81d31ee05401f80dd596 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 1 Nov 2017 10:42:23 -0400 Subject: [PATCH 05/42] delete environment files --- environment.csh | 5 ----- environment.sh | 5 ----- 2 files changed, 10 deletions(-) delete mode 100644 environment.csh delete mode 100644 environment.sh diff --git a/environment.csh b/environment.csh deleted file mode 100644 index 99e075cf..00000000 --- a/environment.csh +++ /dev/null @@ -1,5 +0,0 @@ -setenv TANGOS_PROPERTY_MODULES mytangosproperty -if [[ -e environment_local.sh ]] -then - source environment_local.sh -fi diff --git a/environment.sh b/environment.sh deleted file mode 100644 index 1b255fcd..00000000 --- a/environment.sh +++ /dev/null @@ -1,5 +0,0 @@ -export TANGOS_PROPERTY_MODULES=mytangosproperty -if [[ -e $DIR/environment_local.sh ]] -then - source $DIR/environment_local.sh -fi From 7590a089d6d4131f0a33cfa1c1ddced9b4ec6b6f Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 5 Nov 2017 12:26:21 -0500 Subject: [PATCH 06/42] properties for Temperature (ew and mw) profiles --- michaels_properties/profiles.py | 90 +++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 michaels_properties/profiles.py diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py new file mode 100644 index 00000000..1fbbb889 --- /dev/null +++ b/michaels_properties/profiles.py @@ -0,0 +1,90 @@ +from tangos.properties import HaloProperties, TimeChunkedProperty +import numpy as np +import pynbody + + +def emissivity(rho, T, mu, tcool): + kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') + mh = pynbody.array.SimArray(1.6726219e-24, 'g') + return 3. / 2. * rho * kb * T / (mu * mh * tcool) + + +def tcool(rho, T, mu): + # taken from https://arxiv.org/abs/astro-ph/9809159 eq 12 + fm = 1.0 # metalicity dependent factor, 1.0 for solar, 0.03 for pristine + mh = pynbody.array.SimArray(1.6726219e-24, 'g') + C1 = 3.88e11 + C2 = 5e7 + return C1 * mu * mh * T ** 0.5 / (rho * (1 + C2 * fm / T)) + + +@pynbody.analysis.profile.Profile.profile_property +def Tew(self): + temp_cut = 1.26e6 + kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') + temp = np.zeros(self.nbins) + rho_c = pynbody.analysis.cosmology.rho_crit(self.sim, z=0) + for i in range(self.nbins): + subs = self.sim[self.binind[i]] + use = np.where((subs.g['temp'] > temp_cut))[0] + mu = 0.58 + tc = tcool(subs.g['rho'][use].in_units('g cm**-3'), subs.g['temp'][use], mu) + em = emissivity(subs.g['rho'][use].in_units('g cm**-3'), subs.g['temp'][use], mu, tc) + temp[i] = np.sum(em * subs.g['temp'][use]) / np.sum(em) + + return kb.in_units('keV K**-1') * temp + + +@pynbody.analysis.profile.Profile.profile_property +def Tmw(self): + temp_cut = 1.26e6 + kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') + temp = np.zeros(self.nbins) + for i in range(self.nbins): + subs = self.sim[self.binind[i]] + use = np.where((subs.g['temp'] > temp_cut))[0] + temp[i] = np.sum(subs.g['mass'][use] * subs.g['temp'][use]) / np.sum(subs.g['mass'][use]) + return kb.in_units('keV K**-1') * temp + + +class TemperatureProfiles(HaloProperties): + @classmethod + def name(self): + return "Tew_profile", "Tmw_profile" + + def plot_x0(cls): + return 0.05 + + @classmethod + def plot_xdelta(cls): + return 0.1 + + @classmethod + def plot_xlabel(cls): + return "R/kpc" + + @classmethod + def plot_ylog(cls): + return False + + @classmethod + def plot_xlog(cls): + return False + + @staticmethod + def plot_ylabel(): + return "T$_{ew}$ keV", "T$_{mw}$ keV" + + def requires_property(self): + return ["SSC", "Rvir"] + + def calculate(self, halo, existing_properties): + halo['pos'] -= existing_properties['SSC'] + halo.wrap() + delta = self.plot_xdelta() + nbins = int(existing_properties['Rvir']/ delta) + maxrad = delta * (nbins + 1) + ps = pynbody.analysis.profile.Profile(halo.s, type='lin', ndim=2, min=0, max=maxrad, nbins=nbins) + Tew = ps['Tew'] + Tmw = ps['Tmw'] + return Tew, Tmw From 8f66aa3570270f902dceaeb65e1dee003fb92f0c Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 5 Nov 2017 12:31:59 -0500 Subject: [PATCH 07/42] add electron density to profiles. Fix issues with temperature --- michaels_properties/profiles.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index 1fbbb889..65e5190d 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -2,17 +2,17 @@ import numpy as np import pynbody +temp_cut = 1.26e6 +kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') +mh = pynbody.array.SimArray(1.6726219e-24, 'g') def emissivity(rho, T, mu, tcool): - kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') - mh = pynbody.array.SimArray(1.6726219e-24, 'g') return 3. / 2. * rho * kb * T / (mu * mh * tcool) def tcool(rho, T, mu): # taken from https://arxiv.org/abs/astro-ph/9809159 eq 12 fm = 1.0 # metalicity dependent factor, 1.0 for solar, 0.03 for pristine - mh = pynbody.array.SimArray(1.6726219e-24, 'g') C1 = 3.88e11 C2 = 5e7 return C1 * mu * mh * T ** 0.5 / (rho * (1 + C2 * fm / T)) @@ -20,8 +20,6 @@ def tcool(rho, T, mu): @pynbody.analysis.profile.Profile.profile_property def Tew(self): - temp_cut = 1.26e6 - kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') temp = np.zeros(self.nbins) rho_c = pynbody.analysis.cosmology.rho_crit(self.sim, z=0) for i in range(self.nbins): @@ -37,8 +35,6 @@ def Tew(self): @pynbody.analysis.profile.Profile.profile_property def Tmw(self): - temp_cut = 1.26e6 - kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') temp = np.zeros(self.nbins) for i in range(self.nbins): subs = self.sim[self.binind[i]] @@ -46,8 +42,17 @@ def Tmw(self): temp[i] = np.sum(subs.g['mass'][use] * subs.g['temp'][use]) / np.sum(subs.g['mass'][use]) return kb.in_units('keV K**-1') * temp +@pynbody.analysis.profile.Profile.profile_property +def rho_e(self): + n_e = np.zeros(self.nbins) + for i in range(self.nbins): + subs = self.sim[self.binind[i]] + use = np.where(subs.g['temp'] > temp_cut)[0] + n_e[i] = np.sum(subs.g['ne'][use] * subs.g['mass'][use].in_units('m_p'))/self._binsize.in_units('cm**'+str(int(self.ndim)))[i] + return n_e + -class TemperatureProfiles(HaloProperties): +class GasProfiles(HaloProperties): @classmethod def name(self): return "Tew_profile", "Tmw_profile" @@ -84,7 +89,8 @@ def calculate(self, halo, existing_properties): delta = self.plot_xdelta() nbins = int(existing_properties['Rvir']/ delta) maxrad = delta * (nbins + 1) - ps = pynbody.analysis.profile.Profile(halo.s, type='lin', ndim=2, min=0, max=maxrad, nbins=nbins) + ps = pynbody.analysis.profile.Profile(halo.g, type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) Tew = ps['Tew'] Tmw = ps['Tmw'] - return Tew, Tmw + rho_e = ps['rho_e'] + return Tew, Tmw, rho_e \ No newline at end of file From 16ca8e00f6cc88fab4eed34901d658d795e3082d Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 5 Nov 2017 13:09:58 -0500 Subject: [PATCH 08/42] add name for density profile --- michaels_properties/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index 65e5190d..061c7a34 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -55,7 +55,7 @@ def rho_e(self): class GasProfiles(HaloProperties): @classmethod def name(self): - return "Tew_profile", "Tmw_profile" + return "Tew_profile", "Tmw_profile", "rho_e_profile" def plot_x0(cls): return 0.05 From 5cb8f18cc638956c6ebd3650dc045801b267ab37 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 5 Nov 2017 13:10:40 -0500 Subject: [PATCH 09/42] add profiles module into init function for my properties --- michaels_properties/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/michaels_properties/__init__.py b/michaels_properties/__init__.py index c8eec057..01e76542 100644 --- a/michaels_properties/__init__.py +++ b/michaels_properties/__init__.py @@ -1,3 +1,3 @@ from tangos.properties import * -from . import sfhistory \ No newline at end of file +from . import sfhistory, profiles \ No newline at end of file From 76ca96f910fbbef10d2ec1c2205eea6797eff0f9 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 6 Nov 2017 08:51:00 -0500 Subject: [PATCH 10/42] get rid of spurious rho_crit calculation to avoid having to profile all particles and not just gas. --- michaels_properties/profiles.py | 1 - 1 file changed, 1 deletion(-) diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index 061c7a34..1b9dd8f1 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -21,7 +21,6 @@ def tcool(rho, T, mu): @pynbody.analysis.profile.Profile.profile_property def Tew(self): temp = np.zeros(self.nbins) - rho_c = pynbody.analysis.cosmology.rho_crit(self.sim, z=0) for i in range(self.nbins): subs = self.sim[self.binind[i]] use = np.where((subs.g['temp'] > temp_cut))[0] From e939fc9d9501efc1c45a6e205c03babf80619a2d Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 6 Nov 2017 09:58:10 -0500 Subject: [PATCH 11/42] include all particles in profile to avoid errors --- michaels_properties/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index 1b9dd8f1..14cd112f 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -88,7 +88,7 @@ def calculate(self, halo, existing_properties): delta = self.plot_xdelta() nbins = int(existing_properties['Rvir']/ delta) maxrad = delta * (nbins + 1) - ps = pynbody.analysis.profile.Profile(halo.g, type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) + ps = pynbody.analysis.profile.Profile(halo, type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) Tew = ps['Tew'] Tmw = ps['Tmw'] rho_e = ps['rho_e'] From 8610e14c3f153db3a0aa4d9bd81075beb9ee53b6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 6 Nov 2017 14:17:22 -0500 Subject: [PATCH 12/42] go back to profiling whole simulation --- michaels_properties/profiles.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index 14cd112f..253e3e68 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -23,7 +23,7 @@ def Tew(self): temp = np.zeros(self.nbins) for i in range(self.nbins): subs = self.sim[self.binind[i]] - use = np.where((subs.g['temp'] > temp_cut))[0] + use = np.where(subs.g['temp'] > temp_cut)[0] mu = 0.58 tc = tcool(subs.g['rho'][use].in_units('g cm**-3'), subs.g['temp'][use], mu) em = emissivity(subs.g['rho'][use].in_units('g cm**-3'), subs.g['temp'][use], mu, tc) @@ -37,7 +37,7 @@ def Tmw(self): temp = np.zeros(self.nbins) for i in range(self.nbins): subs = self.sim[self.binind[i]] - use = np.where((subs.g['temp'] > temp_cut))[0] + use = np.where(subs.g['temp'] > temp_cut)[0] temp[i] = np.sum(subs.g['mass'][use] * subs.g['temp'][use]) / np.sum(subs.g['mass'][use]) return kb.in_units('keV K**-1') * temp @@ -88,7 +88,7 @@ def calculate(self, halo, existing_properties): delta = self.plot_xdelta() nbins = int(existing_properties['Rvir']/ delta) maxrad = delta * (nbins + 1) - ps = pynbody.analysis.profile.Profile(halo, type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) + ps = pynbody.analysis.profile.Profile(halo.g, type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) Tew = ps['Tew'] Tmw = ps['Tmw'] rho_e = ps['rho_e'] From b31ed25a341cd9105241c85a04ae69bd72a7d744 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 12 Nov 2017 11:17:03 -0500 Subject: [PATCH 13/42] addn new property under michaels_properties to calculate gas flows --- michaels_properties/flows.py | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 michaels_properties/flows.py diff --git a/michaels_properties/flows.py b/michaels_properties/flows.py new file mode 100644 index 00000000..a5e5410d --- /dev/null +++ b/michaels_properties/flows.py @@ -0,0 +1,56 @@ +from tangos.properties import HaloProperties, TimeChunkedProperty +from tangos.properties.spherical_region import SphericalRegionHaloProperties +import numpy as np +import pynbody + + +class FlowProfile(SphericalRegionHaloProperties): + #_xmax = 100.0 + _threshold_vel = 20.0 + + def region_specification(self, db_data): + return pynbody.filt.Sphere(db_data['Rvir'], db_data['SSC']) & \ + (pynbody.filt.FamilyFilter(pynbody.family.gas)|pynbody.filt.FamilyFilter(pynbody.family.star)) + + @classmethod + def name(cls): + return "inflow_Mdot", "outflow_Mdot", \ + "inflow_vel", "outflow_vel", \ + "inflow_vel2", "outflow_vel2", \ + "inflow_temp", "outflow_temp", \ +# "inflow_Mdot_dm", "outflow_Mdot_dm", \ +# "inflow_vel_dm", "outflow_vel_dm", \ +# "inflow_vel2_dm", "outflow_vel2_dm" + + + def plot_x0(cls): + return 0.5 + + @classmethod + def plot_xdelta(cls): + return 1.0 + + def profile_calculation(self, f_gas, vr_cut, rvir): + f_gas['Mdot'] = f_gas['mass'] * f_gas['vr'] / (pynbody.units.Unit("kpc")*self.plot_xdelta()) + if vr_cut<0: + f_gas['Mdot']*=(f_gas['vr']vr_cut).view(np.ndarray) + + + + pro = pynbody.analysis.profile.Profile(f_gas, min=0.0,max=rvir, + nbins=int(self.rvir/self.plot_xdelta()), + ndim=3, weight_by='Mdot') + return pro['weight_fn'].in_units("Msol yr^-1"), pro['vr'], pro['vr2'], pro['temp'] + + + def calculate(self, halo, properties): + with pynbody.analysis.halo.center(halo): + halo.gas['vr2'] = halo.gas['vr'] ** 2 + inflow_Mdot, inflow_vel, inflow_vel2, inflow_temp = self.profile_calculation(halo.ancestor.gas, -self._threshold_vel) + outflow_Mdot, outflow_vel, outflow_vel2, outflow_temp = self.profile_calculation(halo.ancestor.gas, self._threshold_vel) + inflow_Mdot_dm, inflow_vel_dm, inflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, -self._threshold_vel) + outflow_Mdot_dm, outflow_vel_dm, outflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, self._threshold_vel) + return inflow_Mdot, outflow_Mdot, -inflow_vel, outflow_vel, inflow_vel2, outflow_vel2, inflow_temp, outflow_temp, inflow_Mdot_dm, outflow_Mdot_dm, -inflow_vel_dm, outflow_vel_dm, inflow_vel2_dm, outflow_vel2_dm From 8c1dbfeb272f1f1d42ac9022357c19aebfa44a1d Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 12 Nov 2017 16:14:45 -0500 Subject: [PATCH 14/42] edit my functions for new naming scheme (SSC -> shink_center; Rvir -> max_radius) and new way of calling halo centering --- michaels_properties/flows.py | 24 ++++++++++++------------ michaels_properties/profiles.py | 14 ++++++++------ michaels_properties/sfhistory.py | 4 ++-- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/michaels_properties/flows.py b/michaels_properties/flows.py index a5e5410d..0329a1c4 100644 --- a/michaels_properties/flows.py +++ b/michaels_properties/flows.py @@ -2,15 +2,15 @@ from tangos.properties.spherical_region import SphericalRegionHaloProperties import numpy as np import pynbody - +from tangos.properties.centring import centred_calculation class FlowProfile(SphericalRegionHaloProperties): #_xmax = 100.0 _threshold_vel = 20.0 - def region_specification(self, db_data): - return pynbody.filt.Sphere(db_data['Rvir'], db_data['SSC']) & \ - (pynbody.filt.FamilyFilter(pynbody.family.gas)|pynbody.filt.FamilyFilter(pynbody.family.star)) + #def region_specification(self, db_data): + # return pynbody.filt.Sphere(db_data['Rvir'], db_data['SSC']) & \ + # (pynbody.filt.FamilyFilter(pynbody.family.gas)|pynbody.filt.FamilyFilter(pynbody.family.star)) @classmethod def name(cls): @@ -41,16 +41,16 @@ def profile_calculation(self, f_gas, vr_cut, rvir): pro = pynbody.analysis.profile.Profile(f_gas, min=0.0,max=rvir, - nbins=int(self.rvir/self.plot_xdelta()), + nbins=int(rvir/self.plot_xdelta()), ndim=3, weight_by='Mdot') return pro['weight_fn'].in_units("Msol yr^-1"), pro['vr'], pro['vr2'], pro['temp'] - + @centred_calculation def calculate(self, halo, properties): - with pynbody.analysis.halo.center(halo): + with pynbody.analysis.vel_center(halo): halo.gas['vr2'] = halo.gas['vr'] ** 2 - inflow_Mdot, inflow_vel, inflow_vel2, inflow_temp = self.profile_calculation(halo.ancestor.gas, -self._threshold_vel) - outflow_Mdot, outflow_vel, outflow_vel2, outflow_temp = self.profile_calculation(halo.ancestor.gas, self._threshold_vel) - inflow_Mdot_dm, inflow_vel_dm, inflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, -self._threshold_vel) - outflow_Mdot_dm, outflow_vel_dm, outflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, self._threshold_vel) - return inflow_Mdot, outflow_Mdot, -inflow_vel, outflow_vel, inflow_vel2, outflow_vel2, inflow_temp, outflow_temp, inflow_Mdot_dm, outflow_Mdot_dm, -inflow_vel_dm, outflow_vel_dm, inflow_vel2_dm, outflow_vel2_dm + inflow_Mdot, inflow_vel, inflow_vel2, inflow_temp = self.profile_calculation(halo.ancestor.gas, -self._threshold_vel, properties['max_radius']) + outflow_Mdot, outflow_vel, outflow_vel2, outflow_temp = self.profile_calculation(halo.ancestor.gas, self._threshold_vel,properties['max_radius']) + #inflow_Mdot_dm, inflow_vel_dm, inflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, -self._threshold_vel,properties['max_radius']) + #outflow_Mdot_dm, outflow_vel_dm, outflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, self._threshold_vel,properties['max_radius']) + return inflow_Mdot, outflow_Mdot, -inflow_vel, outflow_vel, inflow_vel2, outflow_vel2, inflow_temp, outflow_temp diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index 253e3e68..f20adef8 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -1,8 +1,8 @@ from tangos.properties import HaloProperties, TimeChunkedProperty import numpy as np import pynbody +from tangos.properties.centring import centred_calculation -temp_cut = 1.26e6 kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') mh = pynbody.array.SimArray(1.6726219e-24, 'g') @@ -52,6 +52,7 @@ def rho_e(self): class GasProfiles(HaloProperties): + _temp_cut = 1.26e6 @classmethod def name(self): return "Tew_profile", "Tmw_profile", "rho_e_profile" @@ -80,15 +81,16 @@ def plot_ylabel(): return "T$_{ew}$ keV", "T$_{mw}$ keV" def requires_property(self): - return ["SSC", "Rvir"] + return ["shink_center", "max_radius"] + @centred_calculation def calculate(self, halo, existing_properties): - halo['pos'] -= existing_properties['SSC'] - halo.wrap() + #halo['pos'] -= existing_properties['SSC'] + #halo.wrap() delta = self.plot_xdelta() - nbins = int(existing_properties['Rvir']/ delta) + nbins = int(existing_properties['max_radius']/ delta) maxrad = delta * (nbins + 1) - ps = pynbody.analysis.profile.Profile(halo.g, type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) + ps = pynbody.analysis.profile.Profile(halo.g[pynbody.filt.HighPass('temp',self._temp_cut)], type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) Tew = ps['Tew'] Tmw = ps['Tmw'] rho_e = ps['rho_e'] diff --git a/michaels_properties/sfhistory.py b/michaels_properties/sfhistory.py index fa4aa9e7..479592cd 100644 --- a/michaels_properties/sfhistory.py +++ b/michaels_properties/sfhistory.py @@ -13,10 +13,10 @@ def name(self): return "inner_SFR_histogram" def requires_property(self): - return ["SSC", "Rvir"] + return ["shrink_center", "max_radius"] def calculate(self, halo, existing_properties): - filter = pynbody.filt.Sphere(self._maxr_frac*existing_properties['Rvir'],cen=existing_properties['SSC']) + filter = pynbody.filt.Sphere(self._maxr_frac*existing_properties['max_radius'],cen=existing_properties['shink_center']) M,_ = np.histogram(halo.st[filter]['tform'].in_units("Gyr"),weights=halo.st[filter]['massform'].in_units("Msol"),bins=self.nbins,range=(0,self.tmax_Gyr)) t_now = halo.properties['time'].in_units("Gyr") M/=self.delta_t From dc65895d6154abd76ae4ecc69876da8cf291c7c0 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 12 Nov 2017 16:56:44 -0500 Subject: [PATCH 15/42] fix typo in names --- michaels_properties/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index f20adef8..25523df8 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -81,7 +81,7 @@ def plot_ylabel(): return "T$_{ew}$ keV", "T$_{mw}$ keV" def requires_property(self): - return ["shink_center", "max_radius"] + return ["shrink_center", "max_radius"] @centred_calculation def calculate(self, halo, existing_properties): From f71ec5a420e05827a872d363f18a4ceb00812d3a Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 12 Nov 2017 17:03:22 -0500 Subject: [PATCH 16/42] correctly use temp cut where it is defined --- michaels_properties/profiles.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py index 25523df8..dc4d94e8 100644 --- a/michaels_properties/profiles.py +++ b/michaels_properties/profiles.py @@ -23,11 +23,11 @@ def Tew(self): temp = np.zeros(self.nbins) for i in range(self.nbins): subs = self.sim[self.binind[i]] - use = np.where(subs.g['temp'] > temp_cut)[0] + #use = np.where(subs.g['temp'] > temp_cut)[0] mu = 0.58 - tc = tcool(subs.g['rho'][use].in_units('g cm**-3'), subs.g['temp'][use], mu) - em = emissivity(subs.g['rho'][use].in_units('g cm**-3'), subs.g['temp'][use], mu, tc) - temp[i] = np.sum(em * subs.g['temp'][use]) / np.sum(em) + tc = tcool(subs.g['rho'].in_units('g cm**-3'), subs.g['temp'], mu) + em = emissivity(subs.g['rho'].in_units('g cm**-3'), subs.g['temp'], mu, tc) + temp[i] = np.sum(em * subs.g['temp']) / np.sum(em) return kb.in_units('keV K**-1') * temp @@ -37,8 +37,8 @@ def Tmw(self): temp = np.zeros(self.nbins) for i in range(self.nbins): subs = self.sim[self.binind[i]] - use = np.where(subs.g['temp'] > temp_cut)[0] - temp[i] = np.sum(subs.g['mass'][use] * subs.g['temp'][use]) / np.sum(subs.g['mass'][use]) + #use = np.where(subs.g['temp'] > temp_cut)[0] + temp[i] = np.sum(subs.g['mass'] * subs.g['temp']) / np.sum(subs.g['mass']) return kb.in_units('keV K**-1') * temp @pynbody.analysis.profile.Profile.profile_property @@ -46,8 +46,8 @@ def rho_e(self): n_e = np.zeros(self.nbins) for i in range(self.nbins): subs = self.sim[self.binind[i]] - use = np.where(subs.g['temp'] > temp_cut)[0] - n_e[i] = np.sum(subs.g['ne'][use] * subs.g['mass'][use].in_units('m_p'))/self._binsize.in_units('cm**'+str(int(self.ndim)))[i] + #use = np.where(subs.g['temp'] > temp_cut)[0] + n_e[i] = np.sum(subs.g['ne'] * subs.g['mass'].in_units('m_p'))/self._binsize.in_units('cm**'+str(int(self.ndim)))[i] return n_e From 8ff294aef3ba6519a03ed646c7e1d6fb32f489c6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Nov 2017 09:48:17 -0500 Subject: [PATCH 17/42] add time output to mergers and ignore DM for inflow/outflow in order to conserve memory --- michaels_properties/flows.py | 14 +++++++------- tangos/examples/mergers.py | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/michaels_properties/flows.py b/michaels_properties/flows.py index 0329a1c4..b74c98b8 100644 --- a/michaels_properties/flows.py +++ b/michaels_properties/flows.py @@ -8,16 +8,16 @@ class FlowProfile(SphericalRegionHaloProperties): #_xmax = 100.0 _threshold_vel = 20.0 - #def region_specification(self, db_data): - # return pynbody.filt.Sphere(db_data['Rvir'], db_data['SSC']) & \ - # (pynbody.filt.FamilyFilter(pynbody.family.gas)|pynbody.filt.FamilyFilter(pynbody.family.star)) + def region_specification(self, db_data): + return pynbody.filt.Sphere(db_data['max_radius'], db_data['shrink_center']) & \ + (pynbody.filt.FamilyFilter(pynbody.family.gas)|pynbody.filt.FamilyFilter(pynbody.family.star)) @classmethod def name(cls): - return "inflow_Mdot", "outflow_Mdot", \ - "inflow_vel", "outflow_vel", \ - "inflow_vel2", "outflow_vel2", \ - "inflow_temp", "outflow_temp", \ + return "gas_inflow_Mdot", "gas_outflow_Mdot", \ + "gas_inflow_vel", "gas_outflow_vel", \ + "gas_inflow_vel2", "gas_outflow_vel2", \ + "gas_inflow_temp", "gas_outflow_temp", \ # "inflow_Mdot_dm", "outflow_Mdot_dm", \ # "inflow_vel_dm", "outflow_vel_dm", \ # "inflow_vel2_dm", "outflow_vel2_dm" diff --git a/tangos/examples/mergers.py b/tangos/examples/mergers.py index 2ae1be98..f9c81271 100644 --- a/tangos/examples/mergers.py +++ b/tangos/examples/mergers.py @@ -15,6 +15,7 @@ def get_mergers_of_major_progenitor(input_halo): redshift = [] ratio = [] halo = [] + time = [] while input_halo is not None: mergers = db.relation_finding.MultiHopMostRecentMergerStrategy(input_halo).all() if len(mergers)>0 : @@ -22,11 +23,12 @@ def get_mergers_of_major_progenitor(input_halo): redshift.append(mergers[0].timestep.next.redshift) halo.append((mergers[0], m)) ratio.append(float(mergers[0].NDM)/m.NDM) + time.append(mergers[0].timestep.next.time_gyr) input_halo = mergers[0] else: input_halo = None - return np.array(redshift), np.array(ratio), halo + return np.array(redshift), np.array(ratio), time, halo def most_major_mergers_since(ts, Mvir_min=0.8e12, Mvir_max=2e12, z_merger_max=4.0, no_merger_value=None): """Given a timestep, calculate the most major merger ratio since a given redshift for all halos in a specified mass range From 3c74ee5d832b8166f90851b179a9100388eaa828 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 4 Apr 2018 11:00:15 -0400 Subject: [PATCH 18/42] remove michaels_properties --- michaels_properties/__init__.py | 3 - michaels_properties/flows.py | 56 ------------------ michaels_properties/profiles.py | 97 -------------------------------- michaels_properties/sfhistory.py | 30 ---------- 4 files changed, 186 deletions(-) delete mode 100644 michaels_properties/__init__.py delete mode 100644 michaels_properties/flows.py delete mode 100644 michaels_properties/profiles.py delete mode 100644 michaels_properties/sfhistory.py diff --git a/michaels_properties/__init__.py b/michaels_properties/__init__.py deleted file mode 100644 index 01e76542..00000000 --- a/michaels_properties/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from tangos.properties import * - -from . import sfhistory, profiles \ No newline at end of file diff --git a/michaels_properties/flows.py b/michaels_properties/flows.py deleted file mode 100644 index b74c98b8..00000000 --- a/michaels_properties/flows.py +++ /dev/null @@ -1,56 +0,0 @@ -from tangos.properties import HaloProperties, TimeChunkedProperty -from tangos.properties.spherical_region import SphericalRegionHaloProperties -import numpy as np -import pynbody -from tangos.properties.centring import centred_calculation - -class FlowProfile(SphericalRegionHaloProperties): - #_xmax = 100.0 - _threshold_vel = 20.0 - - def region_specification(self, db_data): - return pynbody.filt.Sphere(db_data['max_radius'], db_data['shrink_center']) & \ - (pynbody.filt.FamilyFilter(pynbody.family.gas)|pynbody.filt.FamilyFilter(pynbody.family.star)) - - @classmethod - def name(cls): - return "gas_inflow_Mdot", "gas_outflow_Mdot", \ - "gas_inflow_vel", "gas_outflow_vel", \ - "gas_inflow_vel2", "gas_outflow_vel2", \ - "gas_inflow_temp", "gas_outflow_temp", \ -# "inflow_Mdot_dm", "outflow_Mdot_dm", \ -# "inflow_vel_dm", "outflow_vel_dm", \ -# "inflow_vel2_dm", "outflow_vel2_dm" - - - def plot_x0(cls): - return 0.5 - - @classmethod - def plot_xdelta(cls): - return 1.0 - - def profile_calculation(self, f_gas, vr_cut, rvir): - f_gas['Mdot'] = f_gas['mass'] * f_gas['vr'] / (pynbody.units.Unit("kpc")*self.plot_xdelta()) - if vr_cut<0: - f_gas['Mdot']*=(f_gas['vr']vr_cut).view(np.ndarray) - - - - pro = pynbody.analysis.profile.Profile(f_gas, min=0.0,max=rvir, - nbins=int(rvir/self.plot_xdelta()), - ndim=3, weight_by='Mdot') - return pro['weight_fn'].in_units("Msol yr^-1"), pro['vr'], pro['vr2'], pro['temp'] - - @centred_calculation - def calculate(self, halo, properties): - with pynbody.analysis.vel_center(halo): - halo.gas['vr2'] = halo.gas['vr'] ** 2 - inflow_Mdot, inflow_vel, inflow_vel2, inflow_temp = self.profile_calculation(halo.ancestor.gas, -self._threshold_vel, properties['max_radius']) - outflow_Mdot, outflow_vel, outflow_vel2, outflow_temp = self.profile_calculation(halo.ancestor.gas, self._threshold_vel,properties['max_radius']) - #inflow_Mdot_dm, inflow_vel_dm, inflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, -self._threshold_vel,properties['max_radius']) - #outflow_Mdot_dm, outflow_vel_dm, outflow_vel2_dm, _ = self.profile_calculation(halo.ancestor.dm, self._threshold_vel,properties['max_radius']) - return inflow_Mdot, outflow_Mdot, -inflow_vel, outflow_vel, inflow_vel2, outflow_vel2, inflow_temp, outflow_temp diff --git a/michaels_properties/profiles.py b/michaels_properties/profiles.py deleted file mode 100644 index dc4d94e8..00000000 --- a/michaels_properties/profiles.py +++ /dev/null @@ -1,97 +0,0 @@ -from tangos.properties import HaloProperties, TimeChunkedProperty -import numpy as np -import pynbody -from tangos.properties.centring import centred_calculation - -kb = pynbody.array.SimArray(1.380658e-16, 'erg K**-1') -mh = pynbody.array.SimArray(1.6726219e-24, 'g') - -def emissivity(rho, T, mu, tcool): - return 3. / 2. * rho * kb * T / (mu * mh * tcool) - - -def tcool(rho, T, mu): - # taken from https://arxiv.org/abs/astro-ph/9809159 eq 12 - fm = 1.0 # metalicity dependent factor, 1.0 for solar, 0.03 for pristine - C1 = 3.88e11 - C2 = 5e7 - return C1 * mu * mh * T ** 0.5 / (rho * (1 + C2 * fm / T)) - - -@pynbody.analysis.profile.Profile.profile_property -def Tew(self): - temp = np.zeros(self.nbins) - for i in range(self.nbins): - subs = self.sim[self.binind[i]] - #use = np.where(subs.g['temp'] > temp_cut)[0] - mu = 0.58 - tc = tcool(subs.g['rho'].in_units('g cm**-3'), subs.g['temp'], mu) - em = emissivity(subs.g['rho'].in_units('g cm**-3'), subs.g['temp'], mu, tc) - temp[i] = np.sum(em * subs.g['temp']) / np.sum(em) - - return kb.in_units('keV K**-1') * temp - - -@pynbody.analysis.profile.Profile.profile_property -def Tmw(self): - temp = np.zeros(self.nbins) - for i in range(self.nbins): - subs = self.sim[self.binind[i]] - #use = np.where(subs.g['temp'] > temp_cut)[0] - temp[i] = np.sum(subs.g['mass'] * subs.g['temp']) / np.sum(subs.g['mass']) - return kb.in_units('keV K**-1') * temp - -@pynbody.analysis.profile.Profile.profile_property -def rho_e(self): - n_e = np.zeros(self.nbins) - for i in range(self.nbins): - subs = self.sim[self.binind[i]] - #use = np.where(subs.g['temp'] > temp_cut)[0] - n_e[i] = np.sum(subs.g['ne'] * subs.g['mass'].in_units('m_p'))/self._binsize.in_units('cm**'+str(int(self.ndim)))[i] - return n_e - - -class GasProfiles(HaloProperties): - _temp_cut = 1.26e6 - @classmethod - def name(self): - return "Tew_profile", "Tmw_profile", "rho_e_profile" - - def plot_x0(cls): - return 0.05 - - @classmethod - def plot_xdelta(cls): - return 0.1 - - @classmethod - def plot_xlabel(cls): - return "R/kpc" - - @classmethod - def plot_ylog(cls): - return False - - @classmethod - def plot_xlog(cls): - return False - - @staticmethod - def plot_ylabel(): - return "T$_{ew}$ keV", "T$_{mw}$ keV" - - def requires_property(self): - return ["shrink_center", "max_radius"] - - @centred_calculation - def calculate(self, halo, existing_properties): - #halo['pos'] -= existing_properties['SSC'] - #halo.wrap() - delta = self.plot_xdelta() - nbins = int(existing_properties['max_radius']/ delta) - maxrad = delta * (nbins + 1) - ps = pynbody.analysis.profile.Profile(halo.g[pynbody.filt.HighPass('temp',self._temp_cut)], type='lin', ndim=3, min=0, max=maxrad, nbins=nbins) - Tew = ps['Tew'] - Tmw = ps['Tmw'] - rho_e = ps['rho_e'] - return Tew, Tmw, rho_e \ No newline at end of file diff --git a/michaels_properties/sfhistory.py b/michaels_properties/sfhistory.py deleted file mode 100644 index 479592cd..00000000 --- a/michaels_properties/sfhistory.py +++ /dev/null @@ -1,30 +0,0 @@ -#from __future__ import absolute_import - -from tangos.properties import HaloProperties, TimeChunkedProperty -import numpy as np -import pynbody - -class InnerStarFormHistogram(TimeChunkedProperty): - - _maxr_frac = 0.1 - - @classmethod - def name(self): - return "inner_SFR_histogram" - - def requires_property(self): - return ["shrink_center", "max_radius"] - - def calculate(self, halo, existing_properties): - filter = pynbody.filt.Sphere(self._maxr_frac*existing_properties['max_radius'],cen=existing_properties['shink_center']) - M,_ = np.histogram(halo.st[filter]['tform'].in_units("Gyr"),weights=halo.st[filter]['massform'].in_units("Msol"),bins=self.nbins,range=(0,self.tmax_Gyr)) - t_now = halo.properties['time'].in_units("Gyr") - M/=self.delta_t - M = M[self.store_slice(t_now)] - - return M - - @classmethod - def reassemble(cls, *options): - reassembled = TimeChunkedProperty.reassemble(*options) - return reassembled/1e9 # Msol per Gyr -> Msol per yr From e1ecc2846255f47ed088569d5381f00150dadda7 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Wed, 4 Apr 2018 13:00:53 -0400 Subject: [PATCH 19/42] add to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7e99e367..08667158 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*.pyc \ No newline at end of file +*.pyc +.idea/vcs.xml From c4eb1eceead1e40ad99cbf6ca77ae206ba9b0261 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 4 Apr 2018 13:57:46 -0400 Subject: [PATCH 20/42] add the ability to reassemble live calculated TimeChunked quantities with optional inputs. Small edit to config file to ensure importing of property modules works ok. Involves changes to LiveProperty class, reassemble functions. --- tangos/config.py | 2 +- tangos/core/halo_data/property.py | 2 +- tangos/live_calculation/__init__.py | 29 ++++++++++++++++++- .../builtin_functions/reassembly.py | 17 ++++++----- tangos/properties/__init__.py | 21 ++++++++------ 5 files changed, 52 insertions(+), 19 deletions(-) diff --git a/tangos/config.py b/tangos/config.py index f0567324..9256e843 100644 --- a/tangos/config.py +++ b/tangos/config.py @@ -27,7 +27,7 @@ # names of property modules to import; default is for backwards compatibility on systems with N-Body-Shop extensions -property_modules = os.environ.get("TANGOS_PROPERTY_MODULES","tangos_nbodyshop_properties") +property_modules = os.environ.get("TANGOS_PROPERTY_MODULES") property_modules = property_modules.split(",") property_modules = map(str.strip, property_modules) diff --git a/tangos/core/halo_data/property.py b/tangos/core/halo_data/property.py index 54d51314..73e1380e 100644 --- a/tangos/core/halo_data/property.py +++ b/tangos/core/halo_data/property.py @@ -73,7 +73,7 @@ def get_data_with_reassembly_options(self, *options): cls = None if hasattr(cls, 'reassemble'): - return cls.reassemble(self, *options) + return cls.reassemble(self, self.halo, *options) else: return self.data_raw diff --git a/tangos/live_calculation/__init__.py b/tangos/live_calculation/__init__.py index 3ea7e36b..39b65890 100644 --- a/tangos/live_calculation/__init__.py +++ b/tangos/live_calculation/__init__.py @@ -358,6 +358,8 @@ def __init__(self, *tokens): super(LiveProperty, self).__init__() self._name = str(tokens[0]) self._inputs = list(tokens[1:]) + self._evaluation_pattern = '_evaluate_function' + self._evaluation_options= [] def __str__(self): return self._name + "(" + (",".join(str(x) for x in self._inputs)) + ")" @@ -384,6 +386,16 @@ def _calculation_retrieves(self): result = result.union(providing_instance.requires_property()) return result + def set_evaluation_pattern(self, eval_name): + self._evaluation_pattern = eval_name + + def set_evaluation_pattern_with_options(self,eval_name,*options): + self._evaluation_pattern = eval_name + self._evaluation_options = options + + def _evaluate(self, halos, input_descriptions, input_values): + return getattr(self, self._evaluation_pattern)(halos, input_descriptions, input_values, *self._evaluation_options) + def values_and_description(self, halos): input_values = [] input_descriptions = [] @@ -392,7 +404,7 @@ def values_and_description(self, halos): input_values.append(iv) input_descriptions.append(id) - calculator, results = self._evaluate_function(halos, input_descriptions, input_values) + calculator, results = self._evaluate(halos, input_descriptions, input_values) return results, calculator @@ -415,6 +427,21 @@ def _evaluate_function(self, halos, input_descriptions, input_values): results.append(None) return calculator, self._as_1xn_array(results) + def _evaluate_function_with_reassemble(self, halos, input_descriptions, input_values, *options): + from .. import properties + sim = consistent_collection.consistent_simulation_from_halos(halos) + results = [] + calculator = properties.providing_class(self.name())(sim, *input_descriptions) + for inputs in zip(halos, *input_values): + if self._has_required_properties(inputs[0]) and all([x is not None for x in inputs]): + if hasattr(calculator,'reassemble'): + results.append(calculator.reassemble(self, inputs[0], *options)) + else: + results.append(calculator.live_calculate_named(self.name(), *inputs)) + else: + results.append(None) + return calculator, self._as_1xn_array(results) + @classmethod def _as_1xn_array(cls, results): results_array = np.empty((1, len(results)), dtype=object) diff --git a/tangos/live_calculation/builtin_functions/reassembly.py b/tangos/live_calculation/builtin_functions/reassembly.py index 2387fae7..f6fe31c6 100644 --- a/tangos/live_calculation/builtin_functions/reassembly.py +++ b/tangos/live_calculation/builtin_functions/reassembly.py @@ -1,23 +1,24 @@ from __future__ import absolute_import from tangos.core import extraction_patterns from . import BuiltinFunction -from .. import StoredProperty, FixedInput +from .. import StoredProperty, FixedInput, LiveProperty @BuiltinFunction.register def raw(halos, values): return values -raw.set_input_options(0, assert_class=StoredProperty) @raw.set_initialisation def raw_initialisation(input): - input.set_extraction_pattern(extraction_patterns.halo_property_raw_value_getter) + if isinstance(input, LiveProperty): + input.set_evaluation_pattern('_evaluate_function') + else: + input.set_extraction_pattern(extraction_patterns.halo_property_raw_value_getter) @BuiltinFunction.register def reassemble(halos, values, *options): return values -reassemble.set_input_options(0, assert_class=StoredProperty) @reassemble.set_initialisation def reassemble_initialisation(input, *options): @@ -27,6 +28,8 @@ def reassemble_initialisation(input, *options): options_values.append(option.proxy_value()) else: raise TypeError("Options to 'reassemble' must be fixed numbers or strings") - - input.set_extraction_pattern( - extraction_patterns.HaloPropertyValueWithReassemblyOptionsGetter(*options_values)) + if isinstance(input,LiveProperty): + input.set_evaluation_pattern('_evaluate_function_with_reassemble') + else: + input.set_extraction_pattern( + extraction_patterns.HaloPropertyValueWithReassemblyOptionsGetter(*options_values)) diff --git a/tangos/properties/__init__.py b/tangos/properties/__init__.py index 6cbf375c..fe9c6556 100644 --- a/tangos/properties/__init__.py +++ b/tangos/properties/__init__.py @@ -5,6 +5,7 @@ from six.moves import zip import importlib import warnings +from ..live_calculation import LiveProperty import functools from .. import input_handlers @@ -203,7 +204,7 @@ class TimeChunkedProperty(PropertyCalculation): """TimeChunkedProperty implements a special type of halo property where chunks of a histogram are stored at each time step, then appropriately reassembled when the histogram is retrieved.""" - nbins = 1000 + nbins = 2000 tmax_Gyr = 20.0 minimum_store_Gyr = 1.0 @@ -226,7 +227,7 @@ def store_slice(self, time): return slice(self.bin_index(time-self.minimum_store_Gyr), self.bin_index(time)) @classmethod - def reassemble(cls, property, reassembly_type='major'): + def reassemble(cls, property, halo, reassembly_type='major'): """Reassemble a histogram by suitable treatment of the merger tree leading up to the current halo. This function is normally called by the framework (see Halo.get_data_with_reassembly_options) and you would @@ -247,14 +248,14 @@ def reassemble(cls, property, reassembly_type='major'): from tangos import relation_finding as rfs if reassembly_type=='major': - return cls._reassemble_using_finding_strategy(property, strategy = rfs.MultiHopMajorProgenitorsStrategy) + return cls._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopMajorProgenitorsStrategy) elif reassembly_type=='major_across_simulations': - return cls._reassemble_using_finding_strategy(property, strategy = rfs.MultiHopMajorProgenitorsStrategy, + return cls._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopMajorProgenitorsStrategy, strategy_kwargs = {'target': None}) elif reassembly_type=='sum': - return cls._reassemble_using_finding_strategy(property, strategy = rfs.MultiHopAllProgenitorsStrategy) + return cls._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopAllProgenitorsStrategy) elif reassembly_type=='place': - return cls._place_data(property.halo.timestep.time_gyr, property.data_raw) + return cls._place_data(property.halo.timestep.time_gyr, halo, property.data_raw) elif reassembly_type=='raw': return property.data_raw else: @@ -269,9 +270,11 @@ def _place_data(cls, time, raw_data): return final @classmethod - def _reassemble_using_finding_strategy(cls, property, strategy, strategy_kwargs={}): - name = property.name.text - halo = property.halo + def _reassemble_using_finding_strategy(cls, property, halo, strategy, strategy_kwargs={}): + if not isinstance(property, LiveProperty): + name = property.name.text + else: + name = property.__str__() t, stack = halo.calculate_for_descendants("t()", "raw(" + name + ")", strategy=strategy, strategy_kwargs=strategy_kwargs) final = np.zeros(cls.bin_index(t[0])) previous_time = -1 From 3ed09bcff7de920911556c118ee381f01e9bd063 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Apr 2018 09:44:04 -0400 Subject: [PATCH 21/42] fix error in previous commit. Added unnecessary argujment to _place_data in reassemble. Fix test_live_calculation to reflect changes to reassemble function (add halo argument) --- tangos/properties/__init__.py | 2 +- tests/test_live_calculation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tangos/properties/__init__.py b/tangos/properties/__init__.py index fe9c6556..52c5a87f 100644 --- a/tangos/properties/__init__.py +++ b/tangos/properties/__init__.py @@ -255,7 +255,7 @@ def reassemble(cls, property, halo, reassembly_type='major'): elif reassembly_type=='sum': return cls._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopAllProgenitorsStrategy) elif reassembly_type=='place': - return cls._place_data(property.halo.timestep.time_gyr, halo, property.data_raw) + return cls._place_data(property.halo.timestep.time_gyr, property.data_raw) elif reassembly_type=='raw': return property.data_raw else: diff --git a/tests/test_live_calculation.py b/tests/test_live_calculation.py index b56ce040..168b76d7 100644 --- a/tests/test_live_calculation.py +++ b/tests/test_live_calculation.py @@ -75,7 +75,7 @@ class DummyPropertyWithReassemblyOptions(properties.PropertyCalculation): names = "dummy_property_with_reassembly" @classmethod - def reassemble(cls, property, test_option=25): + def reassemble(cls, property, halo, test_option=25): if test_option=='actual_data': return property.data_raw else: From 8f3feaafcdc7dd6cfb8dc1030599f25e31ed31ff Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Apr 2018 09:48:28 -0400 Subject: [PATCH 22/42] revert back to old default nbins=1000 from 2000, which was needed to test on specific data set. --- tangos/properties/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tangos/properties/__init__.py b/tangos/properties/__init__.py index 52c5a87f..6337af8a 100644 --- a/tangos/properties/__init__.py +++ b/tangos/properties/__init__.py @@ -204,7 +204,7 @@ class TimeChunkedProperty(PropertyCalculation): """TimeChunkedProperty implements a special type of halo property where chunks of a histogram are stored at each time step, then appropriately reassembled when the histogram is retrieved.""" - nbins = 2000 + nbins = 1000 tmax_Gyr = 20.0 minimum_store_Gyr = 1.0 From 291685442ee6aaec0d76a603171a514d541f7404 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Apr 2018 10:24:15 -0400 Subject: [PATCH 23/42] attempt to handle when there are no user defined property modules --- tangos/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tangos/config.py b/tangos/config.py index 9256e843..31b744a9 100644 --- a/tangos/config.py +++ b/tangos/config.py @@ -28,8 +28,11 @@ # names of property modules to import; default is for backwards compatibility on systems with N-Body-Shop extensions property_modules = os.environ.get("TANGOS_PROPERTY_MODULES") -property_modules = property_modules.split(",") -property_modules = map(str.strip, property_modules) +if property_modules is None: + property_modules = [] +else: + property_modules = property_modules.split(",") + property_modules = map(str.strip, property_modules) file_ignore_pattern = [] From ecf504b013190b386f4f521057356cbfac576bca Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 6 May 2018 16:43:21 -0400 Subject: [PATCH 24/42] fix evaluation_pattern for live calculations so that reassemble options are property passed. Fix "raw" and "place" options to properly handle live properties. Make text class for reassembling a live property in tests. --- .../builtin_functions/reassembly.py | 2 +- tangos/properties/__init__.py | 11 +++++++++-- tests/test_live_calculation.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tangos/live_calculation/builtin_functions/reassembly.py b/tangos/live_calculation/builtin_functions/reassembly.py index f6fe31c6..d652bab8 100644 --- a/tangos/live_calculation/builtin_functions/reassembly.py +++ b/tangos/live_calculation/builtin_functions/reassembly.py @@ -29,7 +29,7 @@ def reassemble_initialisation(input, *options): else: raise TypeError("Options to 'reassemble' must be fixed numbers or strings") if isinstance(input,LiveProperty): - input.set_evaluation_pattern('_evaluate_function_with_reassemble') + input.set_evaluation_pattern_with_options('_evaluate_function_with_reassemble', *options_values) else: input.set_extraction_pattern( extraction_patterns.HaloPropertyValueWithReassemblyOptionsGetter(*options_values)) diff --git a/tangos/properties/__init__.py b/tangos/properties/__init__.py index 6337af8a..00f71fe6 100644 --- a/tangos/properties/__init__.py +++ b/tangos/properties/__init__.py @@ -255,9 +255,16 @@ def reassemble(cls, property, halo, reassembly_type='major'): elif reassembly_type=='sum': return cls._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopAllProgenitorsStrategy) elif reassembly_type=='place': - return cls._place_data(property.halo.timestep.time_gyr, property.data_raw) + if not isinstance(property, LiveProperty): + place_data = property.data_raw + else: + place_data = halo.calculate(property.__str__()) + return cls._place_data(property.halo.timestep.time_gyr, place_data) elif reassembly_type=='raw': - return property.data_raw + if not isinstance(property, LiveProperty): + return property.data_raw + else: + return halo.calculate(property.__str__()) else: raise ValueError("Unknown reassembly type") diff --git a/tests/test_live_calculation.py b/tests/test_live_calculation.py index 168b76d7..89e3601a 100644 --- a/tests/test_live_calculation.py +++ b/tests/test_live_calculation.py @@ -81,6 +81,19 @@ def reassemble(cls, property, halo, test_option=25): else: return test_option +class DummyLivePropertyWithReassembly(properties.LivePropertyCalculation): + names = "dummy_live_property_with_reassembly" + + def live_calculate(self, db_halo_entry, factor=2): + return db_halo_entry.calculate('raw(dummy_property_with_reassembly)') * factor + + @classmethod + def reassemble(cls, property, halo, test_option=25): + if test_option == 'actual_data': + return halo.calculate(property.__str__()) + else: + return halo.calculate('raw(dummy_property_with_reassembly)') * test_option + class LivePropertyRequiringRedirectedProperty(properties.LivePropertyCalculation): names = "first_BHs_BH_mass" From 07168f6cb67b193b035db84529b94a0c57602c8d Mon Sep 17 00:00:00 2001 From: mtremmel Date: Mon, 7 May 2018 15:09:54 -0400 Subject: [PATCH 25/42] make it so that reassembly on live calculation response the same way as normal calculations (default call is to reassemble, responds the same to raw() as reassemble(property,'raw')). Ensure that builtin functions set evaluation pattern correctly now that there are multiple options and the default would be inapplicable. --- tangos/live_calculation/__init__.py | 3 ++- tangos/properties/__init__.py | 2 +- tests/test_live_calculation.py | 27 ++++++++++++++++++++++++--- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tangos/live_calculation/__init__.py b/tangos/live_calculation/__init__.py index 39b65890..9cf2ab16 100644 --- a/tangos/live_calculation/__init__.py +++ b/tangos/live_calculation/__init__.py @@ -358,7 +358,7 @@ def __init__(self, *tokens): super(LiveProperty, self).__init__() self._name = str(tokens[0]) self._inputs = list(tokens[1:]) - self._evaluation_pattern = '_evaluate_function' + self._evaluation_pattern = '_evaluate_function_with_reassemble' self._evaluation_options= [] def __str__(self): @@ -505,6 +505,7 @@ def __init__(self, *tokens): super(BuiltinFunction, self).__init__(*tokens) self._func = self.__registered_functions[self._name]['function'] self._info = self.__registered_functions[self._name] + self.set_evaluation_pattern('_evaluate_function') for i in range(len(self._inputs)): assert_class = self._get_input_option(i, 'assert_class') if assert_class is not None and not isinstance(self._inputs[i], assert_class): diff --git a/tangos/properties/__init__.py b/tangos/properties/__init__.py index 00f71fe6..d72309b1 100644 --- a/tangos/properties/__init__.py +++ b/tangos/properties/__init__.py @@ -264,7 +264,7 @@ def reassemble(cls, property, halo, reassembly_type='major'): if not isinstance(property, LiveProperty): return property.data_raw else: - return halo.calculate(property.__str__()) + return halo.calculate('raw('+property.__str__()+')') else: raise ValueError("Unknown reassembly type") diff --git a/tests/test_live_calculation.py b/tests/test_live_calculation.py index 89e3601a..7643653c 100644 --- a/tests/test_live_calculation.py +++ b/tests/test_live_calculation.py @@ -88,11 +88,11 @@ def live_calculate(self, db_halo_entry, factor=2): return db_halo_entry.calculate('raw(dummy_property_with_reassembly)') * factor @classmethod - def reassemble(cls, property, halo, test_option=25): + def reassemble(cls, property, halo, test_option=10): if test_option == 'actual_data': - return halo.calculate(property.__str__()) + return halo.calculate('raw('+property.__str__()+')') else: - return halo.calculate('raw(dummy_property_with_reassembly)') * test_option + return halo.calculate('raw('+property.__str__()+')') * test_option class LivePropertyRequiringRedirectedProperty(properties.LivePropertyCalculation): names = "first_BHs_BH_mass" @@ -226,6 +226,27 @@ def test_reassembly(): # our dummy reassembly function can return the actual value if suitably requested assert h.calculate('reassemble(dummy_property_with_reassembly, "actual_data")') == 101 +def test_live_reassembly(): + h = tangos.get_halo("sim/ts1/1") + h['dummy_property_with_reassembly'] = 101 + + # default reassembly takes in the dummy property and multiplies it first by 2, then by 10 + assert h.calculate('dummy_live_property_with_reassembly()') == 2020 + + # default live_calculation without reassemble takes dummy property and multiplies by 2 + assert h.calculate('raw(dummy_live_property_with_reassembly())') == 202 + + # ability to give input into live calculation + assert h.calculate('raw(dummy_live_property_with_reassembly(4))') == 404 + + # pass option to reassembly and change how the dummpy property is manipulated + assert h.calculate('reassemble(dummy_live_property_with_reassembly(),3)') == 606 + + # reassemble on live calculation with new variable + assert h.calculate('reassemble(dummy_live_property_with_reassembly(3))') == 3030 + + # will return the original value of the live calculation without reassembly if requested correctly + assert h.calculate('reassemble(dummy_live_property_with_reassembly(3),"actual_data")') == 303 def test_liveproperty_requiring_redirection(): h = tangos.get_halo("sim/ts1/1") From 9d93b3cc3f7cef3acbe5989f563c7486b55f77fa Mon Sep 17 00:00:00 2001 From: mtremmel Date: Sun, 13 May 2018 18:34:37 -0400 Subject: [PATCH 26/42] begin new Live Reassemble documentation, add SpecSFR_histogram as a property mainly to act as a nice example for a new user. --- docs/histogram_properties.md | 12 ++++++++++++ tangos/properties/pynbody/SF.py | 28 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/histogram_properties.md b/docs/histogram_properties.md index 4217676c..892cea5e 100644 --- a/docs/histogram_properties.md +++ b/docs/histogram_properties.md @@ -108,6 +108,18 @@ Note that it is a _very_ bad idea to change this after you have already written for a simulation. You will almost certainly end up getting inconsistent results. Always set `histogram_delta_t_Gyr` before your first `tangos write`. +Reassembling live calculations +------------------------------ + +It is possible to define a [live calculation](live_calculation.md) that manipulates an already stored histogram +property (like `SFR_histogram`) and returns a new histogram. Because such a calculation only requires data already stored +in the database, it can be performed on the fly and reassembled accordingly in the same way as the original stored +histogram data. + +The property `SpecSFR_histogram` calculates the specific star formation rate history of a halo given `SFR_histogram`. + + + Exercising caution ------------------- diff --git a/tangos/properties/pynbody/SF.py b/tangos/properties/pynbody/SF.py index 1005bdbf..02682b21 100644 --- a/tangos/properties/pynbody/SF.py +++ b/tangos/properties/pynbody/SF.py @@ -1,4 +1,4 @@ -from .. import TimeChunkedProperty +from .. import TimeChunkedProperty, LiveHaloProperties from . import pynbody_handler_module import numpy as np @@ -36,3 +36,29 @@ def calculate(self, halo, existing_properties): def reassemble(self, *options): reassembled = super(StarFormHistogram, self).reassemble(*options) return reassembled/1e9 # Msol per Gyr -> Msol per yr + +class SpecStarFormationHistogram(TimeChunkedProperty,LiveHaloProperties): + names = "SpecSFR_histogram" + + def __init__(self, simulation): + super(SpecStarFormationHistogram, self).__init__(simulation) + + def plot_xlabel(self): + return "t/Gyr" + + def plot_ylabel(self): + return r"$SFR/M_{\odot} yr^{-1}$" + + def requires_property(self): + return ['Mstar','SFR_histogram'] + + def live_calculate(self, halo, *args): + sfr = halo.calculate('raw(SFR_histogram)') + Mstar_f = halo['Mstar'] + Mstar_i = Mstar_f - np.sum(sfr*self.pixel_delta_t_Gyr) + Mstar_t = Mstar_i + np.cumsum(sfr*self.pixel_delta_t_Gyr) + return sfr/Mstar_t + + def reassemble(self, *options): + reassembled = super(SpecStarFormationHistogram, self).reassemble(*options) + return reassembled / 1e9 # Msol per Gyr -> Msol per yr From 1361067432c56f4d51d22546926793ca51129efd Mon Sep 17 00:00:00 2001 From: mtremmel Date: Fri, 15 Jun 2018 15:38:46 -0400 Subject: [PATCH 27/42] fix SpecSFR_histogram, add/fix documentation --- docs/histogram_properties.md | 12 ++++++++++-- tangos/properties/pynbody/SF.py | 9 ++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/histogram_properties.md b/docs/histogram_properties.md index 892cea5e..77f7bda6 100644 --- a/docs/histogram_properties.md +++ b/docs/histogram_properties.md @@ -116,10 +116,18 @@ property (like `SFR_histogram`) and returns a new histogram. Because such a calc in the database, it can be performed on the fly and reassembled accordingly in the same way as the original stored histogram data. -The property `SpecSFR_histogram` calculates the specific star formation rate history of a halo given `SFR_histogram`. - +The property `SpecSFR_histogram` calculates the specific star formation rate history of a halo given `SFR_histogram` and +normalizing it by the mass at every intermediate step. We can call this in the same way as any other histogram property, but +this time we are doing a live calculation, which is called like any other using `()` (in this case there are +no relevant arguments to give). +``` +halo.calculate('reassemble(SpecSFR_histogram())') +``` +It is still important to make sure that the time resolution is set correctly. For any live calculation acting on a histogram +property, the time resolution, `sim["histogram_delta_t_Gyr"]`, should be set to what it was when the stored histogram + data was originally calculated. Exercising caution ------------------- diff --git a/tangos/properties/pynbody/SF.py b/tangos/properties/pynbody/SF.py index 02682b21..737de491 100644 --- a/tangos/properties/pynbody/SF.py +++ b/tangos/properties/pynbody/SF.py @@ -54,11 +54,14 @@ def requires_property(self): def live_calculate(self, halo, *args): sfr = halo.calculate('raw(SFR_histogram)') - Mstar_f = halo['Mstar'] - Mstar_i = Mstar_f - np.sum(sfr*self.pixel_delta_t_Gyr) + try: + Mstar_i = halo.previous.calculate('Mstar') + except: + Mstar_i = halo['Mstar'] - np.sum(sfr*self.pixel_delta_t_Gyr) + Mstar_t = Mstar_i + np.cumsum(sfr*self.pixel_delta_t_Gyr) return sfr/Mstar_t def reassemble(self, *options): reassembled = super(SpecStarFormationHistogram, self).reassemble(*options) - return reassembled / 1e9 # Msol per Gyr -> Msol per yr + return reassembled / 1e9 # Gyr^-1 -> yr^-1 From a8380a27849b94c2ec2663e06a4f8e3af63906ec Mon Sep 17 00:00:00 2001 From: mtremmel Date: Fri, 15 Jun 2018 17:01:38 -0400 Subject: [PATCH 28/42] edits to documentation --- docs/histogram_properties.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/histogram_properties.md b/docs/histogram_properties.md index 77f7bda6..1cfd6d70 100644 --- a/docs/histogram_properties.md +++ b/docs/histogram_properties.md @@ -116,18 +116,18 @@ property (like `SFR_histogram`) and returns a new histogram. Because such a calc in the database, it can be performed on the fly and reassembled accordingly in the same way as the original stored histogram data. -The property `SpecSFR_histogram` calculates the specific star formation rate history of a halo given `SFR_histogram` and +The property `SpecSFR_histogram`, for example, calculates the specific star formation rate history of a halo given `SFR_histogram` and normalizing it by the mass at every intermediate step. We can call this in the same way as any other histogram property, but -this time we are doing a live calculation, which is called like any other using `()` (in this case there are -no relevant arguments to give). +this time we are doing a live calculation, which is called like any other live calculation (in this case there are +no relevant arguments to give, but any argument taken by the live calculation will work like normal). ``` halo.calculate('reassemble(SpecSFR_histogram())') ``` It is still important to make sure that the time resolution is set correctly. For any live calculation acting on a histogram -property, the time resolution, `sim["histogram_delta_t_Gyr"]`, should be set to what it was when the stored histogram - data was originally calculated. +property, the time resolution, defined for each simulation by `sim["histogram_delta_t_Gyr"]`, should be set to what it +was when the stored histogram data was originally calculated. Exercising caution ------------------- From 814b51113325b0a503e8e89036be67e26a16f764 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 20 Jun 2018 15:55:46 -0400 Subject: [PATCH 29/42] combine the two different evaluation methods, using a flag instead for when reassembly needs to occur --- tangos/live_calculation/__init__.py | 47 ++++++++++--------- .../builtin_functions/reassembly.py | 5 +- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/tangos/live_calculation/__init__.py b/tangos/live_calculation/__init__.py index 9cf2ab16..44616a46 100644 --- a/tangos/live_calculation/__init__.py +++ b/tangos/live_calculation/__init__.py @@ -358,7 +358,7 @@ def __init__(self, *tokens): super(LiveProperty, self).__init__() self._name = str(tokens[0]) self._inputs = list(tokens[1:]) - self._evaluation_pattern = '_evaluate_function_with_reassemble' + self._evaluation_with_reassemble = True self._evaluation_options= [] def __str__(self): @@ -386,15 +386,17 @@ def _calculation_retrieves(self): result = result.union(providing_instance.requires_property()) return result - def set_evaluation_pattern(self, eval_name): - self._evaluation_pattern = eval_name + def set_reassemble(self): + self._evaluation_with_reassemble = True - def set_evaluation_pattern_with_options(self,eval_name,*options): - self._evaluation_pattern = eval_name + def set_raw(self): + self._evaluation_with_reassemble = False + + def set_evaluation_options(self,*options): self._evaluation_options = options def _evaluate(self, halos, input_descriptions, input_values): - return getattr(self, self._evaluation_pattern)(halos, input_descriptions, input_values, *self._evaluation_options) + return self._evaluate_function(halos, input_descriptions, input_values, *self._evaluation_options) def values_and_description(self, halos): input_values = [] @@ -415,26 +417,14 @@ def _input_value_and_description(self, input_id, halos): raise ValueError("Functions cannot receive more than one value per input") return val[0], desc - def _evaluate_function(self, halos, input_descriptions, input_values): + def _evaluate_function(self, halos, input_descriptions, input_values, *options): from .. import properties sim = consistent_collection.consistent_simulation_from_halos(halos) results = [] calculator = properties.providing_class(self.name())(sim, *input_descriptions) for inputs in zip(halos, *input_values): if self._has_required_properties(inputs[0]) and all([x is not None for x in inputs]): - results.append(calculator.live_calculate_named(self.name(), *inputs)) - else: - results.append(None) - return calculator, self._as_1xn_array(results) - - def _evaluate_function_with_reassemble(self, halos, input_descriptions, input_values, *options): - from .. import properties - sim = consistent_collection.consistent_simulation_from_halos(halos) - results = [] - calculator = properties.providing_class(self.name())(sim, *input_descriptions) - for inputs in zip(halos, *input_values): - if self._has_required_properties(inputs[0]) and all([x is not None for x in inputs]): - if hasattr(calculator,'reassemble'): + if hasattr(calculator,'reassemble') & self._evaluation_with_reassemble is True: results.append(calculator.reassemble(self, inputs[0], *options)) else: results.append(calculator.live_calculate_named(self.name(), *inputs)) @@ -442,6 +432,21 @@ def _evaluate_function_with_reassemble(self, halos, input_descriptions, input_va results.append(None) return calculator, self._as_1xn_array(results) + #def _evaluate_function_with_reassemble(self, halos, input_descriptions, input_values, *options): + # from .. import properties + # sim = consistent_collection.consistent_simulation_from_halos(halos) + # results = [] + # calculator = properties.providing_class(self.name())(sim, *input_descriptions) + # for inputs in zip(halos, *input_values): + # if self._has_required_properties(inputs[0]) and all([x is not None for x in inputs]): + # if hasattr(calculator,'reassemble'): + # results.append(calculator.reassemble(self, inputs[0], *options)) + # else: + # results.append(calculator.live_calculate_named(self.name(), *inputs)) + # else: + # results.append(None) + # return calculator, self._as_1xn_array(results) + @classmethod def _as_1xn_array(cls, results): results_array = np.empty((1, len(results)), dtype=object) @@ -505,7 +510,7 @@ def __init__(self, *tokens): super(BuiltinFunction, self).__init__(*tokens) self._func = self.__registered_functions[self._name]['function'] self._info = self.__registered_functions[self._name] - self.set_evaluation_pattern('_evaluate_function') + self.set_raw() for i in range(len(self._inputs)): assert_class = self._get_input_option(i, 'assert_class') if assert_class is not None and not isinstance(self._inputs[i], assert_class): diff --git a/tangos/live_calculation/builtin_functions/reassembly.py b/tangos/live_calculation/builtin_functions/reassembly.py index d652bab8..92670fa4 100644 --- a/tangos/live_calculation/builtin_functions/reassembly.py +++ b/tangos/live_calculation/builtin_functions/reassembly.py @@ -11,7 +11,7 @@ def raw(halos, values): @raw.set_initialisation def raw_initialisation(input): if isinstance(input, LiveProperty): - input.set_evaluation_pattern('_evaluate_function') + input.set_raw() else: input.set_extraction_pattern(extraction_patterns.halo_property_raw_value_getter) @@ -29,7 +29,8 @@ def reassemble_initialisation(input, *options): else: raise TypeError("Options to 'reassemble' must be fixed numbers or strings") if isinstance(input,LiveProperty): - input.set_evaluation_pattern_with_options('_evaluate_function_with_reassemble', *options_values) + input.set_reassemble() + input.set_evaluation_options(*options_values) else: input.set_extraction_pattern( extraction_patterns.HaloPropertyValueWithReassemblyOptionsGetter(*options_values)) From 2a807efb7c5640117018c33ae874b062a6ce0993 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jun 2018 09:12:33 -0400 Subject: [PATCH 30/42] remove commented out code --- tangos/live_calculation/__init__.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tangos/live_calculation/__init__.py b/tangos/live_calculation/__init__.py index 6ce53f15..55bf57ec 100644 --- a/tangos/live_calculation/__init__.py +++ b/tangos/live_calculation/__init__.py @@ -432,21 +432,6 @@ def _evaluate_function(self, halos, input_descriptions, input_values, *options): results.append(None) return calculator, self._as_1xn_array(results) - #def _evaluate_function_with_reassemble(self, halos, input_descriptions, input_values, *options): - # from .. import properties - # sim = consistent_collection.consistent_simulation_from_halos(halos) - # results = [] - # calculator = properties.providing_class(self.name())(sim, *input_descriptions) - # for inputs in zip(halos, *input_values): - # if self._has_required_properties(inputs[0]) and all([x is not None for x in inputs]): - # if hasattr(calculator,'reassemble'): - # results.append(calculator.reassemble(self, inputs[0], *options)) - # else: - # results.append(calculator.live_calculate_named(self.name(), *inputs)) - # else: - # results.append(None) - # return calculator, self._as_1xn_array(results) - @classmethod def _as_1xn_array(cls, results): results_array = np.empty((1, len(results)), dtype=object) From 27ee54226b936b6481a8467f1e037c3fced90dbb Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 6 Jul 2018 15:44:41 -0400 Subject: [PATCH 31/42] fix propery module importing --- tangos/config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tangos/config.py b/tangos/config.py index f0567324..7fc51bf4 100644 --- a/tangos/config.py +++ b/tangos/config.py @@ -27,9 +27,13 @@ # names of property modules to import; default is for backwards compatibility on systems with N-Body-Shop extensions -property_modules = os.environ.get("TANGOS_PROPERTY_MODULES","tangos_nbodyshop_properties") -property_modules = property_modules.split(",") -property_modules = map(str.strip, property_modules) +property_modules = os.environ.get("TANGOS_PROPERTY_MODULES") +if property_modules is None: + property_modules = [] +else: + property_modules = property_modules.split(",") + property_modules = map(str.strip, property_modules) + file_ignore_pattern = [] From 365024f7b90bc7e5c8f9a4cb8cc8c055b40ba9c8 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 11 Jul 2019 14:15:52 -0400 Subject: [PATCH 32/42] make stellar mass profiles not count black holes (particles with tform < 0) --- tangos/properties/pynbody/profile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tangos/properties/pynbody/profile.py b/tangos/properties/pynbody/profile.py index a0396522..250da603 100644 --- a/tangos/properties/pynbody/profile.py +++ b/tangos/properties/pynbody/profile.py @@ -1,6 +1,7 @@ import numpy as np from .spherical_region import SphericalRegionPropertyCalculation from .centring import centred_calculation +import pynbody class HaloDensityProfile(SphericalRegionPropertyCalculation): # include @@ -51,5 +52,5 @@ class BaryonicHaloDensityProfile(HaloDensityProfile): @centred_calculation def calculate(self, data, existing_properties): gas_a, gas_b = self._get_profile(data.gas, existing_properties["max_radius"]) - star_a, star_b = self._get_profile(data.star, existing_properties["max_radius"]) + star_a, star_b = self._get_profile(data.star[pynbody.filt.HighPass('tform',0)], existing_properties["max_radius"]) return gas_a, gas_b, star_a, star_b From 6e5dd43acc6b817b1575b62777f435151853b968 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 11 Jul 2019 14:20:39 -0400 Subject: [PATCH 33/42] remove import pynbody within _get_profile since it is now global --- tangos/properties/pynbody/profile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tangos/properties/pynbody/profile.py b/tangos/properties/pynbody/profile.py index 250da603..867d777c 100644 --- a/tangos/properties/pynbody/profile.py +++ b/tangos/properties/pynbody/profile.py @@ -21,8 +21,6 @@ def plot_ylabel(self): return r"$\rho/M_{\odot}\,kpc^{-3}$", r"$M/M_{\odot}$" def _get_profile(self, halo, maxrad): - import pynbody - delta = self.plot_xdelta() nbins = int(maxrad / delta) maxrad = delta * (nbins + 1) From 0e50a12dd9d73b104b008f999a405ad77d3f9414 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 11 Jul 2019 14:36:00 -0400 Subject: [PATCH 34/42] move import of pynbody in profile functions --- tangos/properties/pynbody/profile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tangos/properties/pynbody/profile.py b/tangos/properties/pynbody/profile.py index 867d777c..64714817 100644 --- a/tangos/properties/pynbody/profile.py +++ b/tangos/properties/pynbody/profile.py @@ -1,7 +1,6 @@ import numpy as np from .spherical_region import SphericalRegionPropertyCalculation from .centring import centred_calculation -import pynbody class HaloDensityProfile(SphericalRegionPropertyCalculation): # include @@ -21,6 +20,7 @@ def plot_ylabel(self): return r"$\rho/M_{\odot}\,kpc^{-3}$", r"$M/M_{\odot}$" def _get_profile(self, halo, maxrad): + import pynbody delta = self.plot_xdelta() nbins = int(maxrad / delta) maxrad = delta * (nbins + 1) @@ -49,6 +49,7 @@ class BaryonicHaloDensityProfile(HaloDensityProfile): @centred_calculation def calculate(self, data, existing_properties): + import pynbody gas_a, gas_b = self._get_profile(data.gas, existing_properties["max_radius"]) star_a, star_b = self._get_profile(data.star[pynbody.filt.HighPass('tform',0)], existing_properties["max_radius"]) return gas_a, gas_b, star_a, star_b From e4d3ceddaf4e8e252e5967881e02e475b14afc04 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 11 Jul 2019 14:36:33 -0400 Subject: [PATCH 35/42] move import of pynbody in profile functions --- tangos/properties/pynbody/profile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tangos/properties/pynbody/profile.py b/tangos/properties/pynbody/profile.py index 867d777c..64714817 100644 --- a/tangos/properties/pynbody/profile.py +++ b/tangos/properties/pynbody/profile.py @@ -1,7 +1,6 @@ import numpy as np from .spherical_region import SphericalRegionPropertyCalculation from .centring import centred_calculation -import pynbody class HaloDensityProfile(SphericalRegionPropertyCalculation): # include @@ -21,6 +20,7 @@ def plot_ylabel(self): return r"$\rho/M_{\odot}\,kpc^{-3}$", r"$M/M_{\odot}$" def _get_profile(self, halo, maxrad): + import pynbody delta = self.plot_xdelta() nbins = int(maxrad / delta) maxrad = delta * (nbins + 1) @@ -49,6 +49,7 @@ class BaryonicHaloDensityProfile(HaloDensityProfile): @centred_calculation def calculate(self, data, existing_properties): + import pynbody gas_a, gas_b = self._get_profile(data.gas, existing_properties["max_radius"]) star_a, star_b = self._get_profile(data.star[pynbody.filt.HighPass('tform',0)], existing_properties["max_radius"]) return gas_a, gas_b, star_a, star_b From 246629af769047d5f557c46beef180afba091e30 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Thu, 30 Apr 2020 13:23:16 -0400 Subject: [PATCH 36/42] correct missing argument to reassemble call in extraction_patterns. Update SF.py to include new halo argument to reassemble() in star formation rate histogram classes. --- tangos/core/extraction_patterns.py | 2 +- tangos/properties/pynbody/SF.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tangos/core/extraction_patterns.py b/tangos/core/extraction_patterns.py index d474284b..a617fcde 100644 --- a/tangos/core/extraction_patterns.py +++ b/tangos/core/extraction_patterns.py @@ -131,7 +131,7 @@ def _postprocess_one_result(self, property_object): if hasattr(self._providing_class, 'reassemble'): instance = self._providing_class(property_object.halo.timestep.simulation) - return instance.reassemble(property_object, *self._options) + return instance.reassemble(property_object, property_object.halo, *self._options) else: self._setup_data_mapper(property_object) return self._mapper.get(property_object) diff --git a/tangos/properties/pynbody/SF.py b/tangos/properties/pynbody/SF.py index 97cff394..0437932a 100644 --- a/tangos/properties/pynbody/SF.py +++ b/tangos/properties/pynbody/SF.py @@ -33,8 +33,8 @@ def calculate(self, halo, existing_properties): return M - def reassemble(self, *options): - reassembled = super(StarFormHistogram, self).reassemble(*options) + def reassemble(self, halo, *options): + reassembled = super(StarFormHistogram, self).reassemble(halo, *options) return reassembled/1e9 # Msol per Gyr -> Msol per yr class SpecStarFormationHistogram(TimeChunkedProperty,LiveHaloProperties): @@ -59,8 +59,8 @@ def live_calculate(self, halo, *args): Marray = Marray[::-1] return sfr/Marray - def reassemble(self, *options): - reassembled = super(SpecStarFormationHistogram, self).reassemble(*options) + def reassemble(self, halo, *options): + reassembled = super(SpecStarFormationHistogram, self).reassemble(halo, *options) return reassembled / 1e9 # Gyr^-1 -> yr^-1 class StarForm(PynbodyPropertyCalculation): From eceec766450cc43e9b18c55bfcf97282a233a220 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Thu, 30 Apr 2020 15:09:43 -0400 Subject: [PATCH 37/42] fix missing halo input to _reassemble_using_finding_strategy() in reassemble function --- tangos/properties/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tangos/properties/__init__.py b/tangos/properties/__init__.py index 8256aa0a..da4bf354 100644 --- a/tangos/properties/__init__.py +++ b/tangos/properties/__init__.py @@ -270,7 +270,7 @@ def reassemble(self, property, halo, reassembly_type='major'): if reassembly_type=='major': return self._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopMajorProgenitorsStrategy) elif reassembly_type=='major_across_simulations': - return self._reassemble_using_finding_strategy(property, strategy = rfs.MultiHopMajorProgenitorsStrategy, + return self._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopMajorProgenitorsStrategy, strategy_kwargs = {'target': None, 'one_simulation': False}) elif reassembly_type=='sum': return self._reassemble_using_finding_strategy(property, halo, strategy = rfs.MultiHopAllProgenitorsStrategy) From e1fdd80208c64798733aa5ba1ea52923aa73045f Mon Sep 17 00:00:00 2001 From: mtremmel Date: Thu, 30 Apr 2020 15:10:43 -0400 Subject: [PATCH 38/42] remove changes accidentally included to examples/mergers --- tangos/examples/mergers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tangos/examples/mergers.py b/tangos/examples/mergers.py index 8f5f3ca0..6b0037f2 100644 --- a/tangos/examples/mergers.py +++ b/tangos/examples/mergers.py @@ -15,7 +15,6 @@ def get_mergers_of_major_progenitor(input_halo): redshift = [] ratio = [] halo = [] - time = [] while input_halo is not None: mergers = db.relation_finding.MultiHopMostRecentMergerStrategy(input_halo, order_by='weight').all() if len(mergers)>0 : @@ -23,12 +22,11 @@ def get_mergers_of_major_progenitor(input_halo): redshift.append(mergers[0].timestep.next.redshift) halo.append((mergers[0], m)) ratio.append(float(mergers[0].NDM)/m.NDM) - time.append(mergers[0].timestep.next.time_gyr) input_halo = mergers[0] else: input_halo = None - return np.array(redshift), np.array(ratio), time, halo + return np.array(redshift), np.array(ratio), halo def most_major_mergers_since(ts, Mvir_min=0.8e12, Mvir_max=2e12, z_merger_max=4.0, no_merger_value=None): """Given a timestep, calculate the most major merger ratio since a given redshift for all halos in a specified mass range From 5c47a878960874850f2fffd920a599a935882665 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Thu, 30 Apr 2020 15:11:51 -0400 Subject: [PATCH 39/42] remove superfluous whitespace change --- tangos/core/halo_data/property.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tangos/core/halo_data/property.py b/tangos/core/halo_data/property.py index 819bd98a..51e0862c 100644 --- a/tangos/core/halo_data/property.py +++ b/tangos/core/halo_data/property.py @@ -69,6 +69,7 @@ def data(self): def get_data_with_reassembly_options(self, *options): return extraction_patterns.HaloPropertyValueWithReassemblyOptionsGetter(*options).postprocess_data_objects([self])[0] + @property def description(self): return self.name.providing_class(type(self.halo.handler))(self.halo.timestep.simulation) From 503617bf394d53e17df932d2124b1dc627a2eee2 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Thu, 30 Apr 2020 15:16:31 -0400 Subject: [PATCH 40/42] remove changes to config.py that were spuriously included here --- tangos/config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tangos/config.py b/tangos/config.py index 3b1d86d7..5f82df2a 100644 --- a/tangos/config.py +++ b/tangos/config.py @@ -29,12 +29,18 @@ # names of property modules to import; default is for backwards compatibility on systems with N-Body-Shop extensions +# N.B. since version 1.0.10 it is also possible to use entry point tangos.property_modules. For example, in your +# my_fab_tangos_properties package setup.py's call to setup() add the keyword argument: +# +# entry_points={"tangos.property_modules" : [ +# "fab = my_fab_tangos_properties" +# ]} +# + +property_modules = os.environ.get("TANGOS_PROPERTY_MODULES","tangos_nbodyshop_properties") +property_modules = property_modules.split(",") +property_modules = map(str.strip, property_modules) property_modules = os.environ.get("TANGOS_PROPERTY_MODULES") -if property_modules is None: - property_modules = [] -else: - property_modules = property_modules.split(",") - property_modules = map(str.strip, property_modules) file_ignore_pattern = [] From 86b58571ee15f48bef6a27b5b0bb662fbeef694b Mon Sep 17 00:00:00 2001 From: mtremmel Date: Thu, 30 Apr 2020 15:20:05 -0400 Subject: [PATCH 41/42] remove spurious changes to .gitignore which were included --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d1fe95e7..4170c8ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *.pyc -.idea/vcs.xml .idea build/* *.db @@ -7,4 +6,4 @@ build/* test_tutorial_build/tutorial_* dist/* tests/test_dbs/ -.DS_store +.DS_store \ No newline at end of file From 6a6db251d4e0494fb9e01e0d48c030deed3ecbb1 Mon Sep 17 00:00:00 2001 From: mtremmel Date: Thu, 30 Apr 2020 15:53:35 -0400 Subject: [PATCH 42/42] remove bad line from config.py accidentally left from previous commit --- tangos/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tangos/config.py b/tangos/config.py index 5f82df2a..4c586e42 100644 --- a/tangos/config.py +++ b/tangos/config.py @@ -40,7 +40,6 @@ property_modules = os.environ.get("TANGOS_PROPERTY_MODULES","tangos_nbodyshop_properties") property_modules = property_modules.split(",") property_modules = map(str.strip, property_modules) -property_modules = os.environ.get("TANGOS_PROPERTY_MODULES") file_ignore_pattern = []