From d0e813c4ae5136414e9fdaeb4ed0ccda6679e919 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Mon, 21 Apr 2025 21:58:47 +0100 Subject: [PATCH 01/22] libstempo.pyx: shuffle some observation/pulsar stricture contents to match order in tempo2.h file --- libstempo/libstempo.pyx | 100 +++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index cd30843..9f20622 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -150,23 +150,21 @@ cdef extern from "tempo2.h": long double sat_day # Just the Day part long double sat_sec # Just the Sec part long double bat # barycentric arrival time - long double bbat # barycentric arrival time long double batCorr #update from sat-> bat + long double bbat # barycentric arrival time long double pet # pulsar emission time int clockCorr # = 1 for clock corrections to be applied, = 0 for BAT int delayCorr # = 1 for time delay corrections to be applied, = 0 for BAT int deleted # 1 if observation deleted, -1 if not in fit long double prefitResidual long double residual - double toaErr # error on TOA (in us) - double origErr # original error on TOA after reading tim file (in us) - double toaDMErr # error on TOA due to DM (in us) - char **flagID # ID of flags - char **flagVal # Value of flags - int nFlags # Number of flags set double freq # frequency of observation (in MHz) double freqSSB # frequency of observation in barycentric frame (in Hz) - char fname[MAX_FILELEN] # name of datafile giving TOA + double toaErr # error on TOA (in us) + double toaDMErr # error on TOA due to DM (in us) + double origErr # original error on TOA after reading tim file (in us) + double phaseOffset # Phase offset + char fname[MAX_FILELEN + 1] # name of datafile giving TOA char telID[100] # telescope ID double sun_ssb[6] # Sun wrt SSB double earth_ssb[6] # Earth center wrt SSB @@ -174,18 +172,21 @@ cdef extern from "tempo2.h": double observatory_earth[6] # Obs wrt Earth center double psrPos[3] # Unit vector to the pulsar position double zenith[3] # Zenith vector, in BC frame. Length=geodetic height - long double torb # Combined binary delay - long long pulseN # Pulse number - long double roemer # Roemer delay double shapiroDelaySun # Shapiro delay caused by the Sun - double phaseOffset # Phase offset + long double roemer # Roemer delay + long double torb # Combined binary delay long double phase # the phase (cycles) - double efac # Error multiplication factor - double equad # Value to add in quadrature + long long pulseN # Pulse number + char flagID[MAX_FLAGS][MAX_FLAG_LEN] # ID of flags + char flagVal[MAX_FLAGS][MAX_FLAG_LEN] # Value of flags + int nFlags # Number of flags set int jump[MAX_FLAGS] # Jump region + #double jumpScale[MAX_FLAGS] int obsNjump # Number of jumps for this observation int fdjump[MAX_FLAGS] int obsNfdjump + double efac # Error multiplication factor + double equad # Value to add in quadrature ctypedef int param_label @@ -204,27 +205,44 @@ cdef extern from "tempo2.h": double height_grs80 # GRS80 geodetic height ctypedef struct pulsar: + char name[100] parameter param[MAX_PARAMS] - observation *obsn - char *name - int nobs - int rescaleErrChisq - int noWarnings - double fitChisq + char binaryModel[100] + + double posPulsar[3] # 3-unitvector pointing at the pulsar + double ne_sw + int nJumps char fjumpID[16] double jumpVal[MAX_JUMPS] # char jumpSAT[MAX_JUMPS] int fitJump[MAX_JUMPS] double jumpValErr[MAX_JUMPS] - char *binaryModel + # char jumpScaled[MAX_JUMPS] + + # new parameters for fdjumps + int nfdJumps + char ffdjumpID[16] + double fdjumpVal[MAX_JUMPS] + int fdjumpIdx[MAX_JUMPS] + int fitfdJump[MAX_JUMPS] + double fdjumpValErr[MAX_JUMPS] + char fdjump_log + + double fitChisq + int rescaleErrChisq + + observation *obsn + int nobs int t2cMethod # How to transform from terrestrial to celestial coords. Set in parfile with T2CMETHOD # tempo2 supports T2C_IAU2000B (default) and T2C_TEMPO - char *JPL_EPHEMERIS - char *ephemeris + int noWarnings + + char JPL_EPHEMERIS[MAX_FILELEN] + char ephemeris[MAX_FILELEN] int useCalceph + char tzrsite[100] int eclCoord # = 1 for ecliptic coords otherwise celestial coords - double posPulsar[3] # 3-unitvector pointing at the pulsar # long double phaseJump[MAX_JUMPS] # Time of phase jump (Deprecated. WHY?) int phaseJumpID[MAX_JUMPS] # ID of closest point to phase jump int phaseJumpDir[MAX_JUMPS] # Size and direction of phase jump @@ -232,39 +250,22 @@ cdef extern from "tempo2.h": double rmsPost char clock[16] FitInfo fitinfo - - double ne_sw - double ne_sw_ifuncT[MAX_IFUNC] - double ne_sw_ifuncV[MAX_IFUNC] - double ne_sw_ifuncE[MAX_IFUNC] - int ne_sw_ifuncN - - # new parameters for fdjumps - int nfdJumps - char ffdjumpID[16] - double fdjumpVal[MAX_JUMPS] - int fdjumpIdx[MAX_JUMPS] - int fitfdJump[MAX_JUMPS] - double fdjumpValErr[MAX_JUMPS] - char fdjump_log # noise parameters follow - # T2EFAC + # T2EFAC/EQUAD int nT2efac + int nT2equad char T2efacFlagID[MAX_T2EFAC][MAX_FLAG_LEN] char T2efacFlagVal[MAX_T2EFAC][MAX_FLAG_LEN] double T2efacVal[MAX_T2EFAC] - - # GLOBAL_EFAC in timfile??? - double T2globalEfac - - # T2EQUAD - int nT2equad char T2equadFlagID[MAX_T2EQUAD][MAX_FLAG_LEN] char T2equadFlagVal[MAX_T2EQUAD][MAX_FLAG_LEN] double T2equadVal[MAX_T2EQUAD] + # GLOBAL_EFAC in timfile??? + double T2globalEfac + # TNEF int nTNEF char TNEFFlagID[MAX_TNEF][MAX_FLAG_LEN] @@ -305,7 +306,12 @@ cdef extern from "tempo2.h": # set reference observation char refphs - char tzrsite[100] + + double ne_sw_ifuncT[MAX_IFUNC] + double ne_sw_ifuncV[MAX_IFUNC] + double ne_sw_ifuncE[MAX_IFUNC] + int ne_sw_ifuncN + void initialise(pulsar *psr, int noWarnings) void destroyOne(pulsar *psr) From 40762da6533a933c279c8e329abdc2f214d5c0e9 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Sun, 27 Apr 2025 21:51:10 +0100 Subject: [PATCH 02/22] Update packaging and setuptools build-time requirements --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 091ce44..a77fca3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,8 @@ [build-system] requires = [ - "setuptools>=61", + "packaging>=24.2", + "setuptools>=77.0.1", "setuptools_scm[toml]>=6.2", "wheel", # ephem package likes to have wheel installed "cython", From da52f3dea04a77f480cb3298adfffd0264237f9c Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Sun, 27 Apr 2025 22:11:25 +0100 Subject: [PATCH 03/22] pyproject.toml: switch license back to table style to still allow Python 3.8 usage --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a77fca3..1804459 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,8 @@ [build-system] requires = [ "packaging>=24.2", - "setuptools>=77.0.1", + 'setuptools>=77; python_version > "3.8"', + 'setuptools>=74; python_version == 3.8"', "setuptools_scm[toml]>=6.2", "wheel", # ephem package likes to have wheel installed "cython", @@ -26,7 +27,7 @@ description = "A Python wrapper for tempo2" authors = [{name = "Michele Vallisneri", email = "vallis@vallis.org"}] urls = { Homepage = "https://github.com/vallis/libstempo" } readme = "README.md" -license = "MIT" +license = { text = "MIT" } license-files = [ "LICENSE" ] classifiers=[ "Intended Audience :: Developers", From c32ada455265f8c6e9feaa36d2b407c3314a7e04 Mon Sep 17 00:00:00 2001 From: Matt Pitkin Date: Mon, 28 Apr 2025 09:05:30 +0100 Subject: [PATCH 04/22] Update pyproject.toml - add missing " --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1804459..4942278 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires = [ "packaging>=24.2", 'setuptools>=77; python_version > "3.8"', - 'setuptools>=74; python_version == 3.8"', + 'setuptools>=74; python_version == "3.8"', "setuptools_scm[toml]>=6.2", "wheel", # ephem package likes to have wheel installed "cython", From e5b8022caad2b2d716a414bd6e2942d47cda8dd1 Mon Sep 17 00:00:00 2001 From: Matt Pitkin Date: Mon, 28 Apr 2025 09:16:14 +0100 Subject: [PATCH 05/22] Update pyproject.toml - move license file back to setuptools section --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4942278..cdf05a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ [build-system] requires = [ "packaging>=24.2", - 'setuptools>=77; python_version > "3.8"', + 'setuptools>=77, <80; python_version > "3.8"', 'setuptools>=74; python_version == "3.8"', "setuptools_scm[toml]>=6.2", "wheel", # ephem package likes to have wheel installed @@ -28,8 +28,7 @@ authors = [{name = "Michele Vallisneri", email = "vallis@vallis.org"}] urls = { Homepage = "https://github.com/vallis/libstempo" } readme = "README.md" license = { text = "MIT" } -license-files = [ "LICENSE" ] -classifiers=[ + classifiers=[ "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Operating System :: MacOS", @@ -73,6 +72,7 @@ dev = [ [tool.setuptools] include-package-data = true +license-files = [ "LICENSE" ] [tool.setuptools.packages.find] include = [ From 7448b6beaad3989414a3c11252779149583449ce Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Mon, 28 Apr 2025 22:11:36 +0100 Subject: [PATCH 06/22] install_tempo2.sh: allow install version to be specified --- .github/workflows/ci_tests.yml | 5 ++-- install_tempo2.sh | 47 ++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 31140b8..347e0db 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -21,6 +21,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + tempo2-version: ['2021.07.1-correct', '2023.05.1', '2024.11.1', '2025.02.1'] steps: - name: Checkout repository @@ -34,11 +35,11 @@ jobs: run: | brew unlink gcc && brew link gcc brew install automake libtool - ./install_tempo2.sh + ./install_tempo2.sh -v ${{ matrix.tempo2-version }} - name: Install tempo2 on linux if: runner.os == 'Linux' run: | - ./install_tempo2.sh + ./install_tempo2.sh -v ${{ matrix.tempo2-version }} - name: Install dependencies and package run: | python -m pip install --upgrade pip diff --git a/install_tempo2.sh b/install_tempo2.sh index 44eeb69..91b85d8 100755 --- a/install_tempo2.sh +++ b/install_tempo2.sh @@ -1,21 +1,46 @@ #!/bin/bash -e -# get install location -if [ $# -eq 0 ] - then - echo 'No install location defined, using' $HOME'/.local/' - prefix=$HOME/.local/ - else - prefix=$1 - echo 'Will install in' $prefix +# default Tempo2 version +tempo2version="2021.07.1-correct" + +usage() { echo "Usage: $0 [-p ] [-v ]" 1>&2; exit 1; } + +# default install location +prefix=$HOME/.local/ +if [[ $# -eq 1 && "$1" != "-h" ]]; then + # interpret single argument as install location + prefix=$1 + echo 'Will install in' $prefix +else + # allow arguments + while getopts ":p:v:" o; do + case "${o}" in + p) + prefix=${OPTARG} + ;; + v) + tempo2version=${OPTARG} + ;; + *) + usage + ;; + esac +done fi +echo 'Will install in' $prefix +echo 'Will attempt to install tempo2 version ' $tempo2version + # make a destination directory for runtime files export TEMPO2=$prefix/share/tempo2 mkdir -p $TEMPO2 -curl -O https://bitbucket.org/psrsoft/tempo2/get/2021.07.1-correct.tar.gz -tar zxvf 2021.07.1-correct.tar.gz +curl -O https://bitbucket.org/psrsoft/tempo2/get/${tempo2version}.tar.gz +if [ $? -eq 0 ]; then + echo 'Version '${tempo2version}' of Tempo2 does not exist. Please see, e.g., https://github.com/mattpitkin/tempo2/tags for a list of allowed version tags.' + exit 1 +fi +tar -zxvf ${tempo2version}.tar.gz cd psrsoft-tempo2-* @@ -33,5 +58,5 @@ cp -r T2runtime/* $TEMPO2 cd .. rm -rf psrsoft-tempo2-* -rm -rf 2021.07.1-correct.tar.gz +rm -rf ${tempo2version}.tar.gz echo "Set TEMPO2 environment variable to ${TEMPO2} to make things run more smoothly." From fcbf749fb829ecf6e9e66e4310aee8e87159c24c Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Mon, 28 Apr 2025 22:34:53 +0100 Subject: [PATCH 07/22] install_tempo2.sh: fix script exit --- install_tempo2.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install_tempo2.sh b/install_tempo2.sh index 91b85d8..338c0b9 100755 --- a/install_tempo2.sh +++ b/install_tempo2.sh @@ -29,14 +29,14 @@ done fi echo 'Will install in' $prefix -echo 'Will attempt to install tempo2 version ' $tempo2version +echo 'Will attempt to install tempo2 version' $tempo2version # make a destination directory for runtime files export TEMPO2=$prefix/share/tempo2 mkdir -p $TEMPO2 -curl -O https://bitbucket.org/psrsoft/tempo2/get/${tempo2version}.tar.gz -if [ $? -eq 0 ]; then +dl=$(curl -Of https://bitbucket.org/psrsoft/tempo2/get/${tempo2version}.tar.gz || echo $?) +if [ $dl -ne 0 ]; then echo 'Version '${tempo2version}' of Tempo2 does not exist. Please see, e.g., https://github.com/mattpitkin/tempo2/tags for a list of allowed version tags.' exit 1 fi From d37526d60b15a94f98d3068b45d47f937e6c73e6 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Mon, 28 Apr 2025 23:12:05 +0100 Subject: [PATCH 08/22] libstempo.pyx: switch from using sprintf to copy par file and time file name strings --- libstempo/libstempo.pyx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 9f20622..41c004d 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -2190,17 +2190,19 @@ cdef class tempopulsar: Save current par file (calls tempo2's `textOutput(...)`).""" - cdef char parFile[MAX_FILELEN] + #cdef char parFile[MAX_FILELEN] if not parfile: parfile = self.parfile parfile_bytes = parfile.encode() + cdef char *parFile = parfile_bytes if len(parfile_bytes) > MAX_FILELEN - 1: raise IOError("Parfile name {0} too long for tempo2!".format(parfile)) - stdio.sprintf(parFile,"%s",parfile_bytes) + + #stdio.sprintf(parFile,"%s",parfile_bytes) # void textOutput(pulsar *psr,int npsr, # double globalParameter, -- ? @@ -2220,17 +2222,18 @@ cdef class tempopulsar: Save current par file (calls tempo2's `writeTim(...)`).""" - cdef char timFile[MAX_FILELEN] + #cdef char timFile[MAX_FILELEN] if not timfile: timfile = self.timfile timfile_bytes = timfile.encode() + cdef char *timFile = timfile_bytes if len(timfile_bytes) > MAX_FILELEN - 1: raise IOError("Timfile name {0} too long for tempo2!".format(timfile)) - stdio.sprintf(timFile,"%s",timfile_bytes) + #stdio.sprintf(timFile,"%s",timfile_bytes) writeTim(timFile,&(self.psr[0]),'tempo2') From d28986e3f657406b5a3a956c26e478749c88c9b5 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Wed, 30 Apr 2025 22:47:07 +0100 Subject: [PATCH 09/22] libstempo.pyx: various string copying changes --- libstempo/libstempo.pyx | 72 ++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 41c004d..376c48e 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -6,13 +6,23 @@ from packaging import version import collections from collections import OrderedDict + # what is the default encoding here? -string = lambda s: s.decode() +def string(buf): + # take bytes up to the first '\0' + raw = bytes(buf).split(b'\0', 1)[0] + # try UTF-8, else fall back to Latin-1 (one-to-one byte→codepoint) + try: + return raw.decode('utf-8') + except UnicodeDecodeError: + return raw.decode('latin-1') + + string_dtype = 'U' from libc cimport stdlib, stdio -from libc.string cimport strncpy, memset +from libc.string cimport strcpy, strncpy, memset from cython cimport view @@ -816,8 +826,14 @@ cdef class tempopulsar: if len(checkfile) > MAX_FILELEN - 1: raise IOError("Filename {0} is too long for tempo2.".format(checkfile)) - stdio.sprintf(parFile[0],"%s",parfile_bytes) - stdio.sprintf(timFile[0],"%s",timfile_bytes) + cdef const char *par_file_c_str = parfile_bytes + cdef const char *tim_file_c_str = timfile_bytes + + strcpy(parFile[0], par_file_c_str) + strcpy(timFile[0], tim_file_c_str) + + #stdio.sprintf(parFile[0],"%s",parfile_bytes) + #stdio.sprintf(timFile[0],"%s",timfile_bytes) readParfile(self.psr,parFile,timFile,self.npsr) # load the parameters (all pulsars) @@ -956,9 +972,11 @@ cdef class tempopulsar: def _setstring(self,char* string,maxlen,value): value_bytes = value.encode() + cdef const char *value_c_str = value_bytes - if len(value_bytes) < maxlen: - stdio.sprintf(string,"%s",value_bytes) + if len(value_bytes) < maxlen - 1: + #stdio.sprintf(string,"%s",value_bytes) + strcpy(string, value_c_str) else: raise ValueError @@ -1040,7 +1058,7 @@ cdef class tempopulsar: for i in range(self.psr[0].nobs): # make sure fname array is empty memset(&(self.psr[0].obsn[i].fname[0]), 0, MAX_FILELEN * sizeof(char)) - strncpy(&(self.psr[0].obsn[i].fname[0]), "FAKE", 4 * sizeof(char)) + strncpy(&(self.psr[0].obsn[i].fname[0]), "FAKE\0", 5 * sizeof(char)) self._inputtoaerrs() self._inputobservatory() @@ -1119,7 +1137,8 @@ cdef class tempopulsar: # set the observatories for i in range(self.psr[0].nobs): - strncpy(&(self.psr[0].obsn[i].telID[0]), obsv[i], 100 * sizeof(char)) + obstr = obsv[i][:99] + "\0" # append null character + strncpy(&(self.psr[0].obsn[i].telID[0]), obstr, len(obstr) * sizeof(char)) # set which corrections to apply (taken from TEMPO2 readTimfile.C) if obsv[i][0] == "@" or obsv[i] == "bat": @@ -1168,9 +1187,11 @@ cdef class tempopulsar: def __set__(self,value): # this is OK in both Python 2 and 3 name_bytes = value.encode() + cdef const char *name_c_str = name_bytes - if len(name_bytes) < 100: - stdio.sprintf(self.psr[0].name,"%s",name_bytes) + if len(name_bytes) < 100 - 1: + strcpy(self.psr[0].name, name_c_str) + #stdio.sprintf(self.psr[0].name,"%s",name_bytes) else: raise ValueError @@ -1182,9 +1203,11 @@ cdef class tempopulsar: def __set__(self,value): model_bytes = value.encode() + cdef const char *model_c_str = model_bytes - if len(model_bytes) < 100: - stdio.sprintf(self.psr[0].binaryModel,"%s",model_bytes) + if len(model_bytes) < 100 - 1: + strcpy(self.psr[0].binaryModel, model_c_str) + #stdio.sprintf(self.psr[0].binaryModel,"%s",model_bytes) else: raise ValueError @@ -1197,12 +1220,15 @@ cdef class tempopulsar: def __set__(self,value): def seteph(filename,usecalceph=False): model_bytes = filename.encode() + cdef const char *model_c_str = model_bytes - if len(model_bytes) < MAX_FILELEN: - stdio.sprintf(self.psr[0].JPL_EPHEMERIS,"%s",model_bytes) + if len(model_bytes) < MAX_FILELEN - 1: + strcpy(self.psr[0].JPL_EPHEMERIS, model_c_str) + #stdio.sprintf(self.psr[0].JPL_EPHEMERIS,"%s",model_bytes) # older tempo2 versions use ephemeris instead of JPL_EPHEMERIS for calceph. - stdio.sprintf(self.psr[0].ephemeris, "%s",model_bytes) + strcpy(self.psr[0].ephemeris, model_c_str) + #stdio.sprintf(self.psr[0].ephemeris, "%s",model_bytes) self.psr[0].useCalceph = int(usecalceph) @@ -1244,9 +1270,11 @@ cdef class tempopulsar: def __set__(self,value): value_bytes = value.encode() + cdef const char *value_c_str = value_bytes - if len(value_bytes) < 16: - stdio.sprintf(self.psr[0].clock,"%s",value_bytes) + if len(value_bytes) < (16 - 1): + strcpy(self.psr[0].clock, value_c_str) + #stdio.sprintf(self.psr[0].clock,"%s",value_bytes) else: raise ValueError("CLK name '{}' is too long.".format(value)) @@ -1704,7 +1732,8 @@ cdef class tempopulsar: # set reference epoch self["TZRMJD"].val = epoch if site is not None: - strncpy(&(self.psr[0].tzrsite[0]), str.encode(site), 100 * sizeof(char)) + sitestr = str.encode(site) + b"\0" # append null character + strncpy(&(self.psr[0].tzrsite[0]), sitestr, len(sitestr) * sizeof(char)) if freq is not None: self["TZRFRQ"] = freq self.psr[0].refphs = REFPHS_TZR @@ -2196,12 +2225,11 @@ cdef class tempopulsar: parfile = self.parfile parfile_bytes = parfile.encode() - cdef char *parFile = parfile_bytes - + if len(parfile_bytes) > MAX_FILELEN - 1: raise IOError("Parfile name {0} too long for tempo2!".format(parfile)) - + cdef const char *parFile = parfile_bytes #stdio.sprintf(parFile,"%s",parfile_bytes) # void textOutput(pulsar *psr,int npsr, @@ -2228,11 +2256,11 @@ cdef class tempopulsar: timfile = self.timfile timfile_bytes = timfile.encode() - cdef char *timFile = timfile_bytes if len(timfile_bytes) > MAX_FILELEN - 1: raise IOError("Timfile name {0} too long for tempo2!".format(timfile)) + cdef const char *timFile = timfile_bytes #stdio.sprintf(timFile,"%s",timfile_bytes) writeTim(timFile,&(self.psr[0]),'tempo2') From 9cf8842b25e0785eecec6b2595c4d2e754619e43 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Thu, 1 May 2025 22:22:24 +0100 Subject: [PATCH 10/22] libstempo.pyx: try string function from Cython docs --- .github/workflows/ci_tests.yml | 6 +++--- libstempo/libstempo.pyx | 27 ++++++++++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 347e0db..ffb56d0 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -19,9 +19,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - tempo2-version: ['2021.07.1-correct', '2023.05.1', '2024.11.1', '2025.02.1'] + os: [ubuntu-latest] #, macos-latest] + python-version: ['3.8'] #, '3.9', '3.10', '3.11', '3.12'] + tempo2-version: ['2021.07.1-correct'] #, '2023.05.1', '2024.11.1', '2025.02.1'] steps: - name: Checkout repository diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 376c48e..365ef87 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -8,14 +8,27 @@ from collections import OrderedDict # what is the default encoding here? -def string(buf): +#def string(buf): # take bytes up to the first '\0' - raw = bytes(buf).split(b'\0', 1)[0] - # try UTF-8, else fall back to Latin-1 (one-to-one byte→codepoint) - try: - return raw.decode('utf-8') - except UnicodeDecodeError: - return raw.decode('latin-1') +# raw = bytes(buf).split(b'\0', 1)[0] +# # try UTF-8, else fall back to Latin-1 (one-to-one byte→codepoint) +# try: +# return raw.decode('utf-8') +# except UnicodeDecodeError: +# return raw.decode('latin-1') + +cdef str string(s): + if type(s) is str: + # Fast path for most common case(s). + return s + elif isinstance(s, str): + # We know from the fast path above that 's' can only be a subtype here. + # An evil cast to might still work in some(!) cases, + # depending on what the further processing does. To be safe, + # we can always create a copy instead. + return str(s) + else: + raise TypeError("Could not convert to str.") string_dtype = 'U' From 2a502f535e697e68f5fccd07f5b34dada9f78fad Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Thu, 1 May 2025 22:49:50 +0100 Subject: [PATCH 11/22] Minor fix --- libstempo/libstempo.pyx | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 365ef87..e6a4011 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -8,28 +8,13 @@ from collections import OrderedDict # what is the default encoding here? -#def string(buf): +def string(buf): # take bytes up to the first '\0' -# raw = bytes(buf).split(b'\0', 1)[0] -# # try UTF-8, else fall back to Latin-1 (one-to-one byte→codepoint) -# try: -# return raw.decode('utf-8') -# except UnicodeDecodeError: -# return raw.decode('latin-1') - -cdef str string(s): - if type(s) is str: - # Fast path for most common case(s). - return s - elif isinstance(s, str): - # We know from the fast path above that 's' can only be a subtype here. - # An evil cast to might still work in some(!) cases, - # depending on what the further processing does. To be safe, - # we can always create a copy instead. - return str(s) - else: - raise TypeError("Could not convert to str.") - + raw = bytes(buf).split(b'\0', 1)[0] + try: + return raw.decode('utf-8') + except UnicodeDecodeError: + return raw.decode('latin-1') string_dtype = 'U' @@ -1150,7 +1135,7 @@ cdef class tempopulsar: # set the observatories for i in range(self.psr[0].nobs): - obstr = obsv[i][:99] + "\0" # append null character + obstr = obsv[i][:99] + b"\0" # append null character strncpy(&(self.psr[0].obsn[i].telID[0]), obstr, len(obstr) * sizeof(char)) # set which corrections to apply (taken from TEMPO2 readTimfile.C) From 09498a657499ac471102af77b14d91429167f077 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Thu, 1 May 2025 22:54:39 +0100 Subject: [PATCH 12/22] Re-add matrix of test runners --- .github/workflows/ci_tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index ffb56d0..347e0db 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -19,9 +19,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] #, macos-latest] - python-version: ['3.8'] #, '3.9', '3.10', '3.11', '3.12'] - tempo2-version: ['2021.07.1-correct'] #, '2023.05.1', '2024.11.1', '2025.02.1'] + os: [ubuntu-latest, macos-latest] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + tempo2-version: ['2021.07.1-correct', '2023.05.1', '2024.11.1', '2025.02.1'] steps: - name: Checkout repository From 45a00580a6dd8792d03c463fdd660ffafe19dedd Mon Sep 17 00:00:00 2001 From: Matt Pitkin Date: Wed, 7 May 2025 17:31:50 +0100 Subject: [PATCH 13/22] Update libstempo.pyx: try using snprintf --- libstempo/libstempo.pyx | 76 +++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index e6a4011..eb207b3 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -824,14 +824,17 @@ cdef class tempopulsar: if len(checkfile) > MAX_FILELEN - 1: raise IOError("Filename {0} is too long for tempo2.".format(checkfile)) - cdef const char *par_file_c_str = parfile_bytes - cdef const char *tim_file_c_str = timfile_bytes + cdef const char *par_file_c_str = parfile_bytes + b"\0" + cdef const char *tim_file_c_str = timfile_bytes + b"\0" - strcpy(parFile[0], par_file_c_str) - strcpy(timFile[0], tim_file_c_str) + pflen = sizeof(char) * (len(parfile_bytes) + 1) + tflen = sizeof(char) * (len(timfile_bytes) + 1) - #stdio.sprintf(parFile[0],"%s",parfile_bytes) - #stdio.sprintf(timFile[0],"%s",timfile_bytes) + #strcpy(parFile[0], par_file_c_str) + #strcpy(timFile[0], tim_file_c_str) + + stdio.snprintf(parFile[0], pflen, "%s", par_file_c_str) + stdio.snprintf(timFile[0], tflen, "%s", tim_bytes_c_str) readParfile(self.psr,parFile,timFile,self.npsr) # load the parameters (all pulsars) @@ -970,11 +973,12 @@ cdef class tempopulsar: def _setstring(self,char* string,maxlen,value): value_bytes = value.encode() - cdef const char *value_c_str = value_bytes + cdef const char *value_c_str = value_bytes + b"\0" if len(value_bytes) < maxlen - 1: - #stdio.sprintf(string,"%s",value_bytes) - strcpy(string, value_c_str) + vblen = sizeof(char) * (len(value_bytes) + 1) + stdio.snprintf(string, vblen, "%s", value_c_str) + #strcpy(string, value_c_str) else: raise ValueError @@ -1185,11 +1189,12 @@ cdef class tempopulsar: def __set__(self,value): # this is OK in both Python 2 and 3 name_bytes = value.encode() - cdef const char *name_c_str = name_bytes + cdef const char *name_c_str = name_bytes + b"\0" if len(name_bytes) < 100 - 1: - strcpy(self.psr[0].name, name_c_str) - #stdio.sprintf(self.psr[0].name,"%s",name_bytes) + #strcpy(self.psr[0].name, name_c_str) + nlen = sizeof(char) * (len(name_bytes) + 1) + stdio.snprintf(self.psr[0].name, nlen, "%s", name_c_str) else: raise ValueError @@ -1201,11 +1206,12 @@ cdef class tempopulsar: def __set__(self,value): model_bytes = value.encode() - cdef const char *model_c_str = model_bytes + cdef const char *model_c_str = model_bytes + b"\0" if len(model_bytes) < 100 - 1: - strcpy(self.psr[0].binaryModel, model_c_str) - #stdio.sprintf(self.psr[0].binaryModel,"%s",model_bytes) + #strcpy(self.psr[0].binaryModel, model_c_str) + mblen = sizeof(char) * (len(model_bytes) + 1) + stdio.snprintf(self.psr[0].binaryModel, mblen, "%s", model_c_str) else: raise ValueError @@ -1218,15 +1224,16 @@ cdef class tempopulsar: def __set__(self,value): def seteph(filename,usecalceph=False): model_bytes = filename.encode() - cdef const char *model_c_str = model_bytes + cdef const char *model_c_str = model_bytes + b"\0" if len(model_bytes) < MAX_FILELEN - 1: - strcpy(self.psr[0].JPL_EPHEMERIS, model_c_str) - #stdio.sprintf(self.psr[0].JPL_EPHEMERIS,"%s",model_bytes) + #strcpy(self.psr[0].JPL_EPHEMERIS, model_c_str) + mblen = sizeof(char) * (len(model_bytes) + 1) + stdio.snprintf(self.psr[0].JPL_EPHEMERIS, mblen, "%s", model_c_str) # older tempo2 versions use ephemeris instead of JPL_EPHEMERIS for calceph. - strcpy(self.psr[0].ephemeris, model_c_str) - #stdio.sprintf(self.psr[0].ephemeris, "%s",model_bytes) + #strcpy(self.psr[0].ephemeris, model_c_str) + stdio.snprintf(self.psr[0].ephemeris, mblen, "%s", model_c_str) self.psr[0].useCalceph = int(usecalceph) @@ -1268,11 +1275,12 @@ cdef class tempopulsar: def __set__(self,value): value_bytes = value.encode() - cdef const char *value_c_str = value_bytes + cdef const char *value_c_str = value_bytes + b"\0" if len(value_bytes) < (16 - 1): - strcpy(self.psr[0].clock, value_c_str) - #stdio.sprintf(self.psr[0].clock,"%s",value_bytes) + #strcpy(self.psr[0].clock, value_c_str) + vlen = sizeof(char) * (len(value_bytes) + 1) + stdio.snprintf(self.psr[0].clock, vlen, "%s", value_c_str) else: raise ValueError("CLK name '{}' is too long.".format(value)) @@ -2217,18 +2225,19 @@ cdef class tempopulsar: Save current par file (calls tempo2's `textOutput(...)`).""" - #cdef char parFile[MAX_FILELEN] + cdef char parFile[MAX_FILELEN] if not parfile: parfile = self.parfile - parfile_bytes = parfile.encode() + cdef const char *parfile_bytes = parfile.encode() + b"\0" - if len(parfile_bytes) > MAX_FILELEN - 1: + if len(parfile) > MAX_FILELEN - 1: raise IOError("Parfile name {0} too long for tempo2!".format(parfile)) - cdef const char *parFile = parfile_bytes - #stdio.sprintf(parFile,"%s",parfile_bytes) + #cdef const char parFile + pflen = sizeof(char) * (len(parfile) + 1) + stdio.snprintf(parFile, pflen, "%s", parfile_bytes) # void textOutput(pulsar *psr,int npsr, # double globalParameter, -- ? @@ -2248,18 +2257,19 @@ cdef class tempopulsar: Save current par file (calls tempo2's `writeTim(...)`).""" - #cdef char timFile[MAX_FILELEN] + cdef char timFile[MAX_FILELEN] if not timfile: timfile = self.timfile - timfile_bytes = timfile.encode() + cdef const char *timfile_bytes = timfile.encode() + b"\0" - if len(timfile_bytes) > MAX_FILELEN - 1: + if len(timfile) > MAX_FILELEN - 1: raise IOError("Timfile name {0} too long for tempo2!".format(timfile)) - cdef const char *timFile = timfile_bytes - #stdio.sprintf(timFile,"%s",timfile_bytes) + #cdef const char *timFile = timfile_bytes + tflen = sizeof(char) * (len(timefile) + 1) + stdio.snprintf(timFile, tflen, "%s", timfile_bytes) writeTim(timFile,&(self.psr[0]),'tempo2') From 46c9dff85fea435e64eb46efbc7204b379aa3010 Mon Sep 17 00:00:00 2001 From: Matt Pitkin Date: Wed, 7 May 2025 17:39:49 +0100 Subject: [PATCH 14/22] Update libstempo.pyx: remove tabs --- libstempo/libstempo.pyx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index eb207b3..a531afb 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -827,8 +827,8 @@ cdef class tempopulsar: cdef const char *par_file_c_str = parfile_bytes + b"\0" cdef const char *tim_file_c_str = timfile_bytes + b"\0" - pflen = sizeof(char) * (len(parfile_bytes) + 1) - tflen = sizeof(char) * (len(timfile_bytes) + 1) + pflen = sizeof(char) * (len(parfile_bytes) + 1) + tflen = sizeof(char) * (len(timfile_bytes) + 1) #strcpy(parFile[0], par_file_c_str) #strcpy(timFile[0], tim_file_c_str) @@ -1193,7 +1193,7 @@ cdef class tempopulsar: if len(name_bytes) < 100 - 1: #strcpy(self.psr[0].name, name_c_str) - nlen = sizeof(char) * (len(name_bytes) + 1) + nlen = sizeof(char) * (len(name_bytes) + 1) stdio.snprintf(self.psr[0].name, nlen, "%s", name_c_str) else: raise ValueError @@ -1210,7 +1210,7 @@ cdef class tempopulsar: if len(model_bytes) < 100 - 1: #strcpy(self.psr[0].binaryModel, model_c_str) - mblen = sizeof(char) * (len(model_bytes) + 1) + mblen = sizeof(char) * (len(model_bytes) + 1) stdio.snprintf(self.psr[0].binaryModel, mblen, "%s", model_c_str) else: raise ValueError @@ -1228,7 +1228,7 @@ cdef class tempopulsar: if len(model_bytes) < MAX_FILELEN - 1: #strcpy(self.psr[0].JPL_EPHEMERIS, model_c_str) - mblen = sizeof(char) * (len(model_bytes) + 1) + mblen = sizeof(char) * (len(model_bytes) + 1) stdio.snprintf(self.psr[0].JPL_EPHEMERIS, mblen, "%s", model_c_str) # older tempo2 versions use ephemeris instead of JPL_EPHEMERIS for calceph. @@ -1279,7 +1279,7 @@ cdef class tempopulsar: if len(value_bytes) < (16 - 1): #strcpy(self.psr[0].clock, value_c_str) - vlen = sizeof(char) * (len(value_bytes) + 1) + vlen = sizeof(char) * (len(value_bytes) + 1) stdio.snprintf(self.psr[0].clock, vlen, "%s", value_c_str) else: raise ValueError("CLK name '{}' is too long.".format(value)) @@ -2236,7 +2236,7 @@ cdef class tempopulsar: raise IOError("Parfile name {0} too long for tempo2!".format(parfile)) #cdef const char parFile - pflen = sizeof(char) * (len(parfile) + 1) + pflen = sizeof(char) * (len(parfile) + 1) stdio.snprintf(parFile, pflen, "%s", parfile_bytes) # void textOutput(pulsar *psr,int npsr, From 11f047f80e31bbb8afcf5284fd8eb7787a016f2a Mon Sep 17 00:00:00 2001 From: Matt Pitkin Date: Wed, 7 May 2025 17:44:52 +0100 Subject: [PATCH 15/22] Update libstempo.pyx: remove another tab --- libstempo/libstempo.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index a531afb..4dc6f57 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -977,7 +977,7 @@ cdef class tempopulsar: if len(value_bytes) < maxlen - 1: vblen = sizeof(char) * (len(value_bytes) + 1) - stdio.snprintf(string, vblen, "%s", value_c_str) + stdio.snprintf(string, vblen, "%s", value_c_str) #strcpy(string, value_c_str) else: raise ValueError From db4b31f928a6ffe30359ae779268adf1ec5c03dc Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Wed, 7 May 2025 22:32:32 +0100 Subject: [PATCH 16/22] libstempo.pyx: various compilation fixes --- libstempo/libstempo.pyx | 78 ++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 4dc6f57..77e6c83 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -818,23 +818,23 @@ cdef class tempopulsar: else: raise IOError("Cannot find timfile {0}.".format(timfile)) - parfile_bytes, timfile_bytes = parfile.encode(), timfile.encode() + parfile_bytes, timfile_bytes = (parfile + "\0").encode(), (timfile + "\0").encode() - for checkfile in [parfile_bytes,timfile_bytes]: - if len(checkfile) > MAX_FILELEN - 1: + for checkfile in [parfile_bytes, timfile_bytes]: + if len(checkfile) > MAX_FILELEN: raise IOError("Filename {0} is too long for tempo2.".format(checkfile)) - cdef const char *par_file_c_str = parfile_bytes + b"\0" - cdef const char *tim_file_c_str = timfile_bytes + b"\0" + cdef const char *par_file_c_str = parfile_bytes + cdef const char *tim_file_c_str = timfile_bytes - pflen = sizeof(char) * (len(parfile_bytes) + 1) - tflen = sizeof(char) * (len(timfile_bytes) + 1) + cdef size_t pflen = sizeof(char) * len(parfile_bytes) + cdef size_t tflen = sizeof(char) * len(timfile_bytes) #strcpy(parFile[0], par_file_c_str) #strcpy(timFile[0], tim_file_c_str) stdio.snprintf(parFile[0], pflen, "%s", par_file_c_str) - stdio.snprintf(timFile[0], tflen, "%s", tim_bytes_c_str) + stdio.snprintf(timFile[0], tflen, "%s", tim_file_c_str) readParfile(self.psr,parFile,timFile,self.npsr) # load the parameters (all pulsars) @@ -972,11 +972,11 @@ cdef class tempopulsar: return array def _setstring(self,char* string,maxlen,value): - value_bytes = value.encode() - cdef const char *value_c_str = value_bytes + b"\0" + value_bytes = (value + "\0").encode() + cdef const char *value_c_str = value_bytes + cdef size_t vblen = sizeof(char) * len(value_bytes) - if len(value_bytes) < maxlen - 1: - vblen = sizeof(char) * (len(value_bytes) + 1) + if len(value_bytes) < maxlen: stdio.snprintf(string, vblen, "%s", value_c_str) #strcpy(string, value_c_str) else: @@ -1188,12 +1188,12 @@ cdef class tempopulsar: def __set__(self,value): # this is OK in both Python 2 and 3 - name_bytes = value.encode() - cdef const char *name_c_str = name_bytes + b"\0" + name_bytes = (value + "\0").encode() + cdef const char *name_c_str = name_bytes + cdef size_t nlen = sizeof(char) * len(name_bytes) - if len(name_bytes) < 100 - 1: + if len(name_bytes) < 100: #strcpy(self.psr[0].name, name_c_str) - nlen = sizeof(char) * (len(name_bytes) + 1) stdio.snprintf(self.psr[0].name, nlen, "%s", name_c_str) else: raise ValueError @@ -1205,12 +1205,12 @@ cdef class tempopulsar: return string(self.psr[0].binaryModel) def __set__(self,value): - model_bytes = value.encode() - cdef const char *model_c_str = model_bytes + b"\0" + model_bytes = (value + "\0").encode() + cdef const char *model_c_str = model_bytes + cdef size_t mblen = sizeof(char) * len(model_bytes) - if len(model_bytes) < 100 - 1: + if len(model_bytes) < 100: #strcpy(self.psr[0].binaryModel, model_c_str) - mblen = sizeof(char) * (len(model_bytes) + 1) stdio.snprintf(self.psr[0].binaryModel, mblen, "%s", model_c_str) else: raise ValueError @@ -1223,12 +1223,12 @@ cdef class tempopulsar: def __set__(self,value): def seteph(filename,usecalceph=False): - model_bytes = filename.encode() - cdef const char *model_c_str = model_bytes + b"\0" + model_bytes = (filename + "\0").encode() + cdef const char *model_c_str = model_bytes + cdef size_t mblen = sizeof(char) * len(model_bytes) - if len(model_bytes) < MAX_FILELEN - 1: + if len(model_bytes) < MAX_FILELEN: #strcpy(self.psr[0].JPL_EPHEMERIS, model_c_str) - mblen = sizeof(char) * (len(model_bytes) + 1) stdio.snprintf(self.psr[0].JPL_EPHEMERIS, mblen, "%s", model_c_str) # older tempo2 versions use ephemeris instead of JPL_EPHEMERIS for calceph. @@ -1273,13 +1273,13 @@ cdef class tempopulsar: def __get__(self): return string(self.psr[0].clock) - def __set__(self,value): - value_bytes = value.encode() - cdef const char *value_c_str = value_bytes + b"\0" + def __set__(self, value): + value_bytes = (value + "\0").encode() + cdef const char *value_c_str = value_bytes + cdef size_t vlen = sizeof(char) * len(value_bytes) - if len(value_bytes) < (16 - 1): + if len(value_bytes) < 16: #strcpy(self.psr[0].clock, value_c_str) - vlen = sizeof(char) * (len(value_bytes) + 1) stdio.snprintf(self.psr[0].clock, vlen, "%s", value_c_str) else: raise ValueError("CLK name '{}' is too long.".format(value)) @@ -2230,14 +2230,16 @@ cdef class tempopulsar: if not parfile: parfile = self.parfile - cdef const char *parfile_bytes = parfile.encode() + b"\0" + parfile_bytes = (parfile + "\0").encode() + + cdef const char *parfile_c_bytes = parfile_bytes - if len(parfile) > MAX_FILELEN - 1: + if len(parfile_bytes) > MAX_FILELEN: raise IOError("Parfile name {0} too long for tempo2!".format(parfile)) #cdef const char parFile - pflen = sizeof(char) * (len(parfile) + 1) - stdio.snprintf(parFile, pflen, "%s", parfile_bytes) + cdef size_t pflen = sizeof(char) * len(parfile_bytes) + stdio.snprintf(parFile, pflen, "%s", parfile_c_bytes) # void textOutput(pulsar *psr,int npsr, # double globalParameter, -- ? @@ -2262,14 +2264,16 @@ cdef class tempopulsar: if not timfile: timfile = self.timfile - cdef const char *timfile_bytes = timfile.encode() + b"\0" + timfile_bytes = (timfile + "\0").encode() + + cdef const char *timfile_c_bytes = timfile_bytes - if len(timfile) > MAX_FILELEN - 1: + if len(timfile_bytes) > MAX_FILELEN: raise IOError("Timfile name {0} too long for tempo2!".format(timfile)) #cdef const char *timFile = timfile_bytes - tflen = sizeof(char) * (len(timefile) + 1) - stdio.snprintf(timFile, tflen, "%s", timfile_bytes) + cdef size_t tflen = sizeof(char) * len(timfile_bytes) + stdio.snprintf(timFile, tflen, "%s", timfile_c_bytes) writeTim(timFile,&(self.psr[0]),'tempo2') From 65aba0f4c9c7d6d0e249e1e5c6c916c2e97c659a Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Mon, 1 Dec 2025 21:42:47 +0000 Subject: [PATCH 17/22] Ignore bad characters --- libstempo/libstempo.pyx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 77e6c83..8cd4c15 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -10,11 +10,16 @@ from collections import OrderedDict # what is the default encoding here? def string(buf): # take bytes up to the first '\0' - raw = bytes(buf).split(b'\0', 1)[0] + #raw = bytes(buf).split(b'\0', 1)[0] try: - return raw.decode('utf-8') + return buf.decode('utf-8') except UnicodeDecodeError: - return raw.decode('latin-1') + try: + return buf.decode('latin-1') + except UnicodeDecodeError: + # just ignore invalid characters + return buf.decode('ascii', errors='ignore') + string_dtype = 'U' From 2e03d281aa42e08c083f84b545e6a747b98dc145 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Mon, 1 Dec 2025 22:23:05 +0000 Subject: [PATCH 18/22] Make sure encoding is all utf8 and erroring values are ignored --- libstempo/libstempo.pyx | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 8cd4c15..ae0bfc1 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -823,7 +823,7 @@ cdef class tempopulsar: else: raise IOError("Cannot find timfile {0}.".format(timfile)) - parfile_bytes, timfile_bytes = (parfile + "\0").encode(), (timfile + "\0").encode() + parfile_bytes, timfile_bytes = (parfile + "\0").encode("utf-8", errors="ignore"), (timfile + "\0").encode("utf-8", errors="ignore") for checkfile in [parfile_bytes, timfile_bytes]: if len(checkfile) > MAX_FILELEN: @@ -977,7 +977,7 @@ cdef class tempopulsar: return array def _setstring(self,char* string,maxlen,value): - value_bytes = (value + "\0").encode() + value_bytes = (value + "\0").encode("utf-8", errors="ignore") cdef const char *value_c_str = value_bytes cdef size_t vblen = sizeof(char) * len(value_bytes) @@ -1193,7 +1193,7 @@ cdef class tempopulsar: def __set__(self,value): # this is OK in both Python 2 and 3 - name_bytes = (value + "\0").encode() + name_bytes = (value + "\0").encode("utf-8", errors="ignore") cdef const char *name_c_str = name_bytes cdef size_t nlen = sizeof(char) * len(name_bytes) @@ -1210,7 +1210,7 @@ cdef class tempopulsar: return string(self.psr[0].binaryModel) def __set__(self,value): - model_bytes = (value + "\0").encode() + model_bytes = (value + "\0").encode("utf-8", errors="ignore") cdef const char *model_c_str = model_bytes cdef size_t mblen = sizeof(char) * len(model_bytes) @@ -1228,7 +1228,7 @@ cdef class tempopulsar: def __set__(self,value): def seteph(filename,usecalceph=False): - model_bytes = (filename + "\0").encode() + model_bytes = (filename + "\0").encode("utf-8", errors="ignore") cdef const char *model_c_str = model_bytes cdef size_t mblen = sizeof(char) * len(model_bytes) @@ -1279,7 +1279,7 @@ cdef class tempopulsar: return string(self.psr[0].clock) def __set__(self, value): - value_bytes = (value + "\0").encode() + value_bytes = (value + "\0").encode("utf-8", errors="ignore") cdef const char *value_c_str = value_bytes cdef size_t vlen = sizeof(char) * len(value_bytes) @@ -1743,7 +1743,7 @@ cdef class tempopulsar: # set reference epoch self["TZRMJD"].val = epoch if site is not None: - sitestr = str.encode(site) + b"\0" # append null character + sitestr = str(site).encode("utf-8", errors="ignore") + b"\0" # append null character strncpy(&(self.psr[0].tzrsite[0]), sitestr, len(sitestr) * sizeof(char)) if freq is not None: self["TZRFRQ"] = freq @@ -2235,7 +2235,7 @@ cdef class tempopulsar: if not parfile: parfile = self.parfile - parfile_bytes = (parfile + "\0").encode() + parfile_bytes = (parfile + "\0").encode("utf-8", errors="ignore") cdef const char *parfile_c_bytes = parfile_bytes @@ -2269,7 +2269,7 @@ cdef class tempopulsar: if not timfile: timfile = self.timfile - timfile_bytes = (timfile + "\0").encode() + timfile_bytes = (timfile + "\0").encode("utf-8", errors="ignore") cdef const char *timfile_c_bytes = timfile_bytes @@ -2311,11 +2311,11 @@ def rewritetim(timfile): # encodes are needed here because file is open in binary mode if m: - out.write('{0} {1}/{2}\n'.format(m.group(1),os.path.dirname(timfile),m.group(2)).encode()) + out.write('{0} {1}/{2}\n'.format(m.group(1),os.path.dirname(timfile),m.group(2)).encode("utf-8", errors="ignore")) else: - out.write(line.encode()) + out.write(line.encode("utf-8", errors="ignore")) else: - out.write(line.encode()) + out.write(line.encode("utf-8", errors="ignore")) return out.name From e70a08ee8dfda2a15fa4cd46cd1202e16944f0ee Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Mon, 1 Dec 2025 22:33:09 +0000 Subject: [PATCH 19/22] Print out file names --- libstempo/libstempo.pyx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index ae0bfc1..c9369ef 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -2237,6 +2237,8 @@ cdef class tempopulsar: parfile_bytes = (parfile + "\0").encode("utf-8", errors="ignore") + print(parfile_bytes) + cdef const char *parfile_c_bytes = parfile_bytes if len(parfile_bytes) > MAX_FILELEN: @@ -2271,6 +2273,8 @@ cdef class tempopulsar: timfile_bytes = (timfile + "\0").encode("utf-8", errors="ignore") + print(timfile_bytes) + cdef const char *timfile_c_bytes = timfile_bytes if len(timfile_bytes) > MAX_FILELEN: From 21bf4ed249c0adf36f4de16489900c7330689b35 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Thu, 4 Dec 2025 22:20:31 +0000 Subject: [PATCH 20/22] Try plalform dependent longdouble --- libstempo/libstempo.pyx | 129 ++++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index c9369ef..488c6e2 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -9,16 +9,11 @@ from collections import OrderedDict # what is the default encoding here? def string(buf): - # take bytes up to the first '\0' - #raw = bytes(buf).split(b'\0', 1)[0] try: return buf.decode('utf-8') except UnicodeDecodeError: - try: - return buf.decode('latin-1') - except UnicodeDecodeError: - # just ignore invalid characters - return buf.decode('ascii', errors='ignore') + # just replace invalid characters + return buf.decode('utf-8', errors='replace') string_dtype = 'U' @@ -82,6 +77,8 @@ try: except AttributeError: NP_LONG_DOUBLE_FORMAT = numpy.double +ctypedef numpy.longdouble_t longdouble + # return numpy array as astropy table with unit def dimensionfy(unit): def dimensionfy_decorator(func): @@ -99,16 +96,16 @@ cdef extern from "GWsim-stub.h": cdef bint HAVE_GWSIM ctypedef struct gwSrc: - long double theta_g - long double phi_g - long double omega_g - long double phi_polar_g + longdouble theta_g + longdouble phi_g + longdouble omega_g + longdouble phi_polar_g - void GWbackground(gwSrc *gw,int numberGW,long *idum,long double flo,long double fhi,double gwAmp,double alpha,int loglin) - void GWdipolebackground(gwSrc *gw,int numberGW,long *idum,long double flo,long double fhi, double gwAmp,double alpha,int loglin, double *dipoleamps) + void GWbackground(gwSrc *gw,int numberGW,long *idum,longdouble flo,longdouble fhi,double gwAmp,double alpha,int loglin) + void GWdipolebackground(gwSrc *gw,int numberGW,long *idum,longdouble flo,longdouble fhi, double gwAmp,double alpha,int loglin, double *dipoleamps) void setupGW(gwSrc *gw) - void setupPulsar_GWsim(long double ra_p,long double dec_p,long double *kp) - long double calculateResidualGW(long double *kp,gwSrc *gw,long double obstime,long double dist) + void setupPulsar_GWsim(longdouble ra_p,longdouble dec_p,longdouble *kp) + longdouble calculateResidualGW(longdouble *kp,gwSrc *gw,longdouble obstime,longdouble dist) cdef extern from "tempo2.h": enum: MAX_PSR_VAL @@ -149,28 +146,28 @@ cdef extern from "tempo2.h": ctypedef struct parameter: char **label char **shortlabel - long double *val - long double *err + longdouble *val + longdouble *err int *fitFlag int *paramSet - long double *prefit - long double *prefitErr + longdouble *prefit + longdouble *prefitErr int aSize ctypedef struct observation: - long double sat # site arrival time - long double origsat # Backup of SAT - long double sat_day # Just the Day part - long double sat_sec # Just the Sec part - long double bat # barycentric arrival time - long double batCorr #update from sat-> bat - long double bbat # barycentric arrival time - long double pet # pulsar emission time + longdouble sat # site arrival time + longdouble origsat # Backup of SAT + longdouble sat_day # Just the Day part + longdouble sat_sec # Just the Sec part + longdouble bat # barycentric arrival time + longdouble batCorr #update from sat-> bat + longdouble bbat # barycentric arrival time + longdouble pet # pulsar emission time int clockCorr # = 1 for clock corrections to be applied, = 0 for BAT int delayCorr # = 1 for time delay corrections to be applied, = 0 for BAT int deleted # 1 if observation deleted, -1 if not in fit - long double prefitResidual - long double residual + longdouble prefitResidual + longdouble residual double freq # frequency of observation (in MHz) double freqSSB # frequency of observation in barycentric frame (in Hz) double toaErr # error on TOA (in us) @@ -186,9 +183,9 @@ cdef extern from "tempo2.h": double psrPos[3] # Unit vector to the pulsar position double zenith[3] # Zenith vector, in BC frame. Length=geodetic height double shapiroDelaySun # Shapiro delay caused by the Sun - long double roemer # Roemer delay - long double torb # Combined binary delay - long double phase # the phase (cycles) + longdouble roemer # Roemer delay + longdouble torb # Combined binary delay + longdouble phase # the phase (cycles) long long pulseN # Pulse number char flagID[MAX_FLAGS][MAX_FLAG_LEN] # ID of flags char flagVal[MAX_FLAGS][MAX_FLAG_LEN] # Value of flags @@ -256,7 +253,7 @@ cdef extern from "tempo2.h": int useCalceph char tzrsite[100] int eclCoord # = 1 for ecliptic coords otherwise celestial coords - # long double phaseJump[MAX_JUMPS] # Time of phase jump (Deprecated. WHY?) + # longdouble phaseJump[MAX_JUMPS] # Time of phase jump (Deprecated. WHY?) int phaseJumpID[MAX_JUMPS] # ID of closest point to phase jump int phaseJumpDir[MAX_JUMPS] # Size and direction of phase jump int nPhaseJump # Number of phase jumps @@ -362,10 +359,10 @@ cdef extern from "t2fit-stub.h": void t2fit_fillOneParameterFitInfo(pulsar* psr,param_label fit_param,const int k,FitInfo& OUT) -cdef void set_longdouble_from_array(long double *p,numpy.ndarray[numpy.npy_longdouble,ndim=0] a): - p[0] = ((a.data))[0] +cdef void set_longdouble_from_array(longdouble *p,numpy.ndarray[numpy.npy_longdouble,ndim=0] a): + p[0] = ((a.data))[0] -cdef void set_longdouble(long double *p,object o): +cdef void set_longdouble(longdouble *p,object o): if isinstance(o,numpy.longdouble): set_longdouble_from_array(p,o[...]) elif isinstance(o,numpy.ndarray) and o.dtype == numpy.longdouble and o.ndim == 0: @@ -373,9 +370,9 @@ cdef void set_longdouble(long double *p,object o): else: p[0] = o -cdef object get_longdouble_as_scalar(long double v): +cdef object get_longdouble_as_scalar(longdouble v): cdef numpy.ndarray ret = numpy.array(0,dtype=numpy.longdouble) - (ret.data)[0] = v + (ret.data)[0] = v return ret.item() cdef class tempopar: @@ -406,7 +403,7 @@ cdef class tempopar: property val: def __get__(self): if not self._isjump and not self._isfdjump: - return self._unitify(get_longdouble_as_scalar((self._val)[0])) + return self._unitify(get_longdouble_as_scalar((self._val)[0])) else: return self._unitify(float((self._val)[0])) @@ -423,14 +420,14 @@ cdef class tempopar: if not self._paramSet[0]: self._paramSet[0] = 1 - set_longdouble(self._val,value) + set_longdouble(self._val,value) else: (self._val)[0] = value property err: def __get__(self): if not self._isjump and not self._isfdjump: - return self._unitify(get_longdouble_as_scalar((self._err)[0])) + return self._unitify(get_longdouble_as_scalar((self._err)[0])) else: return self._unitify(float((self._err)[0])) @@ -439,7 +436,7 @@ cdef class tempopar: value = value.to(self.unit).value if not self._isjump and not self._isfdjump: - set_longdouble(self._err,value) + set_longdouble(self._err,value) else: (self._err)[0] = value @@ -631,19 +628,19 @@ cdef class GWB: stdlib.free(self.gw) def gwb_sig(self,tempopulsar pulsar, distance=1): - cdef long double dist = distance * 3.086e19 + cdef longdouble dist = distance * 3.086e19 - cdef long double ra_p = pulsar.psr[0].param[param_raj].val[0] - cdef long double dec_p = pulsar.psr[0].param[param_decj].val[0] + cdef longdouble ra_p = pulsar.psr[0].param[param_raj].val[0] + cdef longdouble dec_p = pulsar.psr[0].param[param_decj].val[0] - cdef long double epoch = pulsar.psr[0].param[param_pepoch].val[0] + cdef longdouble epoch = pulsar.psr[0].param[param_pepoch].val[0] - cdef long double kp[3] + cdef longdouble kp[3] setupPulsar_GWsim(ra_p,dec_p,&kp[0]) - cdef numpy.ndarray[long double,ndim=1] res = numpy.zeros(pulsar.nobs,numpy.longdouble) - cdef long double obstime + cdef numpy.ndarray[longdouble,ndim=1] res = numpy.zeros(pulsar.nobs,numpy.longdouble) + cdef longdouble obstime for i in range(pulsar.nobs): obstime = (pulsar.psr[0].obsn[i].sat - epoch)*86400.0 @@ -1011,8 +1008,8 @@ cdef class tempopulsar: def _set_observation_from_input(self, toas): """Fill in all observation values from input TOAs.""" - cdef long double [:] _satday = &(self.psr[0].obsn[0].sat_day) - cdef long double [:] _satsec = &(self.psr[0].obsn[0].sat_sec) + cdef longdouble [:] _satday = &(self.psr[0].obsn[0].sat_day) + cdef longdouble [:] _satsec = &(self.psr[0].obsn[0].sat_sec) cdef int [:] _deleted = &(self.psr[0].obsn[0].deleted) cdef int [:] _nflags = &(self.psr[0].obsn[0].nFlags) cdef double [:] _phaseoff = &(self.psr[0].obsn[0].phaseOffset) @@ -1412,7 +1409,7 @@ cdef class tempopulsar: Return computed SSB TOAs in MJD as a numpy.longdouble array. You get a copy of the current tempo2 array.""" - cdef long double [:] _toas = &(self.psr[0].obsn[0].bat) + cdef longdouble [:] _toas = &(self.psr[0].obsn[0].bat) _toas.strides[0] = sizeof(observation) if updatebats: @@ -1427,7 +1424,7 @@ cdef class tempopulsar: You get a view of the original tempo2 data structure, which you can write to.""" def __get__(self): - cdef long double [:] _stoas = &(self.psr[0].obsn[0].sat) + cdef longdouble [:] _stoas = &(self.psr[0].obsn[0].sat) _stoas.strides[0] = sizeof(observation) return numpy.asarray(_stoas) @@ -1436,7 +1433,7 @@ cdef class tempopulsar: """Return Roemer delay in seconds as a numpy.longdouble array.""" def __get__(self): - cdef long double [:] _roemer = &(self.psr[0].obsn[0].roemer) + cdef longdouble [:] _roemer = &(self.psr[0].obsn[0].roemer) _roemer.strides[0] = sizeof(observation) return numpy.asarray(_roemer) @@ -1584,10 +1581,10 @@ cdef class tempopulsar: def origSats(self): """tempopulsar.origSats() - Return originally loaded value of the SAT in case it is updated afterwards. Returned as long double array. + Return originally loaded value of the SAT in case it is updated afterwards. Returned as longdouble array. You get a copy of the current values.""" - cdef long double [:] _origSats = &(self.psr[0].obsn[0].origsat) + cdef longdouble [:] _origSats = &(self.psr[0].obsn[0].origsat) _origSats.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_origSats),u.s) if self.units else numpy.asarray(_origSats) @@ -1597,10 +1594,10 @@ cdef class tempopulsar: def satDay(self): """tempopulsar.satDay() - Return the day part of the SAT as long double array. + Return the day part of the SAT as longdouble array. You get a copy of the current values.""" - cdef long double [:] _satDays = &(self.psr[0].obsn[0].sat_day) + cdef longdouble [:] _satDays = &(self.psr[0].obsn[0].sat_day) _satDays.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_satDays),u.s) if self.units else numpy.asarray(_satDays) @@ -1610,10 +1607,10 @@ cdef class tempopulsar: def satSec(self): """tempopulsar.satSec() - Return decimal part of the SAT as a long double array + Return decimal part of the SAT as a longdouble array You get a copy of the current values.""" - cdef long double [:] _satSecs = &(self.psr[0].obsn[0].sat_sec) + cdef longdouble [:] _satSecs = &(self.psr[0].obsn[0].sat_sec) _satSecs.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_satSecs),u.s) if self.units else numpy.asarray(_satSecs) @@ -1626,7 +1623,7 @@ cdef class tempopulsar: Return computed correction to SSB in units of days. You get a copy of the current values.""" - cdef long double [:] _batCorr = &(self.psr[0].obsn[0].batCorr) + cdef longdouble [:] _batCorr = &(self.psr[0].obsn[0].batCorr) _batCorr.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_batCorr),u.s) if self.units else numpy.asarray(_batCorr) @@ -1691,7 +1688,7 @@ cdef class tempopulsar: Return computed pulsar emission times in MJD as a numpy.longdouble array. You get a copy of the current tempo2 array.""" - cdef long double [:] _pets = &(self.psr[0].obsn[0].pet) + cdef longdouble [:] _pets = &(self.psr[0].obsn[0].pet) _pets.strides[0] = sizeof(observation) if updatebats: @@ -1717,7 +1714,7 @@ cdef class tempopulsar: given, the `epoch`, `site` and `freq` values. """ - cdef long double [:] _res = &(self.psr[0].obsn[0].residual) + cdef longdouble [:] _res = &(self.psr[0].obsn[0].residual) _res.strides[0] = sizeof(observation) if removemean not in [True, False, 'weighted', 'first', 'refphs']: @@ -1838,7 +1835,7 @@ cdef class tempopulsar: cdef numpy.ndarray[double,ndim=2] ret = numpy.zeros((self.nobs,self.ndim+1),'d') - cdef long double epoch = self.psr[0].param[param_pepoch].val[0] + cdef longdouble epoch = self.psr[0].param[param_pepoch].val[0] cdef observation *obsns = self.psr[0].obsn if updatebats: @@ -1917,7 +1914,7 @@ cdef class tempopulsar: Does not reform residuals.""" # TODO: Is it not much faster to call DDmodel/XXmodel directly? - cdef long double [:] _torb = &(self.psr[0].obsn[0].torb) + cdef longdouble [:] _torb = &(self.psr[0].obsn[0].torb) _torb.strides[0] = sizeof(observation) return numpy.asarray(_torb).copy() @@ -2080,7 +2077,7 @@ cdef class tempopulsar: the `residuals` method. """ - cdef long double [:] _phase = &(self.psr[0].obsn[0].phase) + cdef longdouble [:] _phase = &(self.psr[0].obsn[0].phase) _phase.strides[0] = sizeof(observation) _ = self.residuals(**kwargs) From cc7e19f9a133b8997fbc3704b6eb8f79d8937f10 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Thu, 4 Dec 2025 22:45:06 +0000 Subject: [PATCH 21/22] Revert "Try plalform dependent longdouble" This reverts commit 21bf4ed249c0adf36f4de16489900c7330689b35. --- libstempo/libstempo.pyx | 129 ++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index 488c6e2..c9369ef 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -9,11 +9,16 @@ from collections import OrderedDict # what is the default encoding here? def string(buf): + # take bytes up to the first '\0' + #raw = bytes(buf).split(b'\0', 1)[0] try: return buf.decode('utf-8') except UnicodeDecodeError: - # just replace invalid characters - return buf.decode('utf-8', errors='replace') + try: + return buf.decode('latin-1') + except UnicodeDecodeError: + # just ignore invalid characters + return buf.decode('ascii', errors='ignore') string_dtype = 'U' @@ -77,8 +82,6 @@ try: except AttributeError: NP_LONG_DOUBLE_FORMAT = numpy.double -ctypedef numpy.longdouble_t longdouble - # return numpy array as astropy table with unit def dimensionfy(unit): def dimensionfy_decorator(func): @@ -96,16 +99,16 @@ cdef extern from "GWsim-stub.h": cdef bint HAVE_GWSIM ctypedef struct gwSrc: - longdouble theta_g - longdouble phi_g - longdouble omega_g - longdouble phi_polar_g + long double theta_g + long double phi_g + long double omega_g + long double phi_polar_g - void GWbackground(gwSrc *gw,int numberGW,long *idum,longdouble flo,longdouble fhi,double gwAmp,double alpha,int loglin) - void GWdipolebackground(gwSrc *gw,int numberGW,long *idum,longdouble flo,longdouble fhi, double gwAmp,double alpha,int loglin, double *dipoleamps) + void GWbackground(gwSrc *gw,int numberGW,long *idum,long double flo,long double fhi,double gwAmp,double alpha,int loglin) + void GWdipolebackground(gwSrc *gw,int numberGW,long *idum,long double flo,long double fhi, double gwAmp,double alpha,int loglin, double *dipoleamps) void setupGW(gwSrc *gw) - void setupPulsar_GWsim(longdouble ra_p,longdouble dec_p,longdouble *kp) - longdouble calculateResidualGW(longdouble *kp,gwSrc *gw,longdouble obstime,longdouble dist) + void setupPulsar_GWsim(long double ra_p,long double dec_p,long double *kp) + long double calculateResidualGW(long double *kp,gwSrc *gw,long double obstime,long double dist) cdef extern from "tempo2.h": enum: MAX_PSR_VAL @@ -146,28 +149,28 @@ cdef extern from "tempo2.h": ctypedef struct parameter: char **label char **shortlabel - longdouble *val - longdouble *err + long double *val + long double *err int *fitFlag int *paramSet - longdouble *prefit - longdouble *prefitErr + long double *prefit + long double *prefitErr int aSize ctypedef struct observation: - longdouble sat # site arrival time - longdouble origsat # Backup of SAT - longdouble sat_day # Just the Day part - longdouble sat_sec # Just the Sec part - longdouble bat # barycentric arrival time - longdouble batCorr #update from sat-> bat - longdouble bbat # barycentric arrival time - longdouble pet # pulsar emission time + long double sat # site arrival time + long double origsat # Backup of SAT + long double sat_day # Just the Day part + long double sat_sec # Just the Sec part + long double bat # barycentric arrival time + long double batCorr #update from sat-> bat + long double bbat # barycentric arrival time + long double pet # pulsar emission time int clockCorr # = 1 for clock corrections to be applied, = 0 for BAT int delayCorr # = 1 for time delay corrections to be applied, = 0 for BAT int deleted # 1 if observation deleted, -1 if not in fit - longdouble prefitResidual - longdouble residual + long double prefitResidual + long double residual double freq # frequency of observation (in MHz) double freqSSB # frequency of observation in barycentric frame (in Hz) double toaErr # error on TOA (in us) @@ -183,9 +186,9 @@ cdef extern from "tempo2.h": double psrPos[3] # Unit vector to the pulsar position double zenith[3] # Zenith vector, in BC frame. Length=geodetic height double shapiroDelaySun # Shapiro delay caused by the Sun - longdouble roemer # Roemer delay - longdouble torb # Combined binary delay - longdouble phase # the phase (cycles) + long double roemer # Roemer delay + long double torb # Combined binary delay + long double phase # the phase (cycles) long long pulseN # Pulse number char flagID[MAX_FLAGS][MAX_FLAG_LEN] # ID of flags char flagVal[MAX_FLAGS][MAX_FLAG_LEN] # Value of flags @@ -253,7 +256,7 @@ cdef extern from "tempo2.h": int useCalceph char tzrsite[100] int eclCoord # = 1 for ecliptic coords otherwise celestial coords - # longdouble phaseJump[MAX_JUMPS] # Time of phase jump (Deprecated. WHY?) + # long double phaseJump[MAX_JUMPS] # Time of phase jump (Deprecated. WHY?) int phaseJumpID[MAX_JUMPS] # ID of closest point to phase jump int phaseJumpDir[MAX_JUMPS] # Size and direction of phase jump int nPhaseJump # Number of phase jumps @@ -359,10 +362,10 @@ cdef extern from "t2fit-stub.h": void t2fit_fillOneParameterFitInfo(pulsar* psr,param_label fit_param,const int k,FitInfo& OUT) -cdef void set_longdouble_from_array(longdouble *p,numpy.ndarray[numpy.npy_longdouble,ndim=0] a): - p[0] = ((a.data))[0] +cdef void set_longdouble_from_array(long double *p,numpy.ndarray[numpy.npy_longdouble,ndim=0] a): + p[0] = ((a.data))[0] -cdef void set_longdouble(longdouble *p,object o): +cdef void set_longdouble(long double *p,object o): if isinstance(o,numpy.longdouble): set_longdouble_from_array(p,o[...]) elif isinstance(o,numpy.ndarray) and o.dtype == numpy.longdouble and o.ndim == 0: @@ -370,9 +373,9 @@ cdef void set_longdouble(longdouble *p,object o): else: p[0] = o -cdef object get_longdouble_as_scalar(longdouble v): +cdef object get_longdouble_as_scalar(long double v): cdef numpy.ndarray ret = numpy.array(0,dtype=numpy.longdouble) - (ret.data)[0] = v + (ret.data)[0] = v return ret.item() cdef class tempopar: @@ -403,7 +406,7 @@ cdef class tempopar: property val: def __get__(self): if not self._isjump and not self._isfdjump: - return self._unitify(get_longdouble_as_scalar((self._val)[0])) + return self._unitify(get_longdouble_as_scalar((self._val)[0])) else: return self._unitify(float((self._val)[0])) @@ -420,14 +423,14 @@ cdef class tempopar: if not self._paramSet[0]: self._paramSet[0] = 1 - set_longdouble(self._val,value) + set_longdouble(self._val,value) else: (self._val)[0] = value property err: def __get__(self): if not self._isjump and not self._isfdjump: - return self._unitify(get_longdouble_as_scalar((self._err)[0])) + return self._unitify(get_longdouble_as_scalar((self._err)[0])) else: return self._unitify(float((self._err)[0])) @@ -436,7 +439,7 @@ cdef class tempopar: value = value.to(self.unit).value if not self._isjump and not self._isfdjump: - set_longdouble(self._err,value) + set_longdouble(self._err,value) else: (self._err)[0] = value @@ -628,19 +631,19 @@ cdef class GWB: stdlib.free(self.gw) def gwb_sig(self,tempopulsar pulsar, distance=1): - cdef longdouble dist = distance * 3.086e19 + cdef long double dist = distance * 3.086e19 - cdef longdouble ra_p = pulsar.psr[0].param[param_raj].val[0] - cdef longdouble dec_p = pulsar.psr[0].param[param_decj].val[0] + cdef long double ra_p = pulsar.psr[0].param[param_raj].val[0] + cdef long double dec_p = pulsar.psr[0].param[param_decj].val[0] - cdef longdouble epoch = pulsar.psr[0].param[param_pepoch].val[0] + cdef long double epoch = pulsar.psr[0].param[param_pepoch].val[0] - cdef longdouble kp[3] + cdef long double kp[3] setupPulsar_GWsim(ra_p,dec_p,&kp[0]) - cdef numpy.ndarray[longdouble,ndim=1] res = numpy.zeros(pulsar.nobs,numpy.longdouble) - cdef longdouble obstime + cdef numpy.ndarray[long double,ndim=1] res = numpy.zeros(pulsar.nobs,numpy.longdouble) + cdef long double obstime for i in range(pulsar.nobs): obstime = (pulsar.psr[0].obsn[i].sat - epoch)*86400.0 @@ -1008,8 +1011,8 @@ cdef class tempopulsar: def _set_observation_from_input(self, toas): """Fill in all observation values from input TOAs.""" - cdef longdouble [:] _satday = &(self.psr[0].obsn[0].sat_day) - cdef longdouble [:] _satsec = &(self.psr[0].obsn[0].sat_sec) + cdef long double [:] _satday = &(self.psr[0].obsn[0].sat_day) + cdef long double [:] _satsec = &(self.psr[0].obsn[0].sat_sec) cdef int [:] _deleted = &(self.psr[0].obsn[0].deleted) cdef int [:] _nflags = &(self.psr[0].obsn[0].nFlags) cdef double [:] _phaseoff = &(self.psr[0].obsn[0].phaseOffset) @@ -1409,7 +1412,7 @@ cdef class tempopulsar: Return computed SSB TOAs in MJD as a numpy.longdouble array. You get a copy of the current tempo2 array.""" - cdef longdouble [:] _toas = &(self.psr[0].obsn[0].bat) + cdef long double [:] _toas = &(self.psr[0].obsn[0].bat) _toas.strides[0] = sizeof(observation) if updatebats: @@ -1424,7 +1427,7 @@ cdef class tempopulsar: You get a view of the original tempo2 data structure, which you can write to.""" def __get__(self): - cdef longdouble [:] _stoas = &(self.psr[0].obsn[0].sat) + cdef long double [:] _stoas = &(self.psr[0].obsn[0].sat) _stoas.strides[0] = sizeof(observation) return numpy.asarray(_stoas) @@ -1433,7 +1436,7 @@ cdef class tempopulsar: """Return Roemer delay in seconds as a numpy.longdouble array.""" def __get__(self): - cdef longdouble [:] _roemer = &(self.psr[0].obsn[0].roemer) + cdef long double [:] _roemer = &(self.psr[0].obsn[0].roemer) _roemer.strides[0] = sizeof(observation) return numpy.asarray(_roemer) @@ -1581,10 +1584,10 @@ cdef class tempopulsar: def origSats(self): """tempopulsar.origSats() - Return originally loaded value of the SAT in case it is updated afterwards. Returned as longdouble array. + Return originally loaded value of the SAT in case it is updated afterwards. Returned as long double array. You get a copy of the current values.""" - cdef longdouble [:] _origSats = &(self.psr[0].obsn[0].origsat) + cdef long double [:] _origSats = &(self.psr[0].obsn[0].origsat) _origSats.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_origSats),u.s) if self.units else numpy.asarray(_origSats) @@ -1594,10 +1597,10 @@ cdef class tempopulsar: def satDay(self): """tempopulsar.satDay() - Return the day part of the SAT as longdouble array. + Return the day part of the SAT as long double array. You get a copy of the current values.""" - cdef longdouble [:] _satDays = &(self.psr[0].obsn[0].sat_day) + cdef long double [:] _satDays = &(self.psr[0].obsn[0].sat_day) _satDays.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_satDays),u.s) if self.units else numpy.asarray(_satDays) @@ -1607,10 +1610,10 @@ cdef class tempopulsar: def satSec(self): """tempopulsar.satSec() - Return decimal part of the SAT as a longdouble array + Return decimal part of the SAT as a long double array You get a copy of the current values.""" - cdef longdouble [:] _satSecs = &(self.psr[0].obsn[0].sat_sec) + cdef long double [:] _satSecs = &(self.psr[0].obsn[0].sat_sec) _satSecs.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_satSecs),u.s) if self.units else numpy.asarray(_satSecs) @@ -1623,7 +1626,7 @@ cdef class tempopulsar: Return computed correction to SSB in units of days. You get a copy of the current values.""" - cdef longdouble [:] _batCorr = &(self.psr[0].obsn[0].batCorr) + cdef long double [:] _batCorr = &(self.psr[0].obsn[0].batCorr) _batCorr.strides[0] = sizeof(observation) return self._dimensionfy(numpy.asarray(_batCorr),u.s) if self.units else numpy.asarray(_batCorr) @@ -1688,7 +1691,7 @@ cdef class tempopulsar: Return computed pulsar emission times in MJD as a numpy.longdouble array. You get a copy of the current tempo2 array.""" - cdef longdouble [:] _pets = &(self.psr[0].obsn[0].pet) + cdef long double [:] _pets = &(self.psr[0].obsn[0].pet) _pets.strides[0] = sizeof(observation) if updatebats: @@ -1714,7 +1717,7 @@ cdef class tempopulsar: given, the `epoch`, `site` and `freq` values. """ - cdef longdouble [:] _res = &(self.psr[0].obsn[0].residual) + cdef long double [:] _res = &(self.psr[0].obsn[0].residual) _res.strides[0] = sizeof(observation) if removemean not in [True, False, 'weighted', 'first', 'refphs']: @@ -1835,7 +1838,7 @@ cdef class tempopulsar: cdef numpy.ndarray[double,ndim=2] ret = numpy.zeros((self.nobs,self.ndim+1),'d') - cdef longdouble epoch = self.psr[0].param[param_pepoch].val[0] + cdef long double epoch = self.psr[0].param[param_pepoch].val[0] cdef observation *obsns = self.psr[0].obsn if updatebats: @@ -1914,7 +1917,7 @@ cdef class tempopulsar: Does not reform residuals.""" # TODO: Is it not much faster to call DDmodel/XXmodel directly? - cdef longdouble [:] _torb = &(self.psr[0].obsn[0].torb) + cdef long double [:] _torb = &(self.psr[0].obsn[0].torb) _torb.strides[0] = sizeof(observation) return numpy.asarray(_torb).copy() @@ -2077,7 +2080,7 @@ cdef class tempopulsar: the `residuals` method. """ - cdef longdouble [:] _phase = &(self.psr[0].obsn[0].phase) + cdef long double [:] _phase = &(self.psr[0].obsn[0].phase) _phase.strides[0] = sizeof(observation) _ = self.residuals(**kwargs) From ff015549b161902a8042292e50e481ac10fd6f25 Mon Sep 17 00:00:00 2001 From: Matthew Pitkin Date: Thu, 4 Dec 2025 22:48:38 +0000 Subject: [PATCH 22/22] Simplify string function --- libstempo/libstempo.pyx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/libstempo/libstempo.pyx b/libstempo/libstempo.pyx index c9369ef..6b83478 100644 --- a/libstempo/libstempo.pyx +++ b/libstempo/libstempo.pyx @@ -9,16 +9,10 @@ from collections import OrderedDict # what is the default encoding here? def string(buf): - # take bytes up to the first '\0' - #raw = bytes(buf).split(b'\0', 1)[0] try: return buf.decode('utf-8') except UnicodeDecodeError: - try: - return buf.decode('latin-1') - except UnicodeDecodeError: - # just ignore invalid characters - return buf.decode('ascii', errors='ignore') + return buf.decode('utf-8', errors='replace') string_dtype = 'U' @@ -2237,8 +2231,6 @@ cdef class tempopulsar: parfile_bytes = (parfile + "\0").encode("utf-8", errors="ignore") - print(parfile_bytes) - cdef const char *parfile_c_bytes = parfile_bytes if len(parfile_bytes) > MAX_FILELEN: @@ -2273,8 +2265,6 @@ cdef class tempopulsar: timfile_bytes = (timfile + "\0").encode("utf-8", errors="ignore") - print(timfile_bytes) - cdef const char *timfile_c_bytes = timfile_bytes if len(timfile_bytes) > MAX_FILELEN: