diff --git a/.gitignore b/.gitignore index d50bf0c..9199fca 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,12 @@ __pycache__/ # Data files *.dat *.fits +*.fits.gz + +# Files downloades from Tammo-Jan gitlab +filterbank.py +sigproc.py +psr_constants.py + +# Data directory on Kapteyn +data diff --git a/examples/B1133+16_filterbank/B1133+16param.yml b/examples/B1133+16_filterbank/B1133+16param.yml new file mode 100644 index 0000000..7d6af1c --- /dev/null +++ b/examples/B1133+16_filterbank/B1133+16param.yml @@ -0,0 +1,17 @@ +# Information about the data file + +FileName: '2018-01-19-21:16:50_B1133+16.fil' + +# Folding parameters +Folding: + nbins: 500 # Number of bins for folding + nbinsdedisp: 500 # Number of bins for dedispersed profile + +# Output parameters +Output: + OutputDir: './output' + SavePlots: true + +ChiFit: false + +GrasMaaier: {} diff --git a/examples/B1133+16_filterbank/getData.sh b/examples/B1133+16_filterbank/getData.sh new file mode 100755 index 0000000..851ab3e --- /dev/null +++ b/examples/B1133+16_filterbank/getData.sh @@ -0,0 +1,6 @@ +#!/bin/bash +wget https://charon.camras.nl/public/pulsars/2018-01-19-21%3A16%3A50_B1133+16.txt + +wget https://charon.camras.nl/public/pulsars/2018-01-19-21%3A16%3A50_B1133+16.fil + + diff --git a/examples/B1133+16_filterbank/run.sh b/examples/B1133+16_filterbank/run.sh new file mode 100755 index 0000000..5f38d31 --- /dev/null +++ b/examples/B1133+16_filterbank/run.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Download the observation data if it is not present +if [ ! -e 2018-01-19-21%3A16%3A50_B1133+16.txt ] +then + echo "Dowloading the observation data" + ./getData.sh +fi + +# Run PSF +echo "Run PSF" +if command -v python3 &>/dev/null; then + python3 ../../main.py B1133+16param.yml +else + python ../../main.py B1133+16param.yml +fi diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..c6fd232 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,15 @@ +# Examples + +In this folder there are several examples that use the code on different datasets. If you want to run an example go to the folder of the example and run the examle by doing: + +`./run.sh` + +This will run the example, show the output and store the output in the output folder. + +## Type of examples + +Currently there are four types of examples: +1. The pulsars with normal fits data. +2. The pulsars with raw data. +3. The pulsars with filterbank files (not working). +4. The pulsars with a chifit (not working). diff --git a/examples/buurpulsar/buurpulsarParam.yml b/examples/buurpulsar/buurpulsarParam.yml new file mode 100644 index 0000000..8a1b194 --- /dev/null +++ b/examples/buurpulsar/buurpulsarParam.yml @@ -0,0 +1,17 @@ +# Information about the data file + +FileName: 'B2016+28_10-04-2018.fits.gz' + +# Folding parameters +Folding: + nbins: 500 # Number of bins for folding + nbinsdedisp: 500 # Number of bins for dedispersed profile + +# Output parameters +Output: + OutputDir: './output' + SavePlots: true + +ChiFit: false + +GrasMaaier: {} diff --git a/examples/buurpulsar/getData.sh b/examples/buurpulsar/getData.sh new file mode 100755 index 0000000..d68da7d --- /dev/null +++ b/examples/buurpulsar/getData.sh @@ -0,0 +1,4 @@ +#!/bin/bash +wget www.astro.rug.nl/~camras/data/CAMRAS/obs-10-04-2018/B2016+28_10-04-2018.fits.gz + + diff --git a/examples/buurpulsar/run.sh b/examples/buurpulsar/run.sh new file mode 100755 index 0000000..1d1452a --- /dev/null +++ b/examples/buurpulsar/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Download the observation data if it is not present +if [ ! -e B2016+28_10-04-2018.fits.gz ] +then + echo "Dowloading the observation data" + ./getData.sh +fi + +# Run PSF +echo "Run PSF" +if command -v python3 &>/dev/null; then + python3 ../../main.py buurpulsarParam.yml +else + python ../../main.py buurpulsarParam.yml +fi + diff --git a/examples/buurpulsar_raw/buurpulsarRawParam.yml b/examples/buurpulsar_raw/buurpulsarRawParam.yml new file mode 100644 index 0000000..9b31786 --- /dev/null +++ b/examples/buurpulsar_raw/buurpulsarRawParam.yml @@ -0,0 +1,23 @@ +# Information about the data file + +FileName: 'B2016-28_10-04-2018.dat' +RawData: true +FileFormat: 'raw' +ObsMetaData: + ObsDate: '2017-12-30T12:30:00(UTC)' + PulsarName: 'B2016+28' + MixFreq: 405 + +# Folding parameters +Folding: + nbins: 200 # Number of bins for folding + nbinsdedisp: 200 # Number of bins for dedispersed profile + +# Output parameters +Output: + OutputDir: './output' + SavePlots: true + +ChiFit: false + +GrasMaaier: {} diff --git a/examples/buurpulsar_raw/getData.sh b/examples/buurpulsar_raw/getData.sh new file mode 100755 index 0000000..53e4727 --- /dev/null +++ b/examples/buurpulsar_raw/getData.sh @@ -0,0 +1,2 @@ +#!/bin/bash +wget http://www.astro.rug.nl/~camras/data/obs-10-04-2018/B2016-28_10-04-2018.dat diff --git a/examples/buurpulsar_raw/run.sh b/examples/buurpulsar_raw/run.sh new file mode 100755 index 0000000..72a7e52 --- /dev/null +++ b/examples/buurpulsar_raw/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Download the observation data if it is not present +if [ ! -e B2016+28_406MHz_2017-12-30.raw ] +then + echo "Dowloading the observation data" + ./getData.sh +fi + +# Run PSF +echo "Run PSF" +if command -v python3 &>/dev/null; then + python3 ../../main.py buurpulsarRawParam.yml +else + python ../../main.py buurpulsarRawParam.yml +fi + diff --git a/examples/huispulsar/getData.sh b/examples/huispulsar/getData.sh new file mode 100755 index 0000000..bb1f15f --- /dev/null +++ b/examples/huispulsar/getData.sh @@ -0,0 +1,4 @@ +#!/bin/bash +wget www.astro.rug.nl/~camras/data/obs-10-04-2018/B0329+54_10-04-2018.fits.gz + + diff --git a/examples/huispulsar/huispulsarParam.yml b/examples/huispulsar/huispulsarParam.yml new file mode 100644 index 0000000..71a61a7 --- /dev/null +++ b/examples/huispulsar/huispulsarParam.yml @@ -0,0 +1,17 @@ +# Information about the data file + +FileName: 'B0329+54_10-04-2018.fits.gz' + +# Folding parameters +Folding: + nbins: 500 # Number of bins for folding + nbinsdedisp: 500 # Number of bins for dedispersed profile + +# Output parameters +Output: + OutputDir: './output' + SavePlots: true + +ChiFit: false + +GrasMaaier: {} diff --git a/examples/huispulsar/run.sh b/examples/huispulsar/run.sh new file mode 100755 index 0000000..6a5690f --- /dev/null +++ b/examples/huispulsar/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Download the observation data if it is not present +if [ ! -e B0329+54_10-04-2018.fits.gz ] +then + echo "Dowloading the observation data" + ./getData.sh +fi + +# Run PSF +echo "Run PSF" +if command -v python3 &>/dev/null; then + python3 ../../main.py huispulsarParam.yml +else + python ../../main.py huispulsarParam.yml +fi + diff --git a/examples/huispulsar_chifit/chifitparam.yml b/examples/huispulsar_chifit/chifitparam.yml new file mode 100644 index 0000000..8fca7e1 --- /dev/null +++ b/examples/huispulsar_chifit/chifitparam.yml @@ -0,0 +1,22 @@ +# Information about the data file + +FileName: 'B0329+54_10-04-2018.fits.gz' + +# Folding parameters +Folding: + nbins: 500 # Number of bins for folding + nbinsdedisp: 500 # Number of bins for dedispersed profile + +# Output parameters +Output: + OutputDir: './output' + SavePlots: true + +ChiFit: + Ntries: 1000 + nbins_time: 1000 + nbins_freq: 1000 + trywidth: 1e-3 + FitFunc: 'Lorentzian' + +GrasMaaier: {} diff --git a/examples/huispulsar_chifit/getData.sh b/examples/huispulsar_chifit/getData.sh new file mode 100755 index 0000000..6175e6c --- /dev/null +++ b/examples/huispulsar_chifit/getData.sh @@ -0,0 +1,4 @@ +#!/bin/bash +wget www.astro.rug.nl/~camras/data/obs-10-04-2018/B0329+54_10-04-2018.fits.gz + + diff --git a/examples/huispulsar_chifit/run.sh b/examples/huispulsar_chifit/run.sh new file mode 100755 index 0000000..a43074a --- /dev/null +++ b/examples/huispulsar_chifit/run.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Download the observation data if it is not present +if [ ! -e B0329+54_10-04-2018.fits.gz ] +then + echo "Dowloading the observation data" + ./getData.sh +fi + +# Run PSF +echo "Run PSF" +if command -v python3 &>/dev/null; then + python3 ../../main.py huispulsarParam.yml +else + python ../../main.py huispulsarParam.yml +fi diff --git a/examples/huispulsar_filterbank/getData.sh b/examples/huispulsar_filterbank/getData.sh new file mode 100755 index 0000000..590cbd7 --- /dev/null +++ b/examples/huispulsar_filterbank/getData.sh @@ -0,0 +1,5 @@ +#!/bin/bash +wget https://charon.camras.nl/public/pulsars/2017-09-20-07%3A05%3A00_B0329+54.txt + +wget https://charon.camras.nl/public/pulsars/2017-09-20-07%3A05%3A00_B0329+54.fil + diff --git a/examples/huispulsar_filterbank/huispulsarFilterbankParam.yml b/examples/huispulsar_filterbank/huispulsarFilterbankParam.yml new file mode 100644 index 0000000..6b7dc11 --- /dev/null +++ b/examples/huispulsar_filterbank/huispulsarFilterbankParam.yml @@ -0,0 +1,17 @@ +# Information about the data file + +FileName: '2017-09-20-07:05:00_B0329+54.fil' + +# Folding parameters +Folding: + nbins: 500 # Number of bins for folding + nbinsdedisp: 500 # Number of bins for dedispersed profile + +# Output parameters +Output: + OutputDir: './output' + SavePlots: true + +ChiFit: false + +GrasMaaier: {} diff --git a/examples/huispulsar_filterbank/run.sh b/examples/huispulsar_filterbank/run.sh new file mode 100755 index 0000000..249bfff --- /dev/null +++ b/examples/huispulsar_filterbank/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Download the observation data if it is not present +if [ ! -e 2017-09-20-07:05:00_B0329+54.fil ] +then + echo "Dowloading the observation data" + ./getData.sh +fi + +# Run PSF +echo "Run PSF" +if command -v python3 &>/dev/null; then + python3 ../../main.py huispulsarFilterbankParam.yml +else + python ../../main.py huispulsarFilterbankParam.yml +fi + diff --git a/examples/huispulsar_raw/getData.sh b/examples/huispulsar_raw/getData.sh new file mode 100755 index 0000000..ad8aca9 --- /dev/null +++ b/examples/huispulsar_raw/getData.sh @@ -0,0 +1,3 @@ +#!/bin/bash +wget -O B0329+54.2016.11.18.1038.5min.dat http://www.astro.rug.nl/~camras/data/obs-18-11-2016/B0329%2054.2016.11.18.1038.5min.dat + diff --git a/examples/huispulsar_raw/huispulsarParam.yml b/examples/huispulsar_raw/huispulsarParam.yml new file mode 100644 index 0000000..9e91834 --- /dev/null +++ b/examples/huispulsar_raw/huispulsarParam.yml @@ -0,0 +1,23 @@ +# Information about the data file + +FileName: 'B0329+54.2016.11.18.1038.5min.dat' +RawData: true +FileFormat: 'raw' +ObsMetaData: + ObsDate: '2016-11-18T10:38:00.00(UTC)' + PulsarName: 'B0329+54' + MixFreq: 405 + +# Folding parameters +Folding: + nbins: 500 # Number of bins for folding + nbinsdedisp: 500 # Number of bins for dedispersed profile + +# Output parameters +Output: + OutputDir: './output' + SavePlots: true + +ChiFit: false + +GrasMaaier: {} diff --git a/examples/huispulsar_raw/run.sh b/examples/huispulsar_raw/run.sh new file mode 100755 index 0000000..ac56e1f --- /dev/null +++ b/examples/huispulsar_raw/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Download the observation data if it is not present +if [ ! -e B0329+54.2016.11.18.1038.5min.dat ] +then + echo "Dowloading the observation data" + ./getData.sh +fi + +# Run PSF +echo "Run PSF" +if command -v python3 &>/dev/null; then + python3 ../../main.py huispulsarParam.yml +else + python ../../main.py huispulsarParam.yml +fi + diff --git a/param.yml b/examples/param.yml similarity index 100% rename from param.yml rename to examples/param.yml diff --git a/param_raw.yml b/examples/param_raw.yml similarity index 100% rename from param_raw.yml rename to examples/param_raw.yml diff --git a/filterBank.py b/filterBank.py new file mode 100755 index 0000000..74ef4d1 --- /dev/null +++ b/filterBank.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +from collections import defaultdict + +# https://stackoverflow.com/questions/28385822/reading-data-separated-by-colon-per-line-in-python +def filterBankReadMetaData(filename): + result = defaultdict(list) + with open(filename) as inf: + print(inf) + for line in inf: + name, score = line.split(": ", 1) + name = name.rstrip() + score = score.rstrip('\n') + result[name].append(score) + return result + + + + + + diff --git a/observation.py b/observation.py index 81452c2..9676df1 100644 --- a/observation.py +++ b/observation.py @@ -5,6 +5,9 @@ from astropy import units as u from barcen import barcen_times, barcen_freqs from loadData import loader +from pathlib import Path +from filterBank import filterBankReadMetaData +from blimpy import Waterfall def calc_central_freqs(mix_freq, throwhighestfreqaway=True): timevoltage = 1/(70e6) # (s), the time of each voltage measurement @@ -43,20 +46,31 @@ def __init__(self, cfg=None, fitsfile=None): self.psr_name = header['SRC_NAME'] self.obs_start_isot = header['DATE-OBS'] self.mix_freq = header['FREQMIX'] + elif fileformat.endswith('fil'): + self.using_fits = False + obs = Waterfall(filename) + header = obs.header + self.psr_name = header[b'source_name'] + self.obs_start_isot = header[b'tstart'] + # Tammo-Jan doesn't store the mix frequency + self.mix_freq = header[b'fch1']-21.668359375 + self.data = obs.data + #raise NotImplementedError('Filterbank supported yet!') else: self.using_fits = False self.psr_name = cfg.ObsMetaData.PulsarName self.obs_start_isot = cfg.ObsMetaData.ObsDate self.mix_freq = cfg.ObsMetaData.MixFreq - if fileformat.endswith('fil'): - raise NotImplementedError('Filterbank supported yet!') - elif fileformat.endswith('raw'): + if fileformat.endswith('raw'): assert cfg.RawData, 'Need to supply the metadata!' self.data = loader(cfg.FileName).astype(np.uint32) else: raise ValueError('Unrecognized format') - self.obs_start = Time(self.obs_start_isot) + if fileformat.endswith('fil'): + self.obs_start = Time(self.obs_start_isot,format='jd') + else: + self.obs_start = Time(self.obs_start_isot) dt = 64*512/(70e6) self.obs_dur = len(self.data) * dt self.obs_end = self.obs_start + self.obs_dur * u.s diff --git a/paramObj.py b/paramObj.py index 215dad1..a432ace 100644 --- a/paramObj.py +++ b/paramObj.py @@ -2,6 +2,7 @@ import numpy as np from ruamel.yaml import YAML from copy import deepcopy +from pathlib import Path # https://stackoverflow.com/questions/36831998/how-to-fill-default-parameters-in-yaml-file-using-python def setdefault_recursively(tgt, default): @@ -47,12 +48,12 @@ class Config(AttrDict): # These values have been replaced with defaults if the user did not input them # The actual user input is still available from param.usercfg. """ - def __init__(self,paramyml, defaults='defaults.yml'): + def __init__(self,paramyml, defaults=Path(__file__).parent/'defaults.yml'): yaml=YAML(typ='safe') with open(paramyml,'r') as ymlfile: cfg = yaml.load(ymlfile) - with open('defaults.yml','r') as ymlfile: + with open(Path(__file__).parent/'defaults.yml','r') as ymlfile: defaults = yaml.load(ymlfile) usercfg = deepcopy(cfg) diff --git a/pulsarsObjects.py b/pulsarsObjects.py index f937bd0..09575a9 100755 --- a/pulsarsObjects.py +++ b/pulsarsObjects.py @@ -3,9 +3,10 @@ from astropy.time import Time from astropy import units as u import barcen +from pathlib import Path # Loading the data -def load_pulsar_data(pulsar_name, pulsarcat_file='small-data-files/pulsarcat.csv'): +def load_pulsar_data(pulsar_name, pulsarcat_file=Path(__file__).parent/'small-data-files/pulsarcat.csv'): # read the data of the pulsar database pulsardata = np.genfromtxt(pulsarcat_file, delimiter=",", dtype=None, names=True, missing_values="*") # Find the index of the correct pulsar