Initial commit

.gitignore

+*.dat
+*.fil
+*.pyc

README.md

+This contains some scripts to add metadata to the camras pulsar format, i.e. the format that comes out of `pulsar_record`.

psr_constants.py

+## Automatically adapted for numpy Apr 14, 2006 by convertcode.py
+
+ARCSECTORAD = float('4.8481368110953599358991410235794797595635330237270e-6')
+RADTOARCSEC = float('206264.80624709635515647335733077861319665970087963')
+SECTORAD    = float('7.2722052166430399038487115353692196393452995355905e-5')
+RADTOSEC    = float('13750.987083139757010431557155385240879777313391975')
+RADTODEG    = float('57.295779513082320876798154814105170332405472466564')
+DEGTORAD    = float('1.7453292519943295769236907684886127134428718885417e-2')
+RADTOHRS    = float('3.8197186342054880584532103209403446888270314977710')
+HRSTORAD    = float('2.6179938779914943653855361527329190701643078328126e-1')
+PI          = float('3.1415926535897932384626433832795028841971693993751')
+TWOPI       = float('6.2831853071795864769252867665590057683943387987502')
+PIBYTWO     = float('1.5707963267948966192313216916397514420985846996876')
+SECPERDAY   = float('86400.0')
+SECPERJULYR = float('31557600.0')
+KMPERPC     = float('3.0856776e13')
+KMPERKPC    = float('3.0856776e16')
+Tsun        = float('4.925490947e-6') # sec
+Msun        = float('1.9891e30')      # kg
+Mjup        = float('1.8987e27')      # kg
+Rsun        = float('6.9551e8')       # m
+Rearth      = float('6.378e6')        # m
+SOL         = float('299792458.0')    # m/s
+MSUN        = float('1.989e+30')      # kg
+G           = float('6.673e-11')      # m^3/s^2/kg 
+C           = SOL

sigproc.py

+#!/usr/bin/env python
+import os
+import struct
+import sys
+import math
+import warnings
+from psr_constants import ARCSECTORAD
+
+telescope_ids = {"Fake": 0, "Arecibo": 1, "ARECIBO 305m": 1, 
+                 "Ooty": 2, "Nancay": 3,
+                 "Parkes": 4, "Jodrell": 5, "GBT": 6, "GMRT": 7,
+                 "Effelsberg": 8, "ATA": 9, "UTR-2": 10, "LOFAR": 11}
+ids_to_telescope = dict(list(zip(list(telescope_ids.values()), list(telescope_ids.keys()))))
+
+machine_ids = {"FAKE": 0, "PSPM": 1, "Wapp": 2, "WAPP": 2, "AOFTM": 3,
+               "BCPM1": 4, "BPP": 4, "OOTY": 5, "SCAMP": 6, 
+               "GBT Pulsar Spigot": 7, 
+               "SPIGOT": 7, "BG/P": 11, "PDEV": 12}
+ids_to_machine = dict(list(zip(list(machine_ids.values()), list(machine_ids.keys()))))
+
+header_params = {
+    "HEADER_START": 'flag',
+    "telescope_id": 'i',
+    "machine_id": 'i',
+    "data_type": 'i', 
+    "rawdatafile": 'str',
+    "source_name": 'str', 
+    "barycentric": 'i', 
+    "pulsarcentric": 'i', 
+    "az_start": 'd',  
+    "za_start": 'd',  
+    "src_raj": 'd',  
+    "src_dej": 'd',  
+    "tstart": 'd',  
+    "tsamp": 'd',  
+    "nbits": 'i', 
+    "nsamples": 'i', 
+    "nbeams": "i",
+    "ibeam": "i",
+    "fch1": 'd',  
+    "foff": 'd',
+    "FREQUENCY_START": 'flag',
+    "fchannel": 'd',  
+    "FREQUENCY_END": 'flag',
+    "nchans": 'i', 
+    "nifs": 'i', 
+    "refdm": 'd',  
+    "period": 'd',  
+    "npuls": 'q',
+    "nbins": 'i', 
+    "HEADER_END": 'flag'}
+
+def dec2radians(src_dej):
+    """
+    dec2radians(src_dej):
+       Convert the SIGPROC-style DDMMSS.SSSS declination to radians
+    """
+    sign = 1.0
+    if (src_dej < 0): sign = -1.0;
+    xx = math.fabs(src_dej)
+    dd = int(math.floor(xx / 10000.0))
+    mm = int(math.floor((xx - dd * 10000.0) / 100.0))
+    ss = xx - dd * 10000.0 - mm * 100.0
+    return sign * ARCSECTORAD * (60.0 * (60.0 * dd + mm) + ss)
+
+def ra2radians(src_raj):
+    """
+    ra2radians(src_raj):
+       Convert the SIGPROC-style HHMMSS.SSSS right ascension to radians
+    """
+    return 15.0 * dec2radians(src_raj)
+
+def read_doubleval(filfile, stdout=False):
+    dblval = struct.unpack('d', filfile.read(8))[0]
+    if stdout:
+        print("  double value = '%20.15f'"%dblval)
+    return dblval
+
+def read_intval(filfile, stdout=False):
+    intval = struct.unpack('i', filfile.read(4))[0]
+    if stdout:
+        print("  int value = '%d'"%intval)
+    return intval
+
+def read_longintval(filfile, stdout=False):
+    longintval = struct.unpack('q', filfile.read(8))[0]
+    if stdout:
+        print("  long int value = '%d'"%longintval)
+    return longintval
+
+def read_string(filfile, stdout=False):
+    strlen = struct.unpack('i', filfile.read(4))[0]
+    strval = filfile.read(strlen)
+    if stdout:
+        print("  string = '%s'"%strval)
+    return strval
+
+def read_paramname(filfile, stdout=False):
+    paramname = read_string(filfile, stdout=False)
+    if stdout:
+        print("Read '%s'"%paramname)
+    return paramname
+
+def read_hdr_val(filfile, stdout=False):
+    paramname = read_paramname(filfile, stdout)
+    if header_params[paramname] == 'd':
+        return paramname, read_doubleval(filfile, stdout)
+    elif header_params[paramname] == 'i':
+        return paramname, read_intval(filfile, stdout)
+    elif header_params[paramname] == 'q':
+        return paramname, read_longintval(filfile, stdout)
+    elif header_params[paramname] == 'str':
+        return paramname, read_string(filfile, stdout)
+    elif header_params[paramname] == 'flag':
+        return paramname, None
+    else:
+        print("Warning:  key '%s' is unknown!" % paramname)
+        return None, None
+
+def prep_string(string):
+    return struct.pack('i', len(string))+string
+
+def prep_double(name, value):
+    return prep_string(name)+struct.pack('d', float(value))
+
+def prep_int(name, value):
+    return prep_string(name)+struct.pack('i', int(value))
+
+def addto_hdr(paramname, value):
+    if header_params[paramname] == 'd':
+        return prep_double(paramname, value)
+    elif header_params[paramname] == 'i':
+        return prep_int(paramname, value)
+    elif header_params[paramname] == 'str':
+        return prep_string(paramname) + prep_string(value)
+    elif header_params[paramname] == 'flag':
+        return prep_string(paramname)
+    else:
+        warnings.warning("key '%s' is unknown!" % paramname)
+    return hdr
+
+def read_header(infile):
+    """
+    read_header(infile):
+       Read a SIGPROC-style header and return the keys/values in a dictionary,
+          as well as the length of the header: (hdrdict, hdrlen)
+    """
+    hdrdict = {}
+    if type(infile) == type("abc"):
+        infile = open(infile)
+    param = ""
+    while (param != "HEADER_END"):
+        param, val = read_hdr_val(infile, stdout=False)
+        hdrdict[param] = val
+    hdrlen = infile.tell()
+    infile.close()
+    return hdrdict, hdrlen
+
+def samples_per_file(infile, hdrdict, hdrlen):
+    """
+    samples_per_file(infile, hdrdict, hdrlen):
+       Given an input SIGPROC-style filterbank file and a header
+           dictionary and length (as returned by read_header()),
+           return the number of (time-domain) samples in the file.
+    """
+    numbytes = os.stat(infile)[6] - hdrlen
+    bytes_per_sample = hdrdict['nchans'] * (hdrdict['nbits']/8)
+    if numbytes % bytes_per_sample:
+        print("Warning!:  File does not appear to be of the correct length!")
+    numsamples = numbytes / bytes_per_sample
+    return numsamples
+
+if __name__ == "__main__":
+    if len(sys.argv)==1:
+        print("\nusage:  mod_filterbank_hdr.py infile.fil [outfile.fil]\n")
+        sys.exit()
+    filhdr = {}
+    newhdr = ""
+    infile = open(sys.argv[1], 'rb')
+
+    # Loop over the values in the .fil file
+    while 1:
+        param, val = read_hdr_val(infile, stdout=False)
+        filhdr[param] = val
+
+        # Add lines here to correct stuff
+        #if param=="nchans":  val = 768
+
+        # Append to the new hdr string
+        newhdr += addto_hdr(param, val)
+
+        # Break out of the loop if the header is over
+        if param=="HEADER_END":  break
+            
+    if len(sys.argv) > 2:
+        print("Writing new header to '%s'"%sys.argv[2])
+        outfile = open(sys.argv[2], 'wb')
+        outfile.write(newhdr)
+        outfile.close()
+    else:
+        print(filhdr)

tofil.py

+#!/usr/bin/env python2
+from __future__ import print_function
+
+import numpy as np
+import filterbank
+import sigproc
+from astropy.time import Time
+
+import os.path
+import sys
+import re
+
+import argparse
+
+def create_filterbank(infile, outfile=None, date=None, source=None):
+    """
+    Create a filterbank file from the CAMRAS-format, which is generated
+    by pulsar_record.
+
+    infile:   name of the file generated by pulsar_record
+    outfile: name of the filterbank file to be created; if not given, a name
+              will be created from the infile
+    date:     the date to be used; default is the file creation date of infile
+              if given, the date must be in a format the astropy.time understands
+    source:   name of the source to be used; default is guessed from filename
+    """
+
+    if not outfile:
+        outfile = re.split('[+-\.]', infile)[0] + ".fil"
+
+    if not source:
+        match = re.match("[BJ][0-9]*[+-][0-9]*", infile)
+        if match:
+            source = "PSR " + match.group(0)
+        else:
+            raise ValueError("Could not guess source name from filename")
+
+    if not date:
+        date = Time(os.path.getctime(infile), format='unix')
+    else:
+        date = Time(date, format='isot')
+
+    print("Creating filterbank file {} from {}".format(outfile, infile))
+    print("Source name: {}".format(source))
+    print("Observation time: {}".format(date))
+
+    date = date.mjd
+ 
+    # Generate filterbank header
+    fil_header = {}
+    fil_header["telescope_id"]  = sigproc.telescope_ids["Effelsberg"]
+    fil_header["machine_id"]    = sigproc.machine_ids["FAKE"]
+    fil_header["data_type"]     = 1
+    fil_header["rawdatafile"]   = infile
+    fil_header["source_name"]   = source
+    fil_header["barycentric"]   = 0
+    fil_header["pulsarcentric"] = 0
+    fil_header["az_start"]      = 0.0
+    fil_header["za_start"]      = 0.0
+    fil_header["src_raj"]       = 0.0
+    fil_header["src_dej"]       = 0.0
+    fil_header["tstart"]        = date
+    fil_header["tsamp"]         = 0.00046811428571414528 # 1.0/35e6*256 chans*64 decimation
+    fil_header["nbits"]         = 32
+    fil_header["fch1"]          = 441.4
+    fil_header["foff"]          = -35.0/256.0
+    fil_header["nchans"]        = 256
+    fil_header["nifs"]          = 1
+
+    # Write header
+    out = filterbank.create_filterbank_file(outfile, fil_header, nbits=32)
+
+    # Read file
+    data = np.flipud(np.fromfile(infile, dtype='>u4'))
+    print(data.shape)
+
+    data = np.flipud(data.reshape(-1, 256))
+
+    # Check that no packets were dropped
+    if np.sum(data[:-1,-1]-data[1:,-1]-1) > 0.1:
+        print("Packets were dropped:", np.count_nonzero(data[:-1,0]-data[1:,0]-1))
+
+    # Write data
+    out.append_spectra(data)
+
+    # Close file
+    out.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Convert a camras pulsar file (output of pulsar_record) to filterbank format")
+    parser.add_argument("infile", help="Input CAMRAS pulsar file")
+    parser.add_argument("-o", "--outfile", help="Name of output file, e.g. 'B0329.fil' (default: guessed from input filename)", default=None)
+    parser.add_argument("-d", "--date", help="Start date/time of observation, in 'isot' format (defaults to creation date of file)", default=None)
+    parser.add_argument("-s", "--source", help="Name of the source, e.g. 'PSR B0329+54' (default: guessed from input filename)", default=None)
+
+    args = parser.parse_args()
+
+    create_filterbank(args.infile, outfile=args.outfile, date=args.date, source=args.source)