Commit 743d1952 authored by Tammo Jan Dijkema's avatar Tammo Jan Dijkema

Chunk data to save memory; don't flip twice

parent 41fed2ad
......@@ -7,6 +7,7 @@ import sigproc
from astropy.time import Time
import os.path
import os
import sys
import re
......@@ -42,7 +43,7 @@ def create_filterbank(infile, outfile=None, date=None, source=None):
print("Creating filterbank file {} from {}".format(outfile, infile))
print("Source name: {}".format(source))
print("Observation time: {}".format(date))
print("Observation time: {}".format(date.isot))
date = date.mjd
......@@ -71,17 +72,26 @@ def create_filterbank(infile, outfile=None, date=None, source=None):
out = filterbank.create_filterbank_file(outfile, fil_header, nbits=32)
# Read file
data = np.flipud(np.fromfile(infile, dtype='>u4'))
print(data.shape)
f = open(infile)
data = np.flipud(data.reshape(-1, 256))
filesize = os.stat(infile).st_size
print("File size:", filesize)
offset = 0
while offset < filesize and False:
# Loop through file in chunks of 256*10000 numbers
# i.e. 256*10000*4 bytes (10Mb) to avoid memory problems
f.seek(offset, os.SEEK_SET)
# Check that no packets were dropped
if np.sum(data[:-1,-1]-data[1:,-1]-1) > 0.1:
print("Packets were dropped:", np.count_nonzero(data[:-1,0]-data[1:,0]-1))
data = np.fromfile(f, dtype='>u4', count=256*100000).reshape(-1,256)
# Write data
out.append_spectra(data)
# Check that no packets were dropped
if np.sum(data[1:,0]-data[:-1,0]-1) > 0.1:
print("Packets were dropped:", np.count_nonzero(data[:-1,0]-data[1:,0]-1), np.sum(data[1:,0]-data[:-1,0]-1))
# Write data
out.append_spectra(data)
offset += 256*10000*4
# Close file
out.close()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment