Chunk data to save memory; don't flip twice

tofil.py

@@ -7,6 +7,7 @@ import sigproc
 from astropy.time import Time
 
 import os.path
+import os
 import sys
 import re
 
@@ -42,7 +43,7 @@ def create_filterbank(infile, outfile=None, date=None, source=None):
 
     print("Creating filterbank file {} from {}".format(outfile, infile))
     print("Source name: {}".format(source))
-    print("Observation time: {}".format(date))
+    print("Observation time: {}".format(date.isot))
 
     date = date.mjd
  
@@ -71,17 +72,26 @@ def create_filterbank(infile, outfile=None, date=None, source=None):
     out = filterbank.create_filterbank_file(outfile, fil_header, nbits=32)
 
     # Read file
-    data = np.flipud(np.fromfile(infile, dtype='>u4'))
-    print(data.shape)
+    f = open(infile)
 
-    data = np.flipud(data.reshape(-1, 256))
+    filesize = os.stat(infile).st_size
+    print("File size:", filesize)
+    offset = 0
+    while offset < filesize and False:
+        # Loop through file in chunks of 256*10000 numbers
+        # i.e. 256*10000*4 bytes (10Mb) to avoid memory problems
+        f.seek(offset, os.SEEK_SET)
 
-    # Check that no packets were dropped
-    if np.sum(data[:-1,-1]-data[1:,-1]-1) > 0.1:
-        print("Packets were dropped:", np.count_nonzero(data[:-1,0]-data[1:,0]-1))
+        data = np.fromfile(f, dtype='>u4', count=256*100000).reshape(-1,256)
 
-    # Write data
-    out.append_spectra(data)
+        # Check that no packets were dropped
+        if np.sum(data[1:,0]-data[:-1,0]-1) > 0.1:
+            print("Packets were dropped:", np.count_nonzero(data[:-1,0]-data[1:,0]-1), np.sum(data[1:,0]-data[:-1,0]-1))
+
+        # Write data
+        out.append_spectra(data)
+
+        offset += 256*10000*4
 
     # Close file
     out.close()