Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Tammo Jan Dijkema
filterbank
Commits
743d1952
Commit
743d1952
authored
Oct 12, 2017
by
Tammo Jan Dijkema
Browse files
Chunk data to save memory; don't flip twice
parent
41fed2ad
Changes
1
Hide whitespace changes
Inline
Side-by-side
tofil.py
View file @
743d1952
...
...
@@ -7,6 +7,7 @@ import sigproc
from
astropy.time
import
Time
import
os.path
import
os
import
sys
import
re
...
...
@@ -42,7 +43,7 @@ def create_filterbank(infile, outfile=None, date=None, source=None):
print
(
"Creating filterbank file {} from {}"
.
format
(
outfile
,
infile
))
print
(
"Source name: {}"
.
format
(
source
))
print
(
"Observation time: {}"
.
format
(
date
))
print
(
"Observation time: {}"
.
format
(
date
.
isot
))
date
=
date
.
mjd
...
...
@@ -71,17 +72,26 @@ def create_filterbank(infile, outfile=None, date=None, source=None):
out
=
filterbank
.
create_filterbank_file
(
outfile
,
fil_header
,
nbits
=
32
)
# Read file
data
=
np
.
flipud
(
np
.
fromfile
(
infile
,
dtype
=
'>u4'
))
print
(
data
.
shape
)
f
=
open
(
infile
)
data
=
np
.
flipud
(
data
.
reshape
(
-
1
,
256
))
filesize
=
os
.
stat
(
infile
).
st_size
print
(
"File size:"
,
filesize
)
offset
=
0
while
offset
<
filesize
and
False
:
# Loop through file in chunks of 256*10000 numbers
# i.e. 256*10000*4 bytes (10Mb) to avoid memory problems
f
.
seek
(
offset
,
os
.
SEEK_SET
)
# Check that no packets were dropped
if
np
.
sum
(
data
[:
-
1
,
-
1
]
-
data
[
1
:,
-
1
]
-
1
)
>
0.1
:
print
(
"Packets were dropped:"
,
np
.
count_nonzero
(
data
[:
-
1
,
0
]
-
data
[
1
:,
0
]
-
1
))
data
=
np
.
fromfile
(
f
,
dtype
=
'>u4'
,
count
=
256
*
100000
).
reshape
(
-
1
,
256
)
# Write data
out
.
append_spectra
(
data
)
# Check that no packets were dropped
if
np
.
sum
(
data
[
1
:,
0
]
-
data
[:
-
1
,
0
]
-
1
)
>
0.1
:
print
(
"Packets were dropped:"
,
np
.
count_nonzero
(
data
[:
-
1
,
0
]
-
data
[
1
:,
0
]
-
1
),
np
.
sum
(
data
[
1
:,
0
]
-
data
[:
-
1
,
0
]
-
1
))
# Write data
out
.
append_spectra
(
data
)
offset
+=
256
*
10000
*
4
# Close file
out
.
close
()
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment