LNCcode/LNC_fileproc.py at master · nchaparr/LNCcode · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
import pandas as pan
import numpy as np
import os,sys
import LNC_tools as LNC

#----------------------------------------------------------------------------
#Uses tools created in LNC_tools to open all files in a folder and resample
#them to a regular spacing in altitude/date the concatenates them into one
#pandas dataframe and plots it using LNC_plot
#July 05, 2012
#----------------------------------------------------------------------------

olddir = os.getcwd()

#os.chdir('K:\CORALNet\Data\ASCII Files')

newdir = LNC.set_dir('Select Event Folder')

os.chdir(newdir)

files = os.listdir(newdir)
maskfiles = []
datafiles = []
procfiles = []
rawfiles = []

#set altitude range and date step sizes

altrange = np.arange(10,10010,10)#meters
timestep = '120S' #seconds

#set buffer around backscatter ratio of 1 for mask

delta = 0.1

#check to see if each file has been processed before and separate processed
#files into a new list

for f in files:
    if '_proc' in f or '.pickle' in f:
        procfiles.append(f)
    elif '.txt' in f:
        rawfiles.append(f)

#search through list of files to separate fields to be used as a mask from those
#with data to be plotted
#initially, mask files are designated BR1064 for 1064nm Backscatter Ratio

for f in rawfiles:
    if 'BR' in f:
        maskfiles.append(f)
    else:
        datafiles.append(f)

#make sure the files are in a common order of ascending date (assuming they're all
#from the same station
maskfiles.sort()
datafiles.sort()

#first check to make sure the same number of files in each list

if len(maskfiles) != len(datafiles):
    sys.exit("Error: Mask files don't match data files")


#double check to make sure the mask files match up with the data files
for d,m in zip(datafiles, maskfiles):
    [d_stat,d_date,d_type] = d.split('_')
    [m_stat,m_date,m_type] = m.split('_')
    print 'Checking mask/data match for %s'%(d_date)
    if d_date == m_date and d_stat == m_stat:
        print 'Check!'
        continue
    else:
        sys.exit("Error: Mask files don't match data files")

#open, altitude resample, and concatenate data and mask files

for d,m in zip(datafiles, maskfiles):
    d_temp, data_prod = LNC.lnc_reader(d)
    d_realt = LNC.alt_resample(d_temp,altrange)

    try:
        d_event = pan.concat([d_event,d_realt])
    except NameError:
        d_event = d_realt

    m_temp, data_prod = LNC.lnc_reader(m)
    m_realt = LNC.alt_resample(m_temp,altrange)

    try:
        m_event = pan.concat([m_event,m_realt])
    except NameError:
        m_event = m_realt


#sort by index to make certain data is in order then set date ranges to match

d_event = d_event.sort_index()
m_event = m_event.sort_index()

start = m_event.index[0]
end = m_event.index[-1]

d_event = LNC.time_resample(d_event,timestep, timerange = [start,end])
m_event = LNC.time_resample(m_event,timestep,timerange = [start,end])


dfmask = LNC.BR_mask(m_event,d_event, delta)

d_filename = datafiles[0].split('.')[0]+'-'+datafiles[-1].split('.')[0]
d_event.save(d_filename+'.pickle')

m_filename = maskfiles[0].split('.')[0]+'-'+maskfiles[-1].split('.')[0]
m_event.save(m_filename+'.pickle')

dfmask.save(d_filename+'_masked.pickle')