LNC_tools.py

def set_dir(titlestring):
    from Tkinter import Tk
    import tkFileDialog
     
     
    master = Tk()
    master.withdraw() #hiding tkinter window
     
    file_path = tkFileDialog.askdirectory(title=titlestring)
     
    if file_path != "":
       return str(file_path)
     
    else:
       print "you didn't open anything!"
     
def get_files(titlestring,filetype = ('.txt','*.txt')):
    from Tkinter import Tk
    import tkFileDialog
     
     
    master = Tk()
    master.withdraw() #hiding tkinter window

    file_path = []
     
    file_path = tkFileDialog.askopenfilename(title=titlestring, filetypes=[filetype,("All files",".*")],multiple='True')
     
    if file_path != "":
       return str(file_path)
     
    else:
       print "you didn't open anything!"

def lnc_reader(filepath):
    #---------------------------------------------------------------------------
    #This program opens an ASCII file generated by the LNC program and outputs a
    #Pandas timeseries dataframe with the data and relevant metadata
    #Note: Input file must be either a 1-D temporal average or a
    #"Single Table with Time in the X Axis vs Altitude in the Y Axis"
    #NOT A "Series of consecutively listed profiles"
    #---------------------------------------------------------------------------
    import pandas as pan
    import numpy as np
    import re
    import csv
    from dateutil.parser import parse

    product = []

    #generate a new filename with _proc attached to the end and open both files
    [fname,fext] = filepath.split('.')
    print 'Pre-Processing '+fname
    fout_name = fname+'_proc'
    fout_name = fout_name+'.'+fext
    fin = open(filepath, 'rb')
    fout = open(fout_name, 'w')

    #copy data table to new *_proc file line by line and replace all spaces
    #between data columns with a comma, empty lines have len(2) & are skipped
    #first line is the product designation, preserved as metadata
    bigspace = re.compile('\s\s\s+')
    for line in fin:
        if not product:
            product.append(line)
        elif len(line) == 2:
            continue  
        else:
            line = bigspace.sub(',',line)+'\n'
            fout.write(line)

    #close both files        
    fin.close()
    fout.close()

    #use csv.reader to read processed file into a list of lists
    temp = []
    for row in csv.reader(open(fout_name,'rb'), delimiter=','):
        temp.append(row)

    #convert to numpy array and transpose list to put datetime entries in first
    #column, which will facilitate conversion to pandas timeseries dataframe
    temparray = np.array(temp).T

    #generate pandas dataframe index by parsing strings into datetime objects
    #note: first entry is the word 'Altitude', last entry is an empty space
    indexdat = []
    for i in temparray[1:-1,0]: indexdat.append(parse(i)) 
    index = pan.Index(indexdat,name = 'Date Time')
    #generate column headers from altitudes (not including the word 'Altitude'
    coldat = np.array(temparray[0,1:],dtype='float')
    columns = pan.Index(coldat,name = temparray[0,0])
    #data for dataframe consists of remaining rows and columns
    data = temparray[1:-1,1:]

    #check data for flags indicating bad results and substitute with NaN

    flags = ['-1.#INF','1.#INF','-1.#IND','1.#IND']

    clean_data = np.copy(data)

    for f in flags: clean_data[data == f] = 'NaN'

    #convert data to pandas dataframe
    df = pan.DataFrame(clean_data,index=index,columns=columns,dtype='float')

    return df, product     

def alt_resample(df, altrange):
    #takes a pandas dataframe generated by lnc and resamples on regular
    #intervals in altitude and resets the limits of the set
    #note: limits of altrange must be within original limits of altitude data
    
    import numpy as np
    import pandas as pan       
    from scipy.interpolate import interp1d
    print 'Altitude step resampling in progress ...'
    
    x = df.columns
    
    numrows = np.size(df.index)
    numcols = np.size(altrange)

    newvalues = np.empty([numrows, numcols])
    n = 0

    for row in df.iterrows():
        f = interp1d(x,row[1].values)
        newvalues[n] = f(altrange)
        n += 1

    dfout = pan.DataFrame(data = newvalues, index = df.index,
                          columns = altrange)
    print '... Done!'
    return dfout

def time_resample(df, timestep, timerange = False, s_mode = 'mean'):
    #resamples a pandas dataframe generated by lnc_reader on a regular timestep
    #and optionally limits it to a preset time range
    #timestep must be in timeseries period format: numF where num=step size and
    #F = offset alias.  Ex: H = hours, M = minutes, S = seconds, L = millieconds
    print 'Time step regularization in progress ...'
    if timerange:
        start_time = timerange[0]
        end_time = timerange[1]

        dfout = df[(df.index>=start_time) & (df.index<=end_time)]

    dfout = dfout.resample(timestep, how = s_mode)

    print '... Done!'
    return dfout
    
    

def BR_mask(backscatter, data, delta):
    #this function takes a pandas timeseries dataframe representing a table of
    #backscatter ratios and produces a masking dataframe with values of 0 where
    #ratio is identically 1, and 1 elsewhere then applies it to data
    print 'masking data'
    mask = backscatter.applymap(lambda x: not x <= 1+delta)

    masked_data = mask*data
    
    print 'Done!'

    return masked_data

if __name__=='__main__':
    import pandas as pan
    import numpy as np
    import datetime as dt

    delta = 0.1

    BR_filepath = 'C:\Users\dashamstyr\Documents\CORALNet\ASCII Files\UBC_July_2012\UBC_07062012_BR1064.txt'
    data_filepath = 'C:\Users\dashamstyr\Documents\CORALNet\ASCII Files\UBC_July_2012\UBC_07062012_PR532.txt'
    maskout, maskprod = lnc_reader(BR_filepath)
    ar = np.arange(10,18000,100,dtype='float')
    dataout, dataprod = lnc_reader(data_filepath)
    dfmasked = BR_mask(maskout, dataout, delta)