data-gizmo.py

#!/usr/bin/env python3

from tkinter import Tk, Label, Button, StringVar, Entry,NONE, END,HORIZONTAL,N, W, E, S, Checkbutton,Radiobutton, IntVar, Radiobutton, Scrollbar, Listbox, LEFT, BOTH, Spinbox, Menu, Text, NORMAL
import tkinter as tk
from tkinter import ttk
#For more themes
#from ttkthemes import ThemedTk
#import math
# to create a dialog interface for the input file
from tkinter.filedialog import askopenfilename
#from tkinter.filedialog import asksavefilename
from tkinter.filedialog import asksaveasfile
# to create a dialog interface for the input file
from tkinter.filedialog import askdirectory
from tkinter import scrolledtext
# to manage a different font
from tkinter import font
# need this to check if a a file exists
import os
import sys
# to manage date
import datetime
import time
# to manage images
#from PIL import Image, ImageTk
# to manage plots
import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D
import numpy
import xarray as xr
import pandas as pd
import pandas
import uuid
#import requests
import random
import string
import requests as rq
from bs4 import BeautifulSoup
import json
import xmltodict
import csv
#from pandastable import Table, TableModel
from ttkthemes import ThemedTk

#For the webserver
import http.server
import socketserver
import threading

'''
To create an exe (Linux/Windows):

https://pypi.org/project/auto-py-to-exe/
pip install auto-py-to-exe
auto-py-to-exe


To include the logo use the following option inside auto-py-to-exe:
--hidden-import='PIL._tkinter_finder'


FOR WINDOWS ONLY with ANACONDA:
--exclude-module scikit-learn,PyQt5,PyQt4,2to3,IPython,Jinja2,pycparser,scipy



TO ADD NEW THEMES:
pip install ttkthemes

N.B. The themes plastik, clearlooks and elegance are recommended to make your
UI look nicer on all platforms when using Tkinter and the ttk extensions in Python.
 When you are targeting Ubuntu, consider using the great radiance theme.
'''



        
        

class GUItemplate:


    def __init__(self, master):
        self.master = master
        #master.title("DATA-GIZMO")

        '''
        THE FOLLOWING THREE ROWS EXPLAIN HOW TO CHANGE THE STATE OF A TAB
        DISABLED: the tab is visible but not active
        NORMAL: the tab is in nomral state
        HIDDEN: the tab is not visible
        these options could be useful if the software must manage different
        input/output LabelFrameINFO and it's necessary show only some tabs
        N.B. in the example the tab is 2 (self.LabelFrameD)
        '''
        #self.nb.tab(2, state="disabled")
        #self.nb.tab(2, state="normal")
        #self.nb.tab(2, state="hidden")

        frameFont = ttk.Style()
        frameFont.configure('new.TFrame', family='Verdana', size=8, weight='bold', underline=1)
        # Defines and places the notebook widget
        self.nb = ttk.Notebook(self.master)
        self.nb.grid(row=1, column=0, columnspan=50, rowspan=49, sticky='NESW')

        # Adds tab of the notebook
        self.LabelFrameINFO = ttk.Frame(self.nb, style='new.TFrame')
        self.nb.add(self.LabelFrameINFO, text='INFO')

       # Adds tab of the notebook
        self.LabelFrameMerger = ttk.Frame(self.nb)
        self.nb.add(self.LabelFrameMerger, text='Merger')      
        

        # Adds tab of the notebook
        self.LabelFrameDataJSON = ttk.Frame(self.nb)
        self.nb.add(self.LabelFrameDataJSON, text='JSON')
        
        # Adds tab of the notebook
        self.LabelFrameNETCDF = ttk.Frame(self.nb)
        self.nb.add(self.LabelFrameNETCDF, text='NETCDF')
        
        # Adds tab of the notebook
        self.LabelFrameMix = ttk.Frame(self.nb)
        self.nb.add(self.LabelFrameMix, text='MIX')
        
        #Common parts we have to wrap each command
        
        openfile = master.register(self.OpenInputFile)
        openfileXLS = master.register(self.OpenInputFileSheet)
        openfileCSV = master.register(self.OpenInputFileCSV)
        openfileXLSOneOutput = master.register(self.OpenInputFileSheetOneOutput)
        plotNetcdf = master.register(self.PlotOpenInputFileNetcdf)
        csvNetcdf = master.register(self.CSVOpenInputFileNetcdf)
        jsonNetcdf = master.register(self.JSONOpenInputFileNetcdf)
        xlsxNetcdf = master.register(self.XLSXOpenInputFileNetcdf)
        openURLlinks = master.register(self.OpenInputURLlinks)
        searchinfilesFunc = master.register(self.searchinfiles)
        generatePWD = master.register(self.generateRNDPWD)
        cropCSV = master.register(self.CropCSVfile)
        startwebserver = master.register(self.webServerStarted)
        guidgenerate = master.register(self.generateGUID)
        openXml2Json = master.register(self.OpenInputFileXml2Json)
        openCsv2Json = master.register(self.OpenInputFileCSV2Json)
        openfileODV = master.register(self.OpenInputFileODV)
        openODVMerge = master.register(self.OpenInputODVMerge)
        
        vcmd = master.register(self.validatenumber)

        mytext=''
        self.create=0



        '''
        START here we add a cascading menu
        '''
        # START here we add a cascading menu
        self.preferences = Menu(self.master)
        self.menuFieldA = Menu(self.master)
        self.menuFieldB = Menu(self.master)
        
     
        appHighlightFont = font.Font(family='helvetica', size=12, weight='normal', underline=0)
#        print(font.families())
#        AVAILABLE FONTS on TKINTER
#        ('fangsong ti', 
#         'fixed', 
#         'clearlyu alternate glyphs', 
#         'charter', 
#         'lucidatypewriter', 
#         'courier 10 pitch', 
#         'lucidabright', 
#         'times', 
#         'open look glyph', 
#         'bitstream charter', 
#         'song ti', 'helvetica', 
#         'open look cursor', 
#         'newspaper', 
#         'clearlyu ligature', 
#         'mincho', 
#         'clearlyu devangari extra', 
#         'clearlyu pua', 
#         'courier', 
#         'clearlyu', 
#         'lucida', 
#         'clean', 
#         'nil', 
#         'clearlyu arabic', 
#         'clearlyu devanagari', 
#         'terminal', 
#         'symbol', 
#         'gothic', 
#         'new century schoolbook', 
#         'clearlyu arabic extra')

        self.labelINFO = Label(self.LabelFrameINFO, text=mytext, bg="brown4", justify="left", fg="white", font=appHighlightFont, height=15, width=69)
        
        new_text = "Remember the Zen of Python:\n"
        new_text += "\n * Beautiful is better than ugly."
        new_text += " Explicit is better than implicit."
        new_text += "\n * Simple is better than complex."
        new_text += " Complex is better than complicated."
        new_text += "\n * Flat is better than nested."
        new_text += " Sparse is better than dense."
        new_text += "\n * Readability counts."
        new_text += " Special cases aren't special enough to break the rules."
        new_text += "\n * Although practicality beats purity."
        new_text += " Errors should never pass silently."
        new_text += "\n * Unless explicitly silenced."
        new_text += " In the face of ambiguity, refuse the temptation to guess."
        new_text += "\n * There should be one-- and preferably only one --obvious way to do it."
        new_text += "\n * Although that way may not be obvious at first unless you're Dutch."
        new_text += "\n * Now is better than never."
        new_text += " Although never is often better than right now."
        new_text += "\n * If the implementation is hard to explain, it's a bad idea."
        new_text += "\n * If the implementation is easy to explain, it may be a good idea."
        new_text += "\n * Namespaces are one honking great idea – let's do more of those!"
        self.labelINFO['text'] = new_text

        self.labelINFO.grid(row=0, column=0, columnspan=12, rowspan=10, padx=55, pady=55)

        
        #START INFO 
 
        self.labelEntryInfoFieldASF = Label(self.LabelFrameNETCDF, text=mytext)
        self.labelEntryInfoFieldASF['text'] = 'The value for FieldA: ' + str(mytext)
        self.labelEntryInfoFieldASF.grid(row=1, column=0, sticky=E)

        self.entryInfoFieldAVarsSF = tk.IntVar()
        self.entryInfoFieldASF = Spinbox(self.LabelFrameNETCDF, from_=1, to=10, textvariable= self.entryInfoFieldAVarsSF, width=2, bg="gold3", validate="key", validatecommand=(vcmd, '%P'))
        self.entryInfoFieldASF.grid(row=1, column=1, sticky=W)

        self.openInputFileButtonSF = Button(self.LabelFrameNETCDF, text="Load NetCDF Time Series Input File", command=(openfile), fg='white', bg='brown4',width=35)
        self.openInputFileButtonSF.grid(row=1, column=0, sticky=E)

        self.entryInfoInputFileSF = Entry(self.LabelFrameNETCDF, width=50, bg="gold3", validate="key")
        self.entryInfoInputFileSF.grid(row=1, column=1, columnspan=3, sticky=W)
        
        
        self.CSVNetcdfButton = Button(self.LabelFrameNETCDF, text="To CSV", width=10, command=(csvNetcdf), fg='white', bg='brown4')
        self.CSVNetcdfButton.grid(row=2, column=1, sticky=W)
        
        self.XLSXNetcdfButton = Button(self.LabelFrameNETCDF, text="To XLSX", width=10, command=(xlsxNetcdf), fg='white', bg='brown4')
        self.XLSXNetcdfButton.grid(row=2, column=2, sticky=W)
        
        self.JSONNetcdfButton = Button(self.LabelFrameNETCDF, text="To JSON", width=10, command=(jsonNetcdf), fg='white', bg='brown4')
        self.JSONNetcdfButton.grid(row=2, column=3, sticky=W)

        self.PlotNetcdfButton = Button(self.LabelFrameNETCDF, text="PLOT", width=10, command=(plotNetcdf), fg='white', bg='brown4')
        self.PlotNetcdfButton.grid(row=2, column=4, sticky=W)

        self.LabelFrameNETCDFFieldA = scrolledtext.ScrolledText(self.LabelFrameNETCDF, height=20, width=103, bg="gold3")
        self.LabelFrameNETCDFFieldA.grid(row=59, column=0, columnspan=16, rowspan=30)
        self.LabelFrameNETCDFFieldA.insert(END, "Infobox:")
               
        
        self.openInputFileButtonXLS = Button(self.LabelFrameMerger, text="Select directory (merge only XLSX/XLS files) - One file for each sheet", command=(openfileXLS), fg='white', bg='brown4',width=60)
        self.openInputFileButtonXLS.grid(row=3, column=0, sticky=E)
        
        self.openInputFileButtonXLSOneOutput = Button(self.LabelFrameMerger, text="Select directory (merge only XLSX/XLS files) - One file for all sheets", command=(openfileXLSOneOutput), fg='white', bg='brown4',width=60)
        self.openInputFileButtonXLSOneOutput.grid(row=4, column=0, sticky=E)
        
        self.openInputFileButtonCSV = Button(self.LabelFrameMerger, text="Select directory  (merge only TXT/CSV files)", command=(openfileCSV), fg='white', bg='brown4',width=40)
        self.openInputFileButtonCSV.grid(row=3, column=1, sticky=E)
                
        self.openInputFileButtonPDT = Button(self.LabelFrameMerger, text="Load ODV directory to merge", command=(openODVMerge), fg='white', bg='brown4',width=40)
        self.openInputFileButtonPDT.grid(row=4, column=1, sticky=E)
        

        self.LabelFrameMergerFieldAXLS = scrolledtext.ScrolledText(self.LabelFrameMerger, height=20, width=103, bg="gold3")
        self.LabelFrameMergerFieldAXLS.grid(row=59, column=0, columnspan=6, rowspan=10)
        self.LabelFrameMergerFieldAXLS.insert(END, "Infobox:")
                
        self.openInputFileButtonURL = Button(self.LabelFrameMix, text="Search URLs in webpage", command=(openURLlinks), fg='white', bg='brown4',width=35)
        self.openInputFileButtonURL.grid(row=1, column=0, sticky=E)
        
        self.SearchTextButton = Button(self.LabelFrameMix, text="Search text in a directory", command=(searchinfilesFunc), fg='white', bg='brown4',width=35)
        self.SearchTextButton.grid(row=2, column=0, sticky=E)
        
        self.RNDPasswordButton = Button(self.LabelFrameMix, text="Generate random password", command=(generatePWD), fg='white', bg='brown4',width=35)
        self.RNDPasswordButton.grid(row=3, column=0, sticky=E)
        
        
        self.CropCSVButton = Button(self.LabelFrameMix, text="CROP large CSV file", command=(cropCSV), fg='white', bg='brown4',width=35)
        self.CropCSVButton.grid(row=4, column=0, sticky=E)
        
        self.webserverButton = Button(self.LabelFrameMix, text="Start webserver (select a dir as web root, port 8000)", command=(startwebserver), fg='white', bg='brown4',width=49)
        self.webserverButton.grid(row=3, column=1, sticky=W)

        self.webserverButton = Button(self.LabelFrameMix, text="Generate GUID (globally unique identifier)", command=(guidgenerate), fg='white', bg='brown4',width=49)
        self.webserverButton.grid(row=4, column=1, sticky=W)

        self.URLinput = Text(self.LabelFrameMix, width=50, height=1, bg="gold3")
        self.URLinput.grid(row=1, column=1, sticky=W)
        
        self.textSearchinput = Text(self.LabelFrameMix, width=50, height=1, bg="gold3")
        self.textSearchinput.grid(row=2, column=1, sticky=W)

              
        self.LabelFrameFieldMix = scrolledtext.ScrolledText(self.LabelFrameMix, height=20, width=103, bg="gold3")
        self.LabelFrameFieldMix.grid(row=59, column=0, columnspan=6, rowspan=10)
        self.LabelFrameFieldMix.insert(END, "Infobox:")
            
        
        self.openInputFileButtonJSON = Button(self.LabelFrameDataJSON, text="From XML to JSON (directory)", command=(openXml2Json), fg='white', bg='brown4',width=35)
        self.openInputFileButtonJSON.grid(row=1, column=0, sticky=E)
        
        self.openInputFileButtonCSVJSON = Button(self.LabelFrameDataJSON, text="From CSV to JSON (directory)", command=(openCsv2Json), fg='white', bg='brown4',width=35)
        self.openInputFileButtonCSVJSON.grid(row=2, column=0, sticky=E)
        
        self.LabelFrameFieldJSON = scrolledtext.ScrolledText(self.LabelFrameDataJSON, height=20, width=103, bg="gold3")
        self.LabelFrameFieldJSON.grid(row=59, column=0, columnspan=6, rowspan=10)
        self.LabelFrameFieldJSON.insert(END, "Infobox:")
        


    def validatenumber(self, new_text):
        if not new_text: # the field is being cleared
            self.entered_number = 0
            return True

        try:
            self.entered_number = int(new_text)
            return True
        except ValueError:
            return False
    
    
    
    def handler_from(self,directory):
        def _init(self, *args, **kwargs):
            return http.server.SimpleHTTPRequestHandler.__init__(self, *args, directory=self.directory, **kwargs)
        return type(f'HandlerFrom<{directory}>',
                    (http.server.SimpleHTTPRequestHandler,),
                    {'__init__': _init, 'directory': directory})

    def webServerStarted(self,):
        PORT = 8000
        DIRECTORY = askdirectory()
        httpd=socketserver.TCPServer(("", PORT), self.handler_from(DIRECTORY))
        print("serving at port", PORT)
        print("Starting Server in background")
        thread = threading.Thread(target = httpd.serve_forever)
        thread.daemon = True
        thread.start()
    
    
    def generateGUID(self,):
        guid = uuid.uuid4()
        self.LabelFrameFieldMix.insert(END, "\n GUID:  "+str(guid))
    
    def searchinfiles(self,):
        f = 0
        dirname = askdirectory()
        #files = os.listdir(dirname)
        text = str(self.textSearchinput.get("1.0", "end-1c"))
        # print(files)
        if text !='' and dirname !='':
            for file_name in os.listdir(dirname):
                #abs_path = os.path.abspath(file_name)
                #if os.path.isdir(abs_path):
                    #self.searchinfiles(abs_path)
                    
                if os.path.isfile(dirname+'/'+file_name):
                    f = open(dirname+'/'+file_name, "r")
                    if text in f.read():
                        f = 1
                        #print(text + " found in ")
                        self.LabelFrameFieldMix.insert(END, "\n found "+str(text))
                        final_path = os.path.abspath(file_name)
                        self.LabelFrameFieldMix.insert(END, "\n in file: "+str(final_path))
                        #print(final_path)
                        return True
            if f == 0:
                #print(text + " not found! ")
                self.LabelFrameFieldMix.insert(END, "\n not found: "+str(text))
                return False
        if text =='':
            self.LabelFrameFieldMix.insert(END, "\n Please write the search term ")
        if dirname =='':
            self.LabelFrameFieldMix.insert(END, "\n Please select a directory ")


    def OpenInputODVMerge(self,):
        #actualdirname = os.getcwd()
        dirname = askdirectory()
        
        CSVNameFullMerge = str(time.strftime("%Y%m%d%H%M%S")+'_FullMerge.csv')
        CSVNameDataMerge = str(time.strftime("%Y%m%d%H%M%S")+'_DataMerge.csv')
        CSVNameDataMergeTS = str(time.strftime("%Y%m%d%H%M%S")+'_DataMergeTS.csv')
        mycounterDummy=1
        
        for fn in os.listdir(dirname):
            counterHeader = 1
            if not os.path.isfile(dirname+'/'+fn):
                return "File not found: " + fn
                self.LabelFrameMergerFieldAXLS.insert(END, "\n File not found: "+fn)
                sys.exit(-1)
        
            try:
                f = open(dirname+'/'+fn,'r')
                CSVOutputFile = open(CSVNameFullMerge,"a")
                #CSVOutputFileData = open(CSVNameDataMerge,"a")
                while True:
                    l=f.readline()
                    #O.header.append(l)
                
                    if l.find('//')==-1:
                        #print ('finished reading semantic header')
                        #CSVOutputFile.write('\t'+l)
                        CSVOutputFile.write(l)
                        CSVOutputFile.close()
        
        
                        if mycounterDummy == 1:
                            CSVOutputFileData = open(CSVNameDataMerge,"a")
                            CSVOutputFileData.write(l.replace('\t', ','))
                            CSVOutputFileData.close()
        
                            CSVOutputFileDataTS = open(CSVNameDataMergeTS,"a")
                            CSVOutputFileDataTS.write(l.replace('\t', ','))
                            CSVOutputFileDataTS.close()
        
        
                        #mycounterDummy += 1
                        
                        break
                    CSVOutputFile.write(l)
                    counterHeader += 1
                #print('counterHeader:'+str(counterHeader))
            except IOError:
                sys.exit(-1)
        
            data = pandas.read_csv(dirname+'/'+fn,sep='\t',index_col=False, na_values=numpy.nan, skiprows = counterHeader-1) #, parse_dates=[3], infer_datetime_format=True, date_parser=odvdatetime)
            data.columns = [c.replace(' ', '_') for c in data.columns]
            data['Cruise'].fillna(method='ffill', inplace = True)
            data['Station'].fillna(method='ffill', inplace = True)
            data['Type'].fillna(method='ffill', inplace = True)
            data['YYYY-MM-DDThh:mm:ss.sss'].fillna(method='ffill', inplace = True)
            data['Longitude_[degrees_east]'].fillna(method='ffill', inplace = True)
            data['Latitude_[degrees_north]'].fillna(method='ffill', inplace = True)
            data['LOCAL_CDI_ID'].fillna(method='ffill', inplace = True)
            data['EDMO_code'].fillna(method='ffill', inplace = True)
            data['Bot._Depth_[m]'].fillna(method='ffill', inplace = True)
        
            #print(O.data.iloc[:, 0:12])
            print(data)
            
            #data.iloc[:,1:].to_csv(CSVName, mode='a', header=False)
            data.to_csv(CSVNameFullMerge, mode='a', header=False,index=False)
            data.to_csv(CSVNameDataMerge, mode='a', header=False,index=False)
            f.close()
            
            mycounterDummy += 1
        
        TSDataSorted = pandas.read_csv(CSVNameDataMerge,sep=',', na_values=numpy.nan,index_col=False) #, parse_dates=[3], infer_datetime_format=True, date_parser=odvdatetime)
        print(TSDataSorted.sort_values(by =['Pres_Z [dBar]', 'YYYY-MM-DDThh:mm:ss.sss']))
        OutputDataSorted=TSDataSorted.sort_values(by =['Pres_Z [dBar]', 'YYYY-MM-DDThh:mm:ss.sss'])
        OutputDataSorted.to_csv(CSVNameDataMergeTS, mode='a', header=False,index=False)
        #df = pd.read_csv(self.name)
        self.LabelFrameMergerFieldAXLS.insert(END, "\n Created file: "+str(CSVNameFullMerge))
        self.LabelFrameMergerFieldAXLS.insert(END, "\n Created file: "+str(CSVNameDataMerge))
        self.LabelFrameMergerFieldAXLS.insert(END, "\n Created file: "+str(CSVNameDataMergeTS))

    
    def generateRNDPWD(self,):
        total = string.ascii_letters + string.digits + string.punctuation
        length = 16
        password = "".join(random.sample(total, length))
        print(password)
        self.LabelFrameFieldMix.insert(END, "\nPassword:  "+str(password))
        
    
    def CropCSVfile(self,):
        actualdirname = os.getcwd()
        self.LabelFrameFieldMix.delete('1.0', END)
        self.name = askopenfilename(initialdir=actualdirname,
                               filetypes =(("CSV", "*.csv"),("TXT", "*.txt"),("All Files","*.*")),
                               title = "Choose a file."
                               )
        
        self.LabelFrameFieldMix.insert(END, "\n Read File: "+str(self.name))
        chunksize = 10 ** 6
        for chunk in pd.read_csv(self.name, chunksize=chunksize):
            # chunk is a DataFrame. To "process" the rows in the chunk:
            outputFile = str(time.strftime("%Y%m%d%H%M%S")+'_output-file.csv')
            chunk.to_csv(outputFile, encoding='utf-8')
            
            self.LabelFrameFieldMix.insert(END, "\n Created File: "+str(outputFile))

    def OpenInputURLlinks(self,):
        
        url = str(self.URLinput.get("1.0", "end-1c"))+ '/'
        self.LabelFrameFieldMix.insert(END, "\n"+str(url))
        if ("https" or "http") in url:
            data = rq.get(url , timeout=10)
        else:
            data = rq.get("https://" + url , timeout=10)
        soup = BeautifulSoup(data.text, "html.parser")
        links = []
        for link in soup.find_all("a"):
            links.append(link.get("href"))
            
        for foundLink in links:
            self.LabelFrameFieldMix.insert(END, "\n"+str(foundLink))
    

    #Define an input file for data.
    def OpenInputFile(self,):
        actualdirname = os.getcwd()
        self.name = askopenfilename(initialdir=actualdirname,
                               filetypes =(("NetCDF", "*.nc"),("All Files","*.*")),
                               title = "Choose a file."
                               )

        self.entryInfoInputFileSF.delete(0,END)
        self.entryInfoInputFileSF.insert(0,self.name)
        #self.src = self.name
        #self.entryInfoInputFileSF.insert(END, str(self.name))
        self.LabelFrameNETCDFFieldA.delete('1.0', END)
        self.LabelFrameNETCDFFieldA.insert(END, "\n NetCDF Input File: "+str(self.name))
        
        
        
    def JSONOpenInputFileNetcdf(self,):
        try:
            
            ds = xr.open_dataset(self.name)
            for attr in ds.attrs.items():
                #print(str(attr))
                self.LabelFrameNETCDFFieldA.insert(END, "\n "+str(attr))
            
            try:
                
                subset = []
                for param in ds.data_vars:
                    if ("_DM" not in param) and ("_QC" not in param):
                        subset.append(param)
                var_w_units = []
                for i in subset:
                    try:
                        var_w_units.append(f"{i} [{ds[i].units}]")
                    except:
                        var_w_units.append(f"")
                        
                #coordType=0
                
                df = pd.DataFrame()
                for i in subset:
                    try:
                        df[i] = ds[i][:,0].values
                    except:
                        try:
                            df[i] = ds[i][:].values
                            #coordType=1
                        except:
                            pass
                
                df.columns = var_w_units
                
                try:
                    df["TIME"] = ds["TIME"].values
                except:
                    pass
                try:
                    df["TIME"] = ds["time"].values
                except:
                    pass
                
                df.set_index("TIME")
                df.set_index("TIME",inplace=True)
                d = df.index
                d.strftime('%Y-%m-%d %H:%M')
                df.set_index(d.strftime('%Y-%m-%d %H:%M'))
                self.LabelFrameNETCDFFieldA.insert(END, "\n"+str(df))
                #if coordType==0:
                df.to_json(str(self.name)+'.json')
                    
                
            except:
                try:
                    
                    df = ds.to_dataframe()
                    df_without_duplicates = df.drop_duplicates()
                    df_without_duplicates.to_json(str(self.name)+'.json')
                    
                    
                except Exception as e:
                    #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
                    self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)
        except Exception as e:
            #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
            self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)
            
    def CSVOpenInputFileNetcdf(self,):
        try:
            
            ds = xr.open_dataset(self.name)
            for attr in ds.attrs.items():
                #print(str(attr))
                self.LabelFrameNETCDFFieldA.insert(END, "\n "+str(attr))
            
            try:
                
                subset = []
                for param in ds.data_vars:
                    if ("_DM" not in param) and ("_QC" not in param):
                        subset.append(param)
                var_w_units = []
                for i in subset:
                    try:
                        var_w_units.append(f"{i} [{ds[i].units}]")
                    except:
                        var_w_units.append(f"")
                        
                #coordType=0
                
                df = pd.DataFrame()
                for i in subset:
                    try:
                        df[i] = ds[i][:,0].values
                    except:
                        try:
                            df[i] = ds[i][:].values
                            #coordType=1
                        except:
                            pass
                
                df.columns = var_w_units
                
                try:
                    df["TIME"] = ds["TIME"].values
                except:
                    pass
                try:
                    df["TIME"] = ds["time"].values
                except:
                    pass
                
                df.set_index("TIME")
                df.set_index("TIME",inplace=True)
                d = df.index
                d.strftime('%Y-%m-%d %H:%M')
                df.set_index(d.strftime('%Y-%m-%d %H:%M'))
                self.LabelFrameNETCDFFieldA.insert(END, "\n"+str(df))
                #if coordType==0:
                df.to_csv(str(self.name)+'.csv', encoding='utf-8')
                    
                
            except:
                try:
                    
                    df = ds.to_dataframe()
                    df_without_duplicates = df.drop_duplicates()
                    df_without_duplicates.to_csv(str(self.name)+'.csv', encoding='utf-8')
                    
                    
                except Exception as e:
                    #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
                    self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)
        except Exception as e:
            #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
            self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)
        
    def XLSXOpenInputFileNetcdf(self,):
        try:
            
            ds = xr.open_dataset(self.name)
            for attr in ds.attrs.items():
                #print(str(attr))
                self.LabelFrameNETCDFFieldA.insert(END, "\n "+str(attr))
            
            try:
                
                subset = []
                for param in ds.data_vars:
                    if ("_DM" not in param) and ("_QC" not in param):
                        subset.append(param)
                var_w_units = []
                for i in subset:
                    try:
                        var_w_units.append(f"{i} [{ds[i].units}]")
                    except:
                        var_w_units.append(f"")
                        
                #coordType=0
                
                df = pd.DataFrame()
                for i in subset:
                    try:
                        df[i] = ds[i][:,0].values
                    except:
                        try:
                            df[i] = ds[i][:].values
                            #coordType=1
                        except:
                            pass
                
                df.columns = var_w_units
                
                try:
                    df["TIME"] = ds["TIME"].values
                except:
                    pass
                try:
                    df["TIME"] = ds["time"].values
                except:
                    pass
                
                df.set_index("TIME")
                df.set_index("TIME",inplace=True)
                d = df.index
                d.strftime('%Y-%m-%d %H:%M')
                df.set_index(d.strftime('%Y-%m-%d %H:%M'))
                self.LabelFrameNETCDFFieldA.insert(END, "\n"+str(df))
                #if coordType==0:
                df.to_excel(str(self.name)+'.xlsx')
                    
                
            except:
                try:
                    
                    df = ds.to_dataframe()
                    df_without_duplicates = df.drop_duplicates()
                    df_without_duplicates.to_excel('test.xlsx')
                    
                    
                except Exception as e:
                    #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
                    self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)
        except Exception as e:
            #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
            self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)


    def PlotOpenInputFileNetcdf(self,):
        try:
            
            ds = xr.open_dataset(self.name)
            for attr in ds.attrs.items():
                #print(str(attr))
                self.LabelFrameNETCDFFieldA.insert(END, "\n "+str(attr))
            
            try:
                
                subset = []
                for param in ds.data_vars:
                    if ("_DM" not in param) and ("_QC" not in param):
                        subset.append(param)
                var_w_units = []
                for i in subset:
                    try:
                        var_w_units.append(f"{i} [{ds[i].units}]")
                    except:
                        var_w_units.append(f"")
                        
                #coordType=0
                
                df = pd.DataFrame()
                for i in subset:
                    try:
                        df[i] = ds[i][:,0].values
                    except:
                        try:
                            df[i] = ds[i][:].values
                            #coordType=1
                        except:
                            pass
                
                df.columns = var_w_units
                
                try:
                    df["TIME"] = ds["TIME"].values
                except:
                    pass
                try:
                    df["TIME"] = ds["time"].values
                except:
                    pass
                
                df.set_index("TIME")
                df.set_index("TIME",inplace=True)
                d = df.index
                d.strftime('%Y-%m-%d %H:%M')
                df.set_index(d.strftime('%Y-%m-%d %H:%M'))
                self.LabelFrameNETCDFFieldA.insert(END, "\n"+str(df))
                #if coordType==0:
                df.plot()
                plt.show()
                    
               
            except:
                try:
                    for u in ds.data_vars.keys():
                        tempVar = ds[u]
                        tempVar_0=tempVar[0, 0, :, :]
                        tempVar_0.plot(cmap='plasma')
                        plt.show()
                   
                except Exception as e:
                    #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
                    self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)
        except Exception as e:
            #self.LabelFrameNETCDFFieldA.insert(END, "\n Error, something went wrong....")
            self.LabelFrameNETCDFFieldA.insert(END, "\n"+e)


    

    #To make it work with PyInstaller (this is due to an update of PyInstaller)
    #this functions is used to define the right file path between two differents
    #enviroments: your PC and PInstaller wrapping
    def resource_path(self,relative_path):
        """ Get absolute path to resource, works for dev and for PyInstaller """
        try:
            # PyInstaller creates a temp folder and stores path in _MEIPASS
            base_path = sys._MEIPASS
        except Exception:
            base_path = os.path.abspath(".")
    
        return os.path.join(base_path, relative_path)
    
   
    #Define an input file for data TXT/CSV.
    def OpenInputFileCSV(self,):
        self.nameSF = askdirectory()

        self.srcSF = self.nameSF
        self.LabelFrameNETCDFFieldB.insert(END, "\n The directory is: "+str(self.nameSF))
        outputFileSF = str(time.strftime("%Y%m%d%H%M%S")+'.txt')
        
        
        t=open(outputFileSF, 'w')
        t.close()
        f=open(outputFileSF, 'a')
        
        for fileSF in os.listdir(self.srcSF):
            if fileSF.endswith(".csv"):
                self.LabelFrameNETCDFFieldB.insert(END, "\n Working the file: "+str(fileSF))
                
                
                file1 = open(self.srcSF+'/'+fileSF, 'r')
                #The slicing [1:] to skip the header row
                Lines = file1.readlines()

                for line in Lines:
                    f.write(str(line))
                    #print("\n Reading the file: "+str(fileSF))
                    #print('scrivo')

        f.close()


    #Define an input file for data XLSX.
    def OpenInputFileSheet(self,):
        #actualdirname = os.getcwd()
        self.name = askdirectory()

        #self.LabelFrameNETCDFFieldA.delete(0,END)
        #self.LabelFrameNETCDFFieldA.insert(0,self.name)
        self.srcXLS = self.name
        self.LabelFrameMergerFieldAXLS.delete('1.0', END)
        self.LabelFrameMergerFieldAXLS.insert(END, "\n The directory is: "+str(self.name))
        #outputFile = str(time.strftime("%Y%m%d%H%M%S")+'.xlsx')
        
        
        
        # list of excel files we want to merge.
        # pd.read_excel(file_path) reads the excel
        # data into pandas dataframe.
        excl_list = []
        recovery_sheet_list = []

        for file in os.listdir(self.srcXLS):
            xls = pd.ExcelFile(self.srcXLS+'/'+file)
            for sheetsName in xls.sheet_names:
                # List all sheets in the file
                #xls.sheet_names
                #print(file+" -- "+sheetsName)
                # ['house', 'house_extra', ...]
                if sheetsName not in recovery_sheet_list:
                    recovery_sheet_list.append(sheetsName)
                      
        
        print('All sheets are: '+str(recovery_sheet_list))
        for mySheet in recovery_sheet_list:
            excl_list = []
            #try:
            for file in os.listdir(self.srcXLS):
                try:
                    excl_list.append(pd.read_excel(self.srcXLS+'/'+file, sheet_name=mySheet))
                except:
                    pass
                
            excl_merged = pd.DataFrame()
                 
            for excl_file in excl_list:
                 
                # appends the data into the excl_merged
                # dataframe.
                excl_merged = excl_merged.append(excl_file, ignore_index=True)
             
            # exports the dataframe into excel file with
            # specified name.
            outputFile = str(mySheet+'_'+time.strftime("%Y%m%d%H%M%S")+'.xlsx')
            excl_merged.to_excel(outputFile+'.xlsx', index=False)
            #except:
            #    print('Sorry something went wrong')
    
    
        
    
    #Define an input file for data XLSX.
    def OpenInputFileODV(self,):
        #actualdirname = os.getcwd()
        self.name = askdirectory()

        #self.LabelFrameNETCDFFieldA.delete(0,END)
        #self.LabelFrameNETCDFFieldA.insert(0,self.name)
        #self.srcXLS = self.name
        dirname=self.name
        mycounterDummy=1
        #CSVNameFullMerge = str('FullMerge_'+time.strftime("%Y%m%d%H%M%S")+'.csv')
        #CSVNameDataMerge = str('DataMerge_'+time.strftime("%Y%m%d%H%M%S")+'.csv')
        #CSVNameDataMergeTS = str('DataMergeTS_'+time.strftime("%Y%m%d%H%M%S")+'.csv')
        
        CSVNameFullMerge = str(time.strftime("%Y%m%d%H%M%S")+'_FullMerge.csv')
        CSVNameDataMerge = str(time.strftime("%Y%m%d%H%M%S")+'_DataMerge.csv')
        CSVNameDataMergeTS = str(time.strftime("%Y%m%d%H%M%S")+'_DataMergeTS.csv')
        

        for fn in os.listdir(dirname):
            #print(u)
            print(fn)
            #filename = fn
            counterHeader = 1
            if not os.path.isfile(dirname+'/'+fn):
                return "File not found: " + fn
                sys.exit(-1)
            
            try:
                f = open(dirname+'/'+fn,'r')
                CSVOutputFile = open(CSVNameFullMerge,"a")
                #CSVOutputFileData = open(CSVNameDataMerge,"a")
                while True:
                    l=f.readline()
                    #O.header.append(l)
                
                    if l.find('//')==-1:
                        #print ('finished reading semantic header')
                        #CSVOutputFile.write('\t'+l)
                        CSVOutputFile.write(l)
                        CSVOutputFile.close()
        
        
                        if mycounterDummy == 1:
                            CSVOutputFileData = open(CSVNameDataMerge,"a")
                            CSVOutputFileData.write(l.replace('\t', ','))
                            CSVOutputFileData.close()
        
                            CSVOutputFileDataTS = open(CSVNameDataMergeTS,"a")
                            CSVOutputFileDataTS.write(l.replace('\t', ','))
                            CSVOutputFileDataTS.close()
        
        
                        #mycounterDummy += 1
                        
                        break
                    CSVOutputFile.write(l)
                    counterHeader += 1
                print('counterHeader:'+str(counterHeader))
            except IOError:
                sys.exit(-1)
        
            data = pd.read_csv(dirname+'/'+fn,sep='\t',index_col=False, na_values=numpy.nan, skiprows = counterHeader-1) #, parse_dates=[3], infer_datetime_format=True, date_parser=odvdatetime)
            
            data.columns = [c.replace(' ', '_') for c in data.columns]
            data['Cruise'].fillna(method='ffill', inplace = True)
            data['Station'].fillna(method='ffill', inplace = True)
            data['Type'].fillna(method='ffill', inplace = True)
            data['YYYY-MM-DDThh:mm:ss.sss'].fillna(method='ffill', inplace = True)
            data['Longitude_[degrees_east]'].fillna(method='ffill', inplace = True)
            data['Latitude_[degrees_north]'].fillna(method='ffill', inplace = True)
            data['LOCAL_CDI_ID'].fillna(method='ffill', inplace = True)
            data['EDMO_code'].fillna(method='ffill', inplace = True)
            data['Bot._Depth_[m]'].fillna(method='ffill', inplace = True)
        
            #print(O.data.iloc[:, 0:12])
            print(data)
            
            #CSVNameFullMerge = str(time.strftime("%Y%m%d%H%M%S")+'_FullMerge.csv')
            #CSVNameDataMerge = str(time.strftime("%Y%m%d%H%M%S")+'_DataMerge.csv')
            #CSVNameDataMergeTS = str(time.strftime("%Y%m%d%H%M%S")+'_DataMergeTS.csv')
            
            #data.iloc[:,1:].to_csv(CSVName, mode='a', header=False)
            data.to_csv(CSVNameFullMerge, mode='a', header=False,index=False)
            data.to_csv(CSVNameDataMerge, mode='a', header=False,index=False)
            f.close()
            mycounterDummy += 1
        
        
        TSDataSorted = pd.read_csv(CSVNameDataMerge,sep=',', na_values=numpy.nan,index_col=False) #, parse_dates=[3], infer_datetime_format=True, date_parser=odvdatetime)
        print(TSDataSorted.sort_values(by =['Pres_Z [dBar]', 'YYYY-MM-DDThh:mm:ss.sss']))
        OutputDataSorted=TSDataSorted.sort_values(by =['Pres_Z [dBar]', 'YYYY-MM-DDThh:mm:ss.sss'])
        OutputDataSorted.to_csv(CSVNameDataMergeTS, mode='a', header=False,index=False)
    
    
    def OpenInputFileXml2Json(self,):
        #actualdirname = os.getcwd()
        self.name = askdirectory()
        xmlDir = self.name
        self.LabelFrameFieldJSON.insert(END, "\n The XML directory is: "+str(xmlDir))

        for file in os.listdir(xmlDir):
            if file.endswith(".xml"):
                self.LabelFrameFieldJSON.insert(END, "\n Working the file: "+str(file))
                with open(xmlDir+'/'+file) as xml_file:
                     
                    data_dict = xmltodict.parse(xml_file.read())
                     
                    json_data = json.dumps(data_dict, indent=4)
                    
                    outputfile=time.strftime("%Y%m%d%H%M%S")+'_'+file+'.json'
                    # Write the json data to output
                    # json file
                    with open(outputfile, "w") as json_file:
                        json_file.write(json_data)
                        # json_file.close()
                        
                    self.LabelFrameFieldJSON.insert(END, "\n Created the JSON file: "+str(outputfile))
                
        self.LabelFrameFieldJSON.insert(END, "\n All done! ")
        
    def OpenInputFileCSV2Json(self,):
        
        self.name = askdirectory()
        csvDir = self.name
        self.LabelFrameFieldJSON.insert(END, "\n The CSV directory is: "+str(csvDir))
        
        for file in os.listdir(csvDir):
            if file.endswith(".csv"):
                self.LabelFrameFieldJSON.insert(END, "\n Working the file: "+str(file))
        
                jsonArray = []
                csvFilePath=csvDir+'/'+file
                #read csv file
                with open(csvFilePath, encoding='utf-8') as csvf: 
                    #load csv file data using csv library's dictionary reader
                    csvReader = csv.DictReader(csvf) 
            
                    #convert each csv row into python dict
                    for row in csvReader: 
                        #add this python dict to json array
                        jsonArray.append(row)
              
                #convert python jsonArray to JSON String and write to file
                outputfile=time.strftime("%Y%m%d%H%M%S")+'_'+file+'.json'
                with open(outputfile, 'w', encoding='utf-8') as jsonf: 
                    jsonString = json.dumps(jsonArray, indent=4)
                    jsonf.write(jsonString)
                    
                self.LabelFrameFieldJSON.insert(END, "\n Created the JSON file: "+str(outputfile))
        
        self.LabelFrameFieldJSON.insert(END, "\n All done! ")
        
        
    #Define an input file for data XLSX.
    def OpenInputFileSheetOneOutput(self,):
        #actualdirname = os.getcwd()
        self.name = askdirectory()

        #self.LabelFrameNETCDFFieldA.delete(0,END)
        #self.LabelFrameNETCDFFieldA.insert(0,self.name)
        self.srcXLSOneOutput = self.name
        self.LabelFrameMergerFieldAXLS.delete('1.0', END)
        self.LabelFrameMergerFieldAXLS.insert(END, "\n The directory is: "+str(self.name))
        #outputFile = str(time.strftime("%Y%m%d%H%M%S")+'.xlsx')
        
        
        
        # list of excel files we want to merge.
        # pd.read_excel(file_path) reads the excel
        # data into pandas dataframe.
        excl_list = []
        recovery_sheet_list = []

        for file in os.listdir(self.srcXLSOneOutput):
            xls = pd.ExcelFile(self.srcXLSOneOutput+'/'+file)
            for sheetsName in xls.sheet_names:
                if sheetsName not in recovery_sheet_list:
                    recovery_sheet_list.append(sheetsName)
            
            
        #outputFile = str('output_'+time.strftime("%Y%m%d%H%M%S")+'.xlsx')
        print('All sheets are: '+str(recovery_sheet_list))
        with pd.ExcelWriter('ALL'+time.strftime("%Y%m%d%H%M%S")+'_'+'.xlsx') as writerExcel:
            for mySheet in recovery_sheet_list:
                excl_list = []
                #try:
                for file in os.listdir(self.srcXLSOneOutput):
                    try:
                        excl_list.append(pd.read_excel(self.srcXLSOneOutput+'/'+file, sheet_name=mySheet))
                    except:
                        pass
                    
                excl_merged = pd.DataFrame()
                     
                for excl_file in excl_list:
                     
                    # appends the data into the excl_merged
                    # dataframe.
                    excl_merged = excl_merged.append(excl_file, ignore_index=True)
                 
                # exports the dataframe into excel file with
                # specified name.
                
                #excl_merged.to_excel(outputFile+'.xlsx', index=False)
                excl_merged.to_excel(writerExcel, sheet_name=mySheet, index=False)
                #except:
                #    print('Sorry something went wrong')


#To make it work with PyInstaller (this is due to an update of PyInstaller)
#this functions is used to define the right file path between two differents
#enviroments: your PC and PInstaller wrapping
def resource_path(relative_path):
    """ Get absolute path to resource, works for dev and for PyInstaller """
    try:
        # PyInstaller creates a temp folder and stores path in _MEIPASS
        base_path = sys._MEIPASS
    except Exception:
        base_path = os.path.abspath(".")

    return os.path.join(base_path, relative_path)        



'''
Second version themes
TO ADD NEW THEMES:
pip install ttkthemes

N.B. The themes plastik, clearlooks and elegance are recommended to make your
UI look nicer on all platforms when using Tkinter and the ttk extensions in Python.
 When you are targeting Ubuntu, consider using the great radiance theme.
'''
#root = ThemedTk(theme="adapta")
#root = ThemedTk(theme="blue")
#root = ThemedTk(theme="scidblue")
#root = ThemedTk(theme="yaru")
#root = ThemedTk(theme="kroc")
#root = ThemedTk(theme="keramik")
#root = ThemedTk(theme="winxpblue")
#root = ThemedTk(theme="itft1")
#root = ThemedTk(theme="radiance")
#root = ThemedTk(theme="black")      # maybe
#root = ThemedTk(theme="elegance")      # maybe

root = ThemedTk(theme="black")
root.title('DATA GIZMO')

root.option_add('*Font', 'Verdana 8')
my_gui = GUItemplate(root)
root.geometry('750x400')
cols = 0
while cols < 20:
    root.rowconfigure(cols, weight=1)
    root.columnconfigure(cols, weight=1)
    cols += 1



root.mainloop()