Commit 03fa3752 authored by Andoni Jimenez's avatar Andoni Jimenez
Browse files

Add concat function, refactor/correct append calls

parent 18256458

src/utils.py

+45 −9
Original line number Diff line number Diff line
@@ -5,6 +5,9 @@ import argparse 
from pathlib import Path

import pandas as pd



import warnings

from typing import Union



### OPTION PARSER UTILS



args = None
@@ -67,7 +70,8 @@ def getFiles(): 
        if len(args.void) > 0:

            for void in args.void:

                if void in availableVoids:

                    selectedFiles = selectedFiles.append(voids[voids.void == void ])

                    selectedFiles = pd_concat(selectedFiles, voids[voids.void == void])

                    #selectedFiles = selectedFiles.append(voids[voids.void == void ])

                    selectedVoids.append(int(void))

                else:

                    print('WARNING:\tVoid ', void, 'not available.')
@@ -133,8 +137,8 @@ def createVoidsFile(): 
                        'dec':float(0),

                        'filename': str(file.name),

                    }



            voids = voids.append(entry, ignore_index=True)

            voids = pd_concat(voids, entry)

            #voids = voids.append(entry, ignore_index=True)

    voids = voids.sort_values(by=['void','galaxy'])

    voids.to_csv(args.voidsfile, columns=VFCOLUMNS, index=False)

    print('Done!')
@@ -217,8 +221,7 @@ def expand_df(selectedFiles): 
                    importData.loc[importData.galaxy == row.galaxy, 'dec'] = row.dec

                # If selected row is not in savefile:

                else:

                    importData = importData.append(newEntry(row),

                                                   ignore_index=True)

                    importData = pd_concat(importData, newEntry(row))



            # Add the full path to imported but unselected data:

            processedUnselectedData = importData[importData.fullpath == '']
@@ -236,8 +239,7 @@ def expand_df(selectedFiles): 
    else:

        importData = pd.DataFrame(columns=COLUMNS)

        for i, row in selectedFiles.iterrows():

            importData = importData.append(newEntry(row),

                                           ignore_index=True)

            importData = pd_concat(importData, newEntry(row))



    return importData
@@ -271,8 +273,10 @@ def save_df(df): 
    global importData



    # Concatenate items in CSV with our processed items:

    processedItems = pd.concat([importData.loc[df['processed'] == True],

                                df.loc[df['processed'] == True]])

    processedItems = pd_concat(importData.loc[df['processed'] == True],

                               df.loc[df['processed'] == True])

    # processedItems = pd.concat([importData.loc[df['processed'] == True],

    #                            df.loc[df['processed'] == True]])



    # Remove old values, keep last ones:

    exportData = processedItems.drop_duplicates(['void','galaxy'],keep='last').sort_values('void')
@@ -280,3 +284,35 @@ def save_df(df): 
    # Export final dataframe:

    exportData.to_csv(args.savefile, columns=getExportableColumns(),

                          index=False)

    

def pd_concat(df: pd.DataFrame, data: Union[pd.DataFrame, list, dict]) -> pd.DataFrame:

        """ Concats data in an array to the given dataframe



        Parameters

        ----------

        df: pandas.Dataframe

            Pandas dataframe to use



        data: list, dict or pandas.Dataframe

            Data to be concatenated to the input pandas Dataframe.

            - List of the values to be concatenated (order of input values and Dataframe columns must match).

            - Dict of the key:values, where keys match the Dataframe columns.



        Returns

        -------

        pandas.DataFrame

        """

        # check if data is list

        if type(data) == list:

            if len(data) != len(df.columns):

                raise Exception('ERROR: Input data [list] length is not equal to input dataframe')

            df_data = pd.DataFrame([data], columns=df.columns)

        elif type(data) == dict:

            if len(data) != len(df.columns):

                warnings.warn('Input data [dict] missing input dataframe keys. Missing values insterted as NaN')

            df_data = pd.DataFrame([data])

        elif type(data) == pd.DataFrame:

            df_data = data

            

        df = pd.concat([df, df_data], ignore_index=True)

        return df
 
 No newline at end of file