Commit ea0d57f9 authored by Andoni Jimenez's avatar Andoni Jimenez
Browse files

Solve rb error. More error info. Dynamic COLUMNS (). Non dependant of GROUPS...

Solve rb error. More error info. Dynamic COLUMNS (). Non dependant of GROUPS (input & output files). Add typing to functions. Change parser input file.
parent 2e12551d
Loading
Loading
Loading
Loading
+2 −2
Original line number Diff line number Diff line
#! /usr/bin/python3
#######################################################
# CAVYssify: Tool to manually classify void galaxies. #
# GALAssify: Tool to manually classify void galaxies. #
# Writen by MrManlu                                   #
#######################################################

@@ -21,7 +21,7 @@ if __name__ == '__main__':

        else:
            print(f"INFO:\t {str(len(selectedFiles))} galaxies found in selected groups.")
            df = utils.expand_df(selectedFiles)
            df = selectedFiles # utils.expand_df(selectedFiles)
            app = QtWidgets.QApplication(sys.argv)
            window = gui.Ui(df, selectedGroups)
            app.exec_()
+110 −69
Original line number Diff line number Diff line
from pathlib import Path
from PyQt5 import QtWidgets, uic
from PyQt5.QtCore import Qt, QEvent, QSize, QCoreApplication
from PyQt5.QtCore import Qt, QObject, QEvent, QSize, QCoreApplication
from PyQt5.QtGui import QPixmap, QIcon
import pandas as pd
from src import utils
@@ -10,7 +10,7 @@ import json


class Ui(QtWidgets.QMainWindow):
    def __init__(self, df, selectedGroups):
    def __init__(self, df:pd.DataFrame, selectedGroups:list):
        super(Ui, self).__init__()
        uic.loadUi(str(Path('src')/Path('gui.ui')), self)

@@ -18,8 +18,9 @@ class Ui(QtWidgets.QMainWindow):
        self.setWindowIcon(QIcon(str(Path('res/window_icon.png'))))

        # Load data frame and make it accessible along the class:
        self.df = df
        # self.df = df # do this later
        self.groups = selectedGroups
        self.has_groups = self.groups is not None

        # Find buttons:
        pb_prev = self.findChild(QtWidgets.QPushButton, 'pb_prev')
@@ -134,7 +135,7 @@ class Ui(QtWidgets.QMainWindow):
                            self.cbNames.append(wid)
                            
                        except KeyError as e:
                            print(e)
                            print(f"KeyError: {e}")
                # RADIOBUTTON TYPE 
                elif tconf['type'] == 'radiobutton':
                    if 'ncolumns' in tconf.keys():
@@ -154,7 +155,7 @@ class Ui(QtWidgets.QMainWindow):
                            widget = QtWidgets.QRadioButton(None)
                            widget.setObjectName(wid)
                            widget.setSizePolicy(QtWidgets.QSizePolicy(policy.Minimum, policy.Fixed))
                            addRadioButtonTexts()
                            addRadioButtonTexts(widget, element)
                            addToolTip(widget, element)
                            
                            gbb.addButton(widget)
@@ -173,7 +174,7 @@ class Ui(QtWidgets.QMainWindow):
                                gb.layout().addWidget(widget, count//cols, count%cols)
                            count += 1
                        except KeyError as e:
                            print(e)
                            print(f"KeyError: {e}")
                    if 'add_clear' in tconf.keys() and tconf['add_clear'] is True:
                        widget = QtWidgets.QPushButton(None)
                        widget.setObjectName(f"pb_clear_{id}")
@@ -195,17 +196,34 @@ class Ui(QtWidgets.QMainWindow):
            except:
                raise
        
        # Dynamic COLUMNS
        utils.IDS['FILE'] = list(df.columns)
        utils.IDS['TB'] = list(self.tb.keys())
        utils.IDS['CB'] = self.cbColumns
        utils.IDS['RB'] = list(self.rb.keys()) + [""]
        utils.IDS['RBG'] = list(self.rbg.keys())

        utils.COLUMNS = utils.IDS['FILE'] + utils.IDS['RBG'] + utils.IDS['CB'] + utils.IDS['TB']
        for v in ['ra', 'dec']:
            try:
                utils.COLUMNS.remove(v)
            except ValueError:
                pass
        utils.COLUMNS.extend(['processed', 'fullpath', 'ra', 'dec'])
        

        # Changes to make
        # 1 - utils COLUMNS and MORPHO need to be dinamic
        # DONE 1 - utils COLUMNS and MORPHO need to be dinamic
        # DONE 2 - aboid finding cb and rb maybe safe them in creation
        # DONE 3 - get radio button ischecked from ButtonGroup
        # 4 - dinamic input file, independent form group.
        # DONE 4 - dinamic input file, independent form group.
        #          also, HEADER created from this file
        # 5 - output file keys have to be dinamic (related to input file and COLUMNS)
        # DONE 5 - output file keys have to be dinamic (related to input file and COLUMNS)
        #          also, newEntry func
        # 6 - what if multiple same named widgets? add groudid to widgetid?
        # 6 - what if multiple same named widgets? add groupid to widgetid?
        # 7 - multiple commentBox not supported (yes, but events need to be solved)
        # 8 - If filename not in input file, detect images (currently fails)
        # 9 - Shortcuts not working
        
        
        # Find checkboxes:
@@ -241,6 +259,7 @@ class Ui(QtWidgets.QMainWindow):
        # Find table:
        self.fileList = self.findChild(QtWidgets.QTableWidget, 'fileList')
        # Fill the list:
        self.df = utils.expand_df(df)
        self.fillList()

        # Draw GUI:
@@ -250,16 +269,14 @@ class Ui(QtWidgets.QMainWindow):

    # List helpers:

    def fillList(self, showAllSavedData=False):
    def fillList(self, showAllSavedData:bool = False) -> None:
        header = ['Group', 'Galaxy', 'Processed', 'Ra', 'Dec', 'Filename']
        if not self.has_groups:
            header.remove('Group')
        #self.fileList.setRowCount(len(self.df.index))
        self.fileList.setColumnCount(6)
        self.fileList.setHorizontalHeaderLabels(['Group',
                                                 'Galaxy',
                                                 'Processed',
                                                 'Ra',
                                                 'Dec',
                                                 'Filename'])
        self.fileList.horizontalHeaderItem(2).setTextAlignment(Qt.AlignHCenter)
        self.fileList.setColumnCount(len(header))
        self.fileList.setHorizontalHeaderLabels(header)
        self.fileList.horizontalHeaderItem(2).setTextAlignment(Qt.AlignmentFlag.AlignHCenter)

        # Make table not editable:
        self.fileList.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
@@ -277,32 +294,39 @@ class Ui(QtWidgets.QMainWindow):
        self.fileList.setRowCount(len(self.df.index))

        for i, row in self.df.iterrows():
            # Place void data in integer format:
            self.fileList.setItem(i, 0, QtWidgets.QTableWidgetItem())
            self.fileList.item(i, 0).setData(Qt.DisplayRole, int(row['group']))

            c = 0
            # Place group data in integer format:
            if self.has_groups:
                self.fileList.setItem(i, c, QtWidgets.QTableWidgetItem())
                self.fileList.item(i, c).setData(Qt.DisplayRole, int(row['group']))
                c += 1
            # Place galaxy data in integer format:
            self.fileList.setItem(i, 1, QtWidgets.QTableWidgetItem())
            self.fileList.item(i, 1).setData(Qt.DisplayRole, int(row['galaxy']))

            self.fileList.setItem(i, c, QtWidgets.QTableWidgetItem())
            self.fileList.item(i, c).setData(Qt.DisplayRole, int(row['galaxy']))
            c += 1
            # Set Icon in the row:
            self.fileList.setCellWidget(i, 2, self.getIconCell(row['processed']))

            self.fileList.setCellWidget(i, c, self.getIconCell(row['processed']))
            c += 1
            #Set RA and Dec data:
            self.fileList.setItem(i, 3, QtWidgets.QTableWidgetItem())
            self.fileList.item(i, 3).setData(Qt.DisplayRole, float(row['ra']))
            self.fileList.setItem(i, 4, QtWidgets.QTableWidgetItem())
            self.fileList.item(i, 4).setData(Qt.DisplayRole, float(row['dec']))

            self.fileList.setItem(i, c, QtWidgets.QTableWidgetItem())
            self.fileList.item(i, c).setData(Qt.DisplayRole, float(row['ra']))
            c += 1
            self.fileList.setItem(i, c, QtWidgets.QTableWidgetItem())
            self.fileList.item(i, c).setData(Qt.DisplayRole, float(row['dec']))
            c += 1
            # Place filename in the row:
            self.fileList.setItem(i, 5, QtWidgets.QTableWidgetItem(str(row['filename'])))
            self.fileList.setItem(i, c, QtWidgets.QTableWidgetItem(str(row['filename'])))
            
            if self.has_groups:
                if (not showAllSavedData) and (int(row['group']) not in self.groups):
                    self.fileList.hideRow(i)

        # Order by RA coordinate:
        self.fileList.setSortingEnabled(True)
        if self.has_groups:
            self.fileList.sortItems(3, Qt.AscendingOrder)
        else:
            self.fileList.sortItems(2, Qt.AscendingOrder)

        # Do the resize of the columns by content:
        self.fileList.resizeColumnsToContents()
@@ -319,17 +343,24 @@ class Ui(QtWidgets.QMainWindow):

    # Button methods:

    def save_row(self):
    def save_row(self) -> None:
        index = self.fileList.selectionModel().selectedRows()[0].row()
        try:
            if self.has_groups:
                fn = self.fileList.item(index, 5).text()
            else:
                fn = self.fileList.item(index, 4).text()

            # Identify row by filename (better than index)
            item_index = self.df['filename'] == fn
            
            self.df.loc[item_index, 'processed'] = True
            if self.has_groups:
                self.fileList.setCellWidget(index, 2,
                                            self.getIconCell(True))
            else:
                self.fileList.setCellWidget(index, 1,
                                            self.getIconCell(True))
            
            for id, bg in self.rbg.items():
                selected = ''
@@ -358,28 +389,36 @@ class Ui(QtWidgets.QMainWindow):
            utils.save_df(self.df)

        #except (AttributeError, KeyError):
        except (KeyError):
            print("WARNING:\tEmpty item")
        except (KeyError) as e:
            print(f"WARNING:\tEmpty item. [{e}]")
        #   imgPath = self.defaultImgPath


    def load_row(self):
    def load_row(self) -> None:
        index = self.fileList.selectionModel().selectedRows()[0].row()
        try:
            if self.has_groups:
                fn = self.fileList.item(index, 5).text()
            else:
                fn = self.fileList.item(index, 4).text()
            
            item = self.df.loc[self.df['filename'] == fn]
            self.imgPath = item['fullpath'].item()
            
            windowTitle = ''
            if 'group' in item.keys():
            if self.has_groups:
                grp = str(item['group'].item())
                windowTitle += f"Grp: {grp} | "
            gal = str(item['galaxy'].item())
            windowTitle += f"Gal: {gal} | {self.title}"
            self.setWindowTitle(windowTitle)
            
            if self.has_groups:
                self.fileList.setCellWidget(index, 2,
                                            self.getIconCell(item['processed'].item()))
            else:
                self.fileList.setCellWidget(index, 1,
                                            self.getIconCell(item['processed'].item()))
            
            # radiobuttons
            for id, bg in self.rbg.items():
@@ -410,8 +449,8 @@ class Ui(QtWidgets.QMainWindow):
            # else:
            #     self.commentBox.setPlainText('')

        except (AttributeError, KeyError):
            print("WARNING:\tEmpty item")
        except (AttributeError, KeyError) as e:
            print(f"WARNING:\tEmpty item. [{e}]")
            self.imgPath = self.defaultImgPath
            self.setWindowTitle(self.title)
            for cb in self.cbColumns:
@@ -420,17 +459,17 @@ class Ui(QtWidgets.QMainWindow):
        self.showImage(self.imgPath)


    def next_row(self):
    def next_row(self) -> None:
        index = self.fileList.selectionModel().selectedRows()[0].row()
        self.fileList.selectRow(index+1)


    def prev_row(self):
    def prev_row(self) -> None:
        index = self.fileList.selectionModel().selectedRows()[0].row()
        self.fileList.selectRow(index-1)


    def clear_ButtonGroup(self, bg):
    def clear_ButtonGroup(self, bg:QtWidgets.QButtonGroup) -> None:
        # for i, cb in enumerate(self.rbNames):
        #     self.rb[self.rbTypes[i]].setAutoExclusive(False)
        #     self.rb[self.rbTypes[i]].setChecked(False)
@@ -442,11 +481,13 @@ class Ui(QtWidgets.QMainWindow):
    
    # Actions helpers

    def exit(self):
    def exit(self) -> None:
        QCoreApplication.quit()

    def toggleAIDVisibility(self, act):
    def toggleAIDVisibility(self, act:QtWidgets.QAction) -> None:
        showAllSavedData = act.isChecked()
        if not self.has_groups:
            return
        for i in range(self.fileList.rowCount()):
            row = self.fileList.item(i, 0).text()
            if (not showAllSavedData) and (int(row) not in self.groups):
@@ -456,7 +497,7 @@ class Ui(QtWidgets.QMainWindow):

    # Image helpers:

    def showImage(self, imageFile):
    def showImage(self, imageFile:str) -> None:
        if imageFile and Path.is_file(imageFile):
            pixmap = QPixmap(str(imageFile))
        else:
@@ -467,7 +508,7 @@ class Ui(QtWidgets.QMainWindow):

        self.imageLabel.setPixmap(pixmap.scaled(w, h, Qt.KeepAspectRatio, Qt.SmoothTransformation))

    def getIconCell(self, active):
    def getIconCell(self, active:bool) -> QtWidgets.QWidget:
        iconLabel = QtWidgets.QLabel()
        icon_true = str(Path("res") / Path("icon_true.png"))
        icon_false = str(Path("res") / Path("icon_false.png"))
@@ -485,21 +526,21 @@ class Ui(QtWidgets.QMainWindow):
        cell_widget = QtWidgets.QWidget()
        lay_out = QtWidgets.QHBoxLayout(cell_widget)
        lay_out.addWidget(iconLabel)
        lay_out.setAlignment(Qt.AlignCenter)
        lay_out.setAlignment(Qt.AlignmentFlag.AlignCenter)
        lay_out.setContentsMargins(0, 0, 0, 0)
        cell_widget.setLayout(lay_out)
        return cell_widget

    # Keyboard shortcuts helpers

    def eventFilter(self, obj, event):
    def eventFilter(self, obj:QObject, event:QEvent) -> bool:
        if event.type() == QEvent.KeyPress and obj is self.commentBox:
            if event.key() == (Qt.Key_Return or Qt.Key_Enter) and \
                                self.commentBox.hasFocus():
                self.pb_save.animateClick()
        return super().eventFilter(obj, event)

    def keyPressEvent(self, event):
    def keyPressEvent(self, event:QEvent) -> None:
        key = event.key()

        if key == Qt.Key_F1 and not self.commentBox.hasFocus():
@@ -511,11 +552,11 @@ class Ui(QtWidgets.QMainWindow):

    # Resizing helpers

    def resizeEvent(self, event):
    def resizeEvent(self, event:QEvent) -> None:
        # Re-draw image:
        self.showImage(self.imgPath)

    def splitterResizeEvent(self, pos, index):
    def splitterResizeEvent(self, pos, index) -> None:
        # Re-draw image:
        self.showImage(self.imgPath)

@@ -533,7 +574,7 @@ def addToolTip(widget: QtWidgets.QWidget, element:dict) -> None:
    try:
        widget.setToolTip(f"{element['description']}")
    except KeyError as e:
        print(e)
        print(f"KeyError: {e}")

def addCheckBoxTexts(widget: QtWidgets.QCheckBox, element:dict) -> None:
    id, name, shortcut = None, None, None
@@ -544,7 +585,7 @@ def addCheckBoxTexts(widget: QtWidgets.QCheckBox, element:dict) -> None:
    except KeyError as e:
        if name is None:
            name = id.capitalize()
        print(e)
        print(f"KeyError: {e}")
    finally:
        text = name
        if shortcut is not None:
@@ -561,7 +602,7 @@ def addRadioButtonTexts(widget: QtWidgets.QRadioButton, element:dict) -> None:
    except KeyError as e:
        if name is None:
            name = id.capitalize()
        print(e)
        print(f"KeyError: {e}")
    finally:
        if shortcut is not None:
            widget.setShortcut(shortcut)
+113 −79
Original line number Diff line number Diff line
@@ -30,16 +30,16 @@ def getOptions(version):
    #                    or '--path' in sys.argv
    #                    or '--version' in sys.argv), type=dir_file,
    #                    help="Image or list to classify. Not required if path or path + group are given.\n")
    parser.add_argument('-s', '--savefile',
    parser.add_argument('-s', '--savefile', type=dir_file,
                        #required=not('--version' in sys.argv),
                        default='output.csv',
                        help="CSV file to load and export changes. If does not exists, a new one is created.\n")
    parser.add_argument("-l", "--list", action="store_true",
                        help="List selected files only and exit.\n")
    parser.add_argument('-vf', '--inputfile', default='galaxies.csv',
    parser.add_argument('-i', '--inputfile', default='galaxies.csv', type=dir_file,
                        help="Galaxy database file in *.csv format.\n")
    parser.add_argument('group', metavar='GROUP', type=int, nargs='*',
                        help="Group number. Selects images with name format: *_<group>_*_*.png\n")
                        help="Group number. Selects images with name format: img_<group>_*.png\n")

    args = parser.parse_args()
    VERSION = version
@@ -58,25 +58,34 @@ def getFiles():
    selectedGroups = []
    selectedFiles = pd.DataFrame(columns = groups.columns)

    if args.path:
        if Path(args.inputfile).is_file():
            groups = readInputFile()
    imgpath = args.path
    
    if imgpath:
        fname = args.inputfile
        if Path(fname).is_file():
            groups = readInputFile(fname)
        else:
            groups = createInputFile()
            groups = createInputFile(imgpath, fname)

        availableGroups = None
        if 'group' in groups.columns:
            availableGroups = groups.group.unique()
            print(f"INFO:\tAvailable groups: {str(availableGroups)}")

        
        if len(args.group) > 0:
            for group in args.group:
            inputGroups = args.group
            if len(inputGroups) > 0:
                for group in inputGroups:
                    if group in availableGroups:
                        selectedFiles = pd_concat(selectedFiles, groups[groups.group == group])
                        selectedGroups.append(int(group))
                    else:
                        print(f'WARNING:\tGroup {group} not available.')
            else:
            print(f'INFO:\tNo subgroup selected. Using all available by default.')
                print(f'INFO:\tNo group selected. Using all available by default.')
                selectedFiles = groups.copy()
                selectedGroups = availableGroups
        else:
            print(f'INFO:\tNo group in input file. Using all galaxies by default.')
            selectedFiles = groups.copy()
            selectedGroups = availableGroups

@@ -98,27 +107,22 @@ def getFiles():

def dir_path(path):
    d = Path(path)
    if d.is_dir():
    if not d.is_dir():
        raise argparse.ArgumentTypeError(f"readable_dir: {path} is not a valid path.")
    return path
    else:
        raise argparse.ArgumentTypeError("readable_dir:" + path
                                         + " is not a valid path.")


def dir_file(file):
    d = Path(file)
    if d.is_file():
    if not d.is_file():
        raise argparse.ArgumentTypeError(f"readable_file: {file} is not a valid file.")
    return file
    else:
        raise argparse.ArgumentTypeError("readable_file:" + file
                                         + " is not a valid file.")

VFCOLUMNS = ['group','galaxy','ra','dec','filename']

def readInputFile():
    fname = args.inputfile
    print(f'INFO:\tReading from {fname} file... ', end='', flush=True)
    groups = pd.read_csv(fname,
INPUTCOLUMNS = ['group','galaxy','ra','dec','filename']

def readInputFile(fname:str) -> pd.DataFrame:
    print(f"INFO:\tReading from '{fname}' file... ", end='', flush=True)
    df = pd.read_csv(fname,
                     converters={
                         'group': int,
                         'galaxy': int,
@@ -126,16 +130,21 @@ def readInputFile():
                         'dec': float,
                         }
                    )
    groups = groups.sort_values(by=['group', 'ra', 'galaxy'])
    sortby=[]
    if 'galaxy' in df.columns:
        sortby.insert(0, 'galaxy')
    if 'group' in df.columns:
        sortby.insert(0, 'group')
    df = df.sort_values(by=sortby)
    
    print('Done!')
    return groups
    return df

def createInputFile():
    fname = args.inputfile
def createInputFile(imgpath:str, fname:str) -> pd.DataFrame:
    print(f'INFO:\tCreating {fname} file... ', end='', flush=True)
    groups = pd.DataFrame(columns=VFCOLUMNS)
    if args.path:
        for file in Path(args.path).glob('pan_*_*_ppak.png'):
    df = pd.DataFrame(columns=INPUTCOLUMNS)
    if imgpath:
        for file in Path(imgpath).glob('img_*_*.png'):
            entry = {
                        'group': int(file.stem.split('_')[1]),
                        'galaxy': int(file.stem.split('_')[2]),
@@ -143,43 +152,57 @@ def createInputFile():
                        'dec':float(0),
                        'filename': str(file.name),
                    }
            groups = pd_concat(groups, entry)
            df = pd_concat(df, entry)
            #groups = groups.append(entry, ignore_index=True)
    groups = groups.sort_values(by=['group','galaxy'])
    groups.to_csv(fname, columns=VFCOLUMNS, index=False)
    # sort
    if sum(df["groups"] == '-') == len(df):
        df.drop("groups", axis=1, inplace=True)
        INPUTCOLUMNS.remove("groups")
    
    sortby=[]
    if 'galaxy' in df.columns:
        sortby.insert(0, 'galaxy')
    if 'group' in df.columns:
        sortby.insert(0, 'group')
    df = df.sort_values(by=sortby)
    
    df.to_csv(fname, columns=INPUTCOLUMNS, index=False)
    print('Done!')
    return groups
    return df


### PANDAS UTILS


COLUMNS = ["filename", "group", "galaxy", "morphology", "large", "tiny", "faceon",
           "edgeon", "star", "calibration", "recentre", "duplicated",
           "member", "hiiregion", "yes", "no", "comment", "processed",
           "fullpath", "ra", "dec"]

MORPHOLOGY = ['elliptical', 'spiral', 'irregular', 'other', '']

IDS = {
    'FILE':[],
    'TB':[],
    'CB':[],
    'RB':[],
    'RBG':[]
}

def getColumns():
    return COLUMNS

def getExportableColumns():
    return COLUMNS[:-4]

def getMorphology():
    return MORPHOLOGY


def getRadioButtonsMorphology():
    return MORPHOLOGY[:-1]
def getRadioButtonGroups():
    return IDS['RBG']

def getRadioButtonsNames():
    return IDS['RB'][:-1]

def getCheckBoxesColumns():
    return COLUMNS[4:-5]
    return IDS['CB']


def getExportableColumns():
    return COLUMNS[:-4]
def getTextBoxes():
    return IDS['TB']


def checkColumnsMismatch(importDataColumns):
@@ -239,8 +262,8 @@ def expand_df(selectedFiles):
                importData.loc[importData.galaxy == row.galaxy, 'ra'] = ra
                importData.loc[importData.galaxy == row.galaxy, 'dec'] = dec

        except KeyError:
                print("ERROR:\tError while parsing CSV.")
        except KeyError as e:
                print(f"ERROR:\tError while parsing CSV. [{e}]")

    else:
        importData = pd.DataFrame(columns=COLUMNS)
@@ -250,21 +273,28 @@ def expand_df(selectedFiles):
    return importData


def newEntry(row):
def newEntry(row:pd.Series) -> dict:
    file = Path(args.path) / Path(row['filename'])
    entry = {
        'filename': file.name,
        'group': row.group,
        'galaxy': row.galaxy,
        'morphology': MORPHOLOGY[-1],
        'filename': file.name,
    }

    if 'group' in row:
        entry.update({'group': row.group})
        
    # ID needed? to separate widget groups?
    for i, rbgCol in enumerate(getRadioButtonGroups()):
        entry.update({rbgCol: getRadioButtonsNames()[-1]}) # default value 

    for i, cbCol in enumerate(getCheckBoxesColumns()):
        entry.update({cbCol: False})

    for i, tbCol in enumerate(getTextBoxes()):
        entry.update({tbCol: ''})

    entry.update(
        {
            'comment': '',
            'processed': False,
            'fullpath': file.absolute(),
            'ra': row.ra,
@@ -274,7 +304,7 @@ def newEntry(row):
    return entry


def save_df(df):
def save_df(df:pd.DataFrame) -> None:
    global args
    global importData

@@ -285,7 +315,10 @@ def save_df(df):
    #                            df.loc[df['processed'] == True]])

    # Remove old values, keep last ones:
    if 'group' in processedItems.columns:
        exportData = processedItems.drop_duplicates(['group','galaxy'], keep='last').sort_values('group')
    else:
        exportData = processedItems.drop_duplicates(['galaxy'], keep='last').sort_values('galaxy')

    # Export final dataframe:
    exportData.to_csv(args.savefile, columns=getExportableColumns(),
@@ -311,6 +344,7 @@ def pd_concat(df: pd.DataFrame, data: Union[pd.DataFrame, list, dict]) -> pd.Dat
        pandas.DataFrame
        """
        # check if data is list
        df_data = pd.DataFrame()
        if type(data) == list:
            if len(data) != len(df.columns):
                raise Exception('ERROR: Input data [list] length is not equal to input dataframe')