update_source.py

@classmethod
def update_source(self, data_source_rows_by_guid, update_data_source_groups):
    histories_to_save = []
    for guid, datasources in update_data_source_groups.items():
        user_edited_entries_values = defaultdict(list)
        for history in DataSourceHistory.objects.filter(datasource_id__in=[ds.id for ds in datasources],
            is_user_action=True).distinct('row_id'):
            user_edited_entries_values[history.datasource_id].append(history.values)

        for ds in datasources:
            map_columns = ds.meta['map_columns']
            hidden_column_guid = map_columns['hidden_column_guid']
            entries = copy.deepcopy(data_source_rows_by_guid[guid])

            user_edited_row_identifier = [values[hidden_column_guid] for values in user_edited_entries_values[ds.id]]
            for values in entries:
                identifier = values['hidden_column_guid']
                if identifier in user_edited_row_identifier:
                    entries.pop(identifier)

            row_qs = DataSourceRow.objects.order_by('order').filter(datasource=ds)

            last_order = row_qs.last().order + 1 if row_qs else 1

            filter_query = {
                'values__{}__in'.format(hidden_column_guid): [entry['hidden_column_guid'] for entry in entries]
            }

            p_filter_query = {
                'p_values__{}__in'.format(hidden_column_guid): [entry['hidden_column_guid'] for entry in entries]
            }
            rows = row_qs.filter(Q(**filter_query) | Q(**p_filter_query))

            for row in rows:
                row_identifier = row.values[hidden_column_guid]
                entry_values = [entry for entry in entries if entry['hidden_column_guid'] == row_identifier][0]

                values = {map_columns[key]: value for key, value in entry_values.items()}

                if ds.meta['source']['as_approved']:
                    row.status = DataSourceRow.APPROVED
                    row.values = values
                else:
                    row.status = DataSourceRow.P_EDIT
                    row.p_value = values

                row.version += 1
                create_at = datetime.now().isoformat()
                row.history = {
                    'actor_id': ds.guid,
                    'actor_name': ds.inode.name,
                    'is_user_action': False,
                    'created_at': create_at,
                    'action': DataSourceHistory.EDIT,
                    'p_values': row.values,
                    'values': row.p_values
                }
                row.save()

                histories_to_save.append(
                    DataSourceHistory.new_history(
                        row,
                        ds.guid,
                        ds.inode.name,
                        create_at,
                        DataSourceHistory.EDIT,
                        False
                    )
                )
                entries = [entry for entry in entries if not entry['hidden_column_guid'] == row_identifier]

            new_items = ds.meta['source']['new_items']

            if new_items or len(row_qs) == 0:
                rows_to_save = []
                for entry in entries:
                    row_identifier = entry['hidden_column_guid']
                    values = {map_columns[key]: value for key, value in entry.items()}
                    values.update({hidden_column_guid: row_identifier})

                    rows_to_save.append(
                        DataSourceRow(
                            datasource=ds,
                            order=last_order,
                            version=ds.version,
                            status=DataSourceRow.APPROVED if ds.meta['source'][
                                'as_approved'] else DataSourceRow.P_INSERT,
                            values=values if ds.meta['source']['as_approved'] else {},
                            p_values={} if ds.meta['source']['as_approved'] else values,
                            history={
                                'actor_id': ds.guid,
                                'actor_name': ds.inode.name,
                                'is_user_action': False,
                                'created_at': datetime.now().isoformat(),
                                'action': DataSourceHistory.CREATE,
                                'p_values': values if ds.meta['source']['as_approved'] else {},
                                'values': {} if ds.meta['source']['as_approved'] else values,
                            }
                        )
                    )

                    last_order += 1

                new_rows = DataSourceRow.objects.bulk_create(rows_to_save)

                for row in new_rows:
                    row.guid = guids.encode(guids.DATA_SOURCE_ROW, row.id)
                    row.save()

                    histories_to_save.append(
                        DataSourceHistory.new_history(
                            row,
                            ds.guid,
                            ds.inode.name,
                            row.history['created_at'],
                            DataSourceHistory.CREATE,
                            False
                        )
                    )

    DataSourceHistory.objects.bulk_create(histories_to_save)