Restore database with the last backup in develop

restore.py

import math
import os
import subprocess
import sys
import time

import boto3

BUCKET = 'erpnext-backups-zeb'


def convert_size(size_bytes):
    if size_bytes == 0:
        return "0B"
    size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
    i = int(math.floor(math.log(size_bytes, 1024)))
    p = math.pow(1024, i)
    s = round(size_bytes / p, 2)
    return "%s %s" % (s, size_name[i])

def get_last_backup_s3(s3_client=None):
    if not s3_client:
        s3_client = boto3.client('s3')
    response = s3_client.list_objects_v2(Bucket=BUCKET, Prefix='development/erp-development.zebrands.mx/minimal_backup/')

    # get last file from the list
    files = response.get("Contents", [])
    files.sort(key=lambda x: x.get("LastModified"))
    last_file = files[-1].get("Key")
    print(f"Last file found in s3 {last_file}")
    return last_file

def download_backup_from_s3():
    print("Trying to download backup from s3")
    s3_client = boto3.client('s3')
    last_file = get_last_backup_s3(s3_client=s3_client)

    file_name = last_file.split("/")[-1]

    meta_data = s3_client.head_object(Bucket=BUCKET, Key=last_file)

    total_length = int(meta_data.get('ContentLength', 0))
    downloaded = 0

    def progress(chunk):
        nonlocal downloaded
        downloaded += chunk
        done = int(50 * downloaded / total_length)

        sys.stdout.write("%s[%s%s] %s/%s\r" % ('', "#" * done, "." * (50 - done), convert_size(downloaded), convert_size(total_length)))
        sys.stdout.flush()

    print(f'Downloading {file_name}')
    with open(file_name, 'wb') as f:
        s3_client.download_fileobj(BUCKET, last_file, f,
                                   Callback=progress)

    print("Downloaded backup from s3")
    return file_name


def verify_pv_installed():
    try:
        subprocess.run(["pv", "--version"])
    except FileNotFoundError:
        subprocess.run(["sudo", "apt-get", "install", "pv", "-y", "-qq"])



if __name__ == "__main__":
    start_time = time.time()
    verify_pv_installed()
    backup_name = 'min_20241231_230023-erp-development_zebrands_mx-database.sql.gz'
    backup_path = os.path.abspath(backup_name)
    subprocess.run(f"bench restore {backup_path} --mariadb-root-username root --mariadb-root-password 123 --admin-password 123", shell=True)
    print("--- %s seconds ---" % (time.time() - start_time))

package main

import (
	"bufio"
	"compress/gzip"
	"flag"
	"log"
	"os"
	"strings"
)

func main() {
	// Parse command-line arguments
	inputFileName := flag.String("input", "", "Input gzip-compressed SQL file name")
	flag.Parse()

	if *inputFileName == "" {
		log.Fatal("Input file name is required")
	}
	// List of strings to remove
	removedTabs := []string{
		"INSERT INTO `tabVersion",
		"INSERT INTO `tabInteraction",
		"INSERT INTO `tabPaymentMachine Log",
		"INSERT INTO `tabPayment Schedule",
		"INSERT INTO `tabRaw Event",
		"INSERT INTO `tabActivity Log",
		"INSERT INTO `tabStaffing Plan",
		"INSERT INTO `tabRaw Segment Event",
		"INSERT INTO `tabEmployee Onboarding",
		"INSERT INTO `tabRaw Event",
		"INSERT INTO `tabReschedule Log",
		"INSERT INTO `tabEvent Ack",
		"INSERT INTO `tabSubscriber Event Log",
		"INSERT INTO `tabQuotation Status Tracking",
		"INSERT INTO `tabCase Status Tracking",
		"INSERT INTO `tabOpportunity Status Tracking",
		"INSERT INTO `tabCommunication",
		"INSERT INTO `tabZeus App Notifications",
		"INSERT INTO `tabRaw Sendgrid Event",
		"INSERT INTO `tabAccess Log",
		"INSERT INTO `tabReturn Machine Log",
		"INSERT INTO `tabError Log",
		"INSERT INTO `tabView Log",
		"INSERT INTO `tabZeCore Shipping Communication Log",
		"INSERT INTO `tabActivity Log",
		"INSERT INTO `tabRoute History",
		"INSERT INTO `gl_map",
		"INSERT INTO `tabScheduled Job Log",
		"INSERT INTO `tabRaw Twilio WhatsApp Event",
		"INSERT INTO `tabAnnual Development Memorandum",
		"INSERT INTO `tabStores Sellout",
		"INSERT INTO `tabScheduled Job",
		"INSERT INTO `tabEmail Queue",
		"INSERT INTO `tabComment",
		"INSERT INTO `tabCommunication",
		"INSERT INTO `tabStaffing Plan Detail",
		"INSERT INTO `__global_search",
		"INSERT INTO `tabData Import",
		"INSERT INTO `tabMarketplace Error Log",
		"INSERT INTO `tabKey Event Log",
		"INSERT INTO `tabLog Tpl to Wms",
		"INSERT INTO `tabJob Offer",
		"INSERT INTO `tabRaw Yotpo Event",
		"INSERT INTO `tabShipping Machine Log",
		"INSERT INTO `tabRefund Machine Log",
		"INSERT INTO `tabShipping Core Item Change History",
		"INSERT INTO `tabClub Premier Dollar",
		"INSERT INTO `tabInterview Feedback Form",
		"INSERT INTO `tabRaw Scheduled Event",
		"INSERT INTO `tabJob Applicant",
		"INSERT INTO `tabError Snapshot",
		"INSERT INTO `tabAccess Log",
		"INSERT INTO `tabData Import",
		"INSERT INTO `tabEmployee Onboarding Template",
		"INSERT INTO `tabPaymentMachine Log",
		"INSERT INTO `tabState Transition Notification",
		"INSERT INTO `tabPayment Machine Log",
		"INSERT INTO `tabClub Premier Redeem",
		"INSERT INTO `tabStores Sellout",
		"INSERT INTO `tabZeus App Notifications",
		"INSERT INTO `tabRoute History",
		"INSERT INTO `tabAppraisal",
		"INSERT INTO `tabAnnual Development Memorandum",
		"INSERT INTO `tabTL Evaluation",
		"INSERT INTO `tabDeleted Document",
		"INSERT INTO `tabClub Premier Dollar",
		"INSERT INTO `tabLog Setting User",
		"INSERT INTO `tabShipping Core Item Change History",
		"INSERT INTO `tabLog TPL Middleware To WMS",
		"INSERT INTO `tabRefund Machine Log",
		"INSERT INTO `tabComment",
		"INSERT INTO `tabError Log",
		"INSERT INTO `tabZeCoreShippingCommunicationLogOLD",
		"INSERT INTO `tabInteraction",
		"INSERT INTO `tabLog Wms to Shipping",
		"INSERT INTO `tabVersion",
		"INSERT INTO `tabCase Interaction",
		"INSERT INTO `tabPayment Schedule",
		"INSERT INTO `tabView Log",
		"INSERT INTO `tabShipping Machine Log",
		"INSERT INTO `tabJob Opening",
		"INSERT INTO `tabJob Applicant",
		"INSERT INTO `tabEmail Queue",
		"INSERT INTO `tabRaw Firebase Event",
		"INSERT INTO `tabJob Aplicant Interview",
		"INSERT INTO `tabJob Offer",
		"INSERT INTO `tabStaffing Plan",
		"INSERT INTO `tabStaffing Plan Detail",
		"INSERT INTO `tabInterview Feedback Form",
		"INSERT INTO `tabEmployee Onboarding",
		"INSERT INTO `tabEmployee Onboarding Template",
		"INSERT INTO `tabReschedule Log",
		"INSERT INTO `tabRaw Scheduled Event",
		"INSERT INTO `tabState Transition",
		"INSERT INTO `tabMarketplace Error Log",
		"INSERT INTO `tabTL Evaluation",
		"INSERT INTO `tabLog Wms to Return",
		"INSERT INTO `tabZeCore Shipping Communication Log",
		"INSERT INTO `tabLog Inhouse to Wms",
		"INSERT INTO `tabReturn Machine Log",
		// Add more strings here as needed
	}

	// Open the input gzip-compressed SQL file
	inputFile, err := os.Open(*inputFileName)
	if err != nil {
		log.Fatalf("Error opening input file: %v", err)
	}
	defer inputFile.Close()

	// Create a gzip reader to read the input file
	log.Println("Create a gzip reader to read the input file")
	gzipReader, err := gzip.NewReader(inputFile)
	if err != nil {
		log.Fatalf("Error creating gzip reader: %v", err)
	}
	defer gzipReader.Close()

	// Create the output gzip-compressed file
	outputFile, err := os.Create("min_" + *inputFileName)
	if err != nil {
		log.Fatalf("Error creating output file: %v", err)
	}
	defer outputFile.Close()

	// Create a gzip writer to write to the output file
	gzipWriter := gzip.NewWriter(outputFile)
	defer gzipWriter.Close()

	// Create a scanner to read the input gzip reader line by line
	scanner := bufio.NewScanner(gzipReader)

	// Set the maximum token size to a large value
	const maxTokenSize = 1024 * 1024 * 100 // 64 KB
	buf := make([]byte, maxTokenSize)
	scanner.Buffer(buf, maxTokenSize)
	log.Println("Init scanner...")
	for scanner.Scan() {
		line := scanner.Text()
		shouldRemove := false

		// Check if the line contains any of the strings to remove
		for _, tab := range removedTabs {
			if strings.Contains(line, tab) {
				shouldRemove = true
				break
			}
		}

		// If the line does not contain any of the strings to remove, write it to the output gzip writer
		if !shouldRemove {
			_, err := gzipWriter.Write([]byte(line + "\n"))
			if err != nil {
				log.Fatalf("Error writing to output file: %v", err)
			}
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatalf("Error scanning input file: %v", err)
	}

	log.Println("Backup file with removed lines created successfully.")
}