Python Script to dump or load data using redis

redis_dump.py

"""
Install the following requirements into your virtual environemnt 
`pip install click redis`
Usage:

To load data into redis
python redis_dump.py load [filepath]

To dump data into redis
python redis_dump.py dump [filepath] --search '*txt'
"""

import click
import redis
import json
import logging
import os


@click.command()
@click.argument('action')
@click.argument('filepath')
@click.option('--search', help="Key search patter. eg `*txt`")
def main(action, filepath, search):
    r = redis.StrictRedis(host='127.0.0.1', port=6379, db=0)  # update your redis settings
    cache_timeout = None
    if action == 'dump':
        out = {}
        for key in r.scan_iter(search):
            out.update({key: r.get(key)})
        if len(out) > 0:
            try:
                with open(filepath, 'w') as outfile:
                    json.dump(out, outfile)
                    print('Dump Successful')
            except Exception as e:
                print(e)
        else:
            print("Keys not found")

    elif action == 'load':
        try:
            with open(filepath) as f:
                data = json.load(f)
                for key in data:
                    r.set(key, data.get(key), cache_timeout)
                print('Data loaded into redis successfully')
        except Exception as e:
            print(e)


if __name__ == '__main__':    
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)
    main()

Thanks for sharing
I think there are faster load approaches available (If load speed is important for the task):

• the fastest way would be to use redic-cli (I was getting about 150k OPS on local docker setup)
• python with redis pipeline on the same machine was giving me about 5k OPS

here is the code snippet

    def process_file(self,file_path):
        chunksize = 10000
        pipe = self.redis.pipeline()
        with open(file_path,"r") as process_f:
            line_index = 0
            for line in process_f:
                try:
                    line_index += 1
                    first_name, last_name = line.strip().split(' ')
                    if not last_name:
                        next
                    user = first_name.strip() + '.' + last_name.strip()       
                    pipe.sadd('users', user.lower())
                    if line_index % chunksize == 0:
                        pipe.execute()
                        logging.info("Processing index %s", line_index)
                except: 
                    logging.info("Something else went wrong")
                finally:
                    pipe.execute() 
            logging.info("Number of records %s", self.conn.scard('users'))

r.get(key) won't work with value types other than string.