Run multiple notebooks in parallel as ephemeral jobs using python threads

parallel-notebooks.py

from threading import Thread

def producer_method():
  dbutils.notebook.run(
    path="./kinesis-producer", 
    timeout_seconds=600, 
    arguments={
      "kinesisRegion": KINESIS_REGION, 
      "inputStream": INPUT_STREAM, 
      "newsgroupDataLocation": NEWSGROUP_DATA_PATH
    }
  )
  
def consumer_method():
  dbutils.notebook.run(
    path="./kinesis-consumer", 
    timeout_seconds=600, 
    arguments={
      "kinesisRegion": KINESIS_REGION, 
      "relayStream": RELAY_STREAM
    }
  )
  
producer_thread = Thread(target=producer_method)
consumer_thread = Thread(target=consumer_method)

producer_thread.start()
consumer_thread.start()

producer_thread.join()
consumer_thread.join()