Example of how to save Spark RDDs to disk using GZip compression in response to https://twitter.com/rjurney/status/533061960128929793.

spark_gzip.py

from pyspark import SparkContext


def main():
    sc = SparkContext(appName="Test Compression")
    # RDD has to be key, value pairs
    data = sc.parallelize([
        ("key1", "value1"),
        ("key2", "value2"),
        ("key3", "value3"),
    ])

    data.saveAsHadoopFile("/tmp/spark_compressed",
                          "org.apache.hadoop.mapred.TextOutputFormat",
                          compressionCodecClass="org.apache.hadoop.io.compress.GzipCodec")
    sc.stop()


if __name__ == "__main__":
    main()