Python Spark: Union, Distinct, Sort, and Save to File

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName('UnionAndDistinct').getOrCreate()

line1 = spark.read.text('A.txt').rdd line2 = spark.read.text('B.txt').rdd

lines = line1.union(line2) distinct_lines = lines.distinct() result = distinct_lines.sortBy(lambda x: x) result = result.coalesce(1) result.saveAsTextFile('result')