آموزش اسپارک- tutorial3- کریش نایک

pip install pyspark

from pyspark.sql import SparkSession
spark=SparkSession.builder.appName('Practise').getOrCreate()

df_pyspark=spark.read.csv('table.csv',header=True,inferSchema=True)

df_pyspark.printSchema()

df_pyspark.show()

##drop the columns
df_pyspark.drop('Name').show()

df_pyspark.show()

df_pyspark.na.drop().show()

### any==how
df_pyspark.na.drop(how="any").show()

##threshold-- 
df_pyspark.na.drop(how="any",thresh=1).show()

##Subset-- delet nan values only from the subset column
df_pyspark.na.drop(how="any",subset=['Age']).show()

### Filling the Missing Value
df_pyspark.na.fill('Missing Values',['Experience','age']).show()

# like the same in sklearn library
from pyspark.ml.feature import Imputer

imputer = Imputer(
    inputCols=['age', 'Experience', 'Salary'], 
    outputCols=["{}_imputed".format(c) for c in ['age', 'Experience', 'Salary']]
    ).setStrategy("median")

imputer.fit(df_pyspark).transform(df_pyspark).show()