pyspark 要处理数据，没有，那就伪造数据 faker 真是个好东西

from faker import Faker
import pandas as pd
gender = ["None","Man","Woman"]fake = Faker()
names = [(fake.first_name(),fake.last_name(),fake.date_of_birth(),fake.random_int(1,99),gender[fake.random_int(1,2)]) for _ in range(1000)]pd.DataFrame(names, columns=["first_name","last_name","birthday","age","gender"]).to_csv("fake_names.csv", index=False)

from pyspark.sql import SparkSession
#字符串处理函数
spark = SparkSession.builder.appName('example').getOrCreate()
df =spark.read.csv("fake_names.csv",header=True,inferSchema=True)
# df.show(3)

1、拼接字符串 concat()  concat_ws()  format_string()
 

from pyspark.sql.functions import concat,concat_ws,format_stringdf.select("first_name","last_name","age",concat(df["first_name"],df['last_name']).alias("concat_result"),concat_ws('#',df["first_name"],df['last_name']).alias("concat_ws_result"),format_string("姓%s名%s年龄%d",df["first_name"],df['last_name'],df['age']).alias("format_string_result")).show(5)

2、字符串的长度和转换大小写 length() lower() upper()
 

from pyspark.sql.functions import length,lower,upperdf.select("first_name",length(df["first_name"]).alias("length_result"),lower(df["first_name"]).alias("lower_result"),upper(df["first_name"]).alias("upper_result")
).show(5)

3、添加常量值到新的列中
 

from pyspark.sql.functions import litdf.withColumn("身份",lit('正式员工')).show(3)
df.withColumn("补贴金额",lit('300')).show(3)df.printSchema()df.withColumn("身份",lit('正式员工')).withColumn("补贴金额",lit('300')).printSchema()

4、去除空格
 

from pyspark.sql.functions import trim,ltrim,rtrim,coldf_new = df.withColumn("new_name",concat(lit(" "),df['first_name'],lit(" ")))df_new.show()
df_new.select("new_name",format_string("#%s#",col("new_name")).alias("source"),format_string("#%s#", ltrim(col("new_name"))).alias("ltrim_result"),format_string("#%s#", rtrim(col("new_name"))).alias("rtrim_result"),format_string("#%s#", trim(col("new_name"))).alias("trim_result")).show(5)

5、正则提取
 

from pyspark.sql.functions import regexp_extractdf.select('birthday',regexp_extract(df['birthday'],r'(\d+)-(\d+)-(\d+)',1).alias('year'),regexp_extract(df['birthday'],r'(\d+)-(\d+)-(\d+)',2).alias('month'),regexp_extract(df['birthday'],r'(\d+)-(\d+)-(\d+)',3).alias('day')).show(5)

6、正则替换
 

from pyspark.sql.functions import regexp_replacedf_new = df.withColumn("姓名",format_string("姓%s 名%s",df['first_name'],df['last_name']))
df_new.show(5)df_new.withColumn("清理姓名",regexp_replace(df_new['姓名'],r'姓|名| ','')).show(5)

7、提取字符串的子串
 

from pyspark.sql.functions import substringdf.select('first_name','birthday',substring(df['birthday'],0,4).alias('year'),substring(df['birthday'],6,2).alias('month'),substring(df['birthday'],-2,2).alias('day'),
).show(5)

8、字符串拆分
 

from pyspark.sql.functions import split,sizedf_new= df.select('birthday',split(df['birthday'],'-').alias('splits'),
)
df_new.show(5)
df_new.printSchema()df_new.select('birthday','splits',df_new['splits'].getItem(0).alias("year"),df_new['splits'].getItem(1).alias("month"),df_new['splits'].getItem(2).alias("day"),size(df_new['splits']).alias('size')
).show(5)