{"id":47156,"date":"2022-11-05T05:09:13","date_gmt":"2022-11-20T09:02:33","guid":{"rendered":"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/"},"modified":"2024-04-28T23:06:36","modified_gmt":"2024-04-28T15:06:36","slug":"databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8","status":"publish","type":"post","link":"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/","title":{"rendered":"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868"},"content":{"rendered":"

\u8fd9\u7bc7\u6587\u7ae0\u7684\u5185\u5bb9\u57fa\u4e8e\u5728Data+AI World Tour 2021\u4e3e\u884c\u7684\u6280\u672f\u7814\u8ba8\u4f1a\u201c\u6253\u9020\u7b80\u5355\u6613\u61c2\u7684\u6570\u636e\u6d41\u6c34\u7ebf\u201d\u5b9e\u65bd\u7684\u5185\u5bb9\u3002<\/p>\n

\u8fd9\u7bc7\u6587\u7ae0\u6d89\u53ca\u7684\u4ee3\u7801\u793a\u4f8b\u7b14\u8bb0\u672c<\/p>\n

\u8bfb\u53d6CSV\u6587\u4ef6<\/h1>\n
\"undefined\"<\/div>\n

CSV\u6587\u4ef6 => \/databricks-datasets\/lending-club-loan-stats\/*.csv<\/p>\n

# \u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306bCSV\u3092\u8aad\u307f\u8fbc\u3080 (\u30b9\u30ad\u30fc\u30de\u306f\u63a8\u5b9a\u3059\u308b)\r\n<\/span>df<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'csv'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'Header'<\/span>,<\/span> True<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'inferSchema'<\/span>,<\/span> True<\/span>)<\/span>\r\n  .<\/span>load<\/span>(<\/span>'\/databricks-datasets\/lending-club-loan-stats\/*.csv'<\/span>)<\/span> \r\n)<\/span>\r\n\r\n# \u65e2\u5b58\u306eDelta\u306b\u8ffd\u8a18\r\n<\/span>df<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'append'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stats.delta'<\/span>)<\/span>\r\n\r\n# \u66f8\u304d\u8fbc\u3093\u3060Delta\u30c6\u30fc\u30d6\u30eb\u306e\u78ba\u8a8d\r\n<\/span>df_delta<\/span> =<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stats.delta'<\/span>)<\/span>\r\ndisplay<\/span>(<\/span>df_delta<\/span>)<\/span>\r\n<\/code><\/pre>\n
ETL\u6570\u636e\u6574\u7406<\/h2>\n
\"undefined\"<\/div>\n
\r\n# \u7c21\u5358\u306a\u51e6\u7406\u3092\u884c\u3044\u3001\u7d50\u679c\u3092\u4fdd\u5b58\r\n<\/span>from<\/span> pyspark.sql.functions<\/span> import<\/span> col<\/span>,<\/span> expr<\/span>\r\n\r\ndf_raw<\/span> =<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stats.delta'<\/span>)<\/span>\r\n\r\n(<\/span>\r\n  df_raw<\/span>\r\n  .<\/span>select<\/span>(<\/span>'loan_amnt'<\/span>,<\/span> # \u5fc5\u8981\u306a\u30ab\u30e9\u30e0\u306e\u62bd\u51fa\r\n<\/span>            'term'<\/span>,<\/span>\r\n            'int_rate'<\/span>,<\/span>\r\n            'grade'<\/span>,<\/span>\r\n            'addr_state'<\/span>,<\/span>\r\n            'emp_title'<\/span>,<\/span>\r\n            'home_ownership'<\/span>,<\/span>\r\n            'annual_inc'<\/span>,<\/span>\r\n            'loan_status'<\/span>)<\/span>\r\n  .<\/span>withColumn<\/span>(<\/span>'int_rate'<\/span>,<\/span> expr<\/span>(<\/span>'cast(replace(int_rate,\"%\",\"\") as float)'<\/span>))<\/span> # \u30c7\u30fc\u30bf\u578b\u306e\u5909\u63db\r\n<\/span>  .<\/span>withColumnRenamed<\/span>(<\/span>'addr_state'<\/span>,<\/span> 'state'<\/span>)<\/span> # \u30ab\u30e9\u30e0\u540d\u5909\u66f4\r\n<\/span>  .<\/span>write<\/span>\r\n  .<\/span>format<\/span>(<\/span>'delta'<\/span>)<\/span> #\u3000Delta\u3067\u4fdd\u5b58(silver\u30c6\u30fc\u30d6\u30eb)\r\n<\/span>  .<\/span>mode<\/span>(<\/span>'overwrite'<\/span>)<\/span>\r\n  .<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_silver.delta'<\/span>)<\/span>\r\n)<\/span>\r\n\r\ndisplay<\/span>(<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_silver.delta'<\/span>)<\/span> )<\/span>\r\n<\/code><\/pre>\n
\u6211\u4e5f\u80fd\u4f7f\u7528SQL\uff0c\u5e76\u4e14\u8fd8\u53ef\u4ee5\u76f4\u63a5\u8fdb\u884c\u53ef\u89c6\u5316\u64cd\u4f5c\u3002<\/h2>\n
%<\/span>sql<\/span>\r\n\r\nDROP<\/span> DATABASE<\/span> IF<\/span> EXISTS<\/span> daiwt2021_kitamura<\/span> CASCADE<\/span>;<\/span>\r\nCREATE<\/span> DATABASE<\/span> daiwt2021_kitamura<\/span>;<\/span>\r\nUSE<\/span> daiwt2021_kitamura<\/span>;<\/span>\r\n\r\nCREATE<\/span> TABLE<\/span> loan_stat_silver<\/span>\r\nUSING<\/span> delta<\/span>\r\nLOCATION<\/span> '\/tmp\/daiwt2021\/loan_stat_silver.delta'<\/span>;<\/span>\r\n\r\nSELECT<\/span> state<\/span>,<\/span> loan_status<\/span>,<\/span> count<\/span>(<\/span>*<\/span>)<\/span> as<\/span> counts<\/span> \r\nFROM<\/span> loan_stat_silver<\/span>\r\nGROUP<\/span> BY<\/span> state<\/span>,<\/span> loan_status<\/span>\r\nORDER<\/span> BY<\/span> counts<\/span> DESC<\/span>\r\n<\/code><\/pre>\n
\"undefined\"<\/div>\n
\u4f7f\u7528Python\u63a5\u6536SQL\u7684\u7ed3\u679c\uff0c\u5e76\u521b\u5efa\u4e00\u4e2a\u6458\u8981\u8868\u3002<\/h2>\n
\"undefined\"<\/div>\n
df_gold<\/span> =<\/span> spark<\/span>.<\/span>sql<\/span>(<\/span>'''\r\n\r\nSELECT state, loan_status, count(*) as counts \r\nFROM loan_stat_silver\r\nGROUP BY state, loan_status\r\nORDER BY counts DESC\r\n\r\n'''<\/span>)<\/span>\r\n\r\n# Delta\u3067\u4fdd\u5b58(gold\u30c6\u30fc\u30d6\u30eb)\r\n<\/span>df_gold<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_gold.delta'<\/span>)<\/span>\r\n\r\n# gold\u30c6\u30fc\u30d6\u30eb\u3092Hive\u3078\u767b\u9332\r\n<\/span>spark<\/span>.<\/span>sql<\/span>(<\/span>'''\r\n  CREATE TABLE IF NOT EXISTS loan_stat_gold\r\n  USING delta\r\n  LOCATION '\/tmp\/daiwt2021\/loan_stat_gold.delta'\r\n'''<\/span>)<\/span>\r\n<\/code><\/pre>\n
\u4eceBI\u4e2d\u5f15\u7528Delta\u8868<\/h2>\n
\"undefined\"<\/div>\n
\u4f7f\u7528BI\u5de5\u5177\u8fde\u63a5\u5230Delta\u8868\u7684JDBC\/ODBC\u7aef\u70b9\u3002<\/p>\n
\"undefined\"<\/div>\n
\u5c06Delta\u8868\u7684\u5185\u5bb9\u5199\u5165DWH\/RDBMS\u3002<\/h2>\n
\"undefined\"<\/div>\n
mysql_url<\/span> =<\/span> f<\/span>'jdbc:mysql:\/\/<\/span>{<\/span>hostname<\/span>}<\/span>:<\/span>{<\/span>port<\/span>}<\/span>\/<\/span>{<\/span>database<\/span>}<\/span>?user=<\/span>{<\/span>username<\/span>}<\/span>&password=<\/span>{<\/span>password<\/span>}<\/span>'<\/span>\r\n\r\ndf_gold<\/span> =<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_gold.delta'<\/span>)<\/span>\r\n\r\n(<\/span>\r\n  df_gold<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>\"jdbc\"<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>\"url\"<\/span>,<\/span> mysql_url<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>\"dbtable\"<\/span>,<\/span> \"test\"<\/span>)<\/span>\r\n  .<\/span>mode<\/span>(<\/span>\"overwrite\"<\/span>)<\/span>\r\n  .<\/span>save<\/span>()<\/span>\r\n)<\/span>\r\n<\/code><\/pre>\n
\u5c06Delta\u8868\u7684\u5185\u5bb9\u5199\u5165\u6587\u4ef6\u4e2d<\/h2>\n
\"undefined\"<\/div>\n
df_gold<\/span> =<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_gold.delta'<\/span>)<\/span>\r\n\r\ndf_gold<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'csv'<\/span>     ).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_gold.csv'<\/span>)<\/span>\r\ndf_gold<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'json'<\/span>    ).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_gold.json'<\/span>)<\/span>\r\ndf_gold<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'parquet'<\/span> ).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_gold.parquet'<\/span>)<\/span>\r\ndf_gold<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'avro'<\/span>    ).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stat_gold.avro'<\/span>)<\/span>\r\n<\/code><\/pre>\n
\u6574\u5408\uff08\u6c47\u603b\uff09\u4e4b\u524d\u7684ETL\u4ee3\u7801<\/h2>\n
\u4ee5\u4e0b\u4ee3\u7801\u53ef\u4ee5\u7b80\u5355\u5730\u8fdb\u884cETL\u5904\u7406\u3002<\/p>\n
\"undefined\"<\/div>\n
### 1. Raw\u30c6\u30fc\u30d6\u30eb (CSV => bornze)\r\n<\/span>df_raw<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'csv'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'Header'<\/span>,<\/span> True<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'inferSchema'<\/span>,<\/span> True<\/span>)<\/span>\r\n  .<\/span>load<\/span>(<\/span>'\/databricks-datasets\/lending-club-loan-stats\/LoanStats_2018Q2.csv'<\/span>)<\/span> \r\n)<\/span>\r\ndf_raw<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stats.delta'<\/span>)<\/span>\r\nsql<\/span>(<\/span>\"CREATE TABLE loan_stat_raw USING delta LOCATION '\/tmp\/daiwt2021\/loan_stat.delta';\"<\/span>)<\/span>\r\n\r\n\r\n### 2. \u30c7\u30fc\u30bf\u6574\u5f62 (bronze => silver)\r\n<\/span>from<\/span> pyspark.sql.functions<\/span> import<\/span> col<\/span>,<\/span> expr<\/span>\r\n\r\ndf_silver<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/loan_stats.delta'<\/span>)<\/span>\r\n  .<\/span>select<\/span>(<\/span>'loan_amnt'<\/span>,<\/span> 'term'<\/span>,<\/span> 'int_rate'<\/span>,<\/span> 'grade'<\/span>,<\/span> 'addr_state'<\/span>,<\/span> 'emp_title'<\/span>,<\/span> 'home_ownership'<\/span>,<\/span> 'annual_inc'<\/span>,<\/span> 'loan_status'<\/span>)<\/span>\r\n  .<\/span>withColumn<\/span>(<\/span>'int_rate'<\/span>,<\/span> expr<\/span>(<\/span>'cast(replace(int_rate,\"%\",\"\") as float)'<\/span>))<\/span>\r\n  .<\/span>withColumnRenamed<\/span>(<\/span>'addr_state'<\/span>,<\/span> 'state'<\/span>)<\/span>\r\n)<\/span>\r\ndf_silver<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span> '\/tmp\/daiwt2021\/loan_stat_silver.delta'<\/span>)<\/span>\r\nsql<\/span>(<\/span>\"CREATE TABLE loan_stat_silver USING delta LOCATION '\/tmp\/daiwt2021\/loan_stat_silver.delta';\"\")\r\n\r\n\r\n### 3. \u30b5\u30de\u30ea\u30c6\u30fc\u30d6\u30eb (silver => gold)\r\ndf_gold = spark.sql('''\r\n  SELECT state, loan_status, count(*) as counts \r\n  FROM loan_stat_silver\r\n  GROUP BY state, loan_status\r\n  ORDER BY counts DESC\r\n''')\r\n\r\ndf_gold.write.format('delta').mode('overwrite').save('\/tmp\/daiwt2021\/loan_stat_gold.delta')\r\nsql(\"<\/span>CREATE<\/span> TABLE<\/span> IF<\/span> NOT<\/span> EXISTS<\/span> loan_stat_gold<\/span> USING<\/span> delta<\/span> LOCATION<\/span> '\/tmp\/daiwt2021\/loan_stat_gold.delta'\")\r\n<\/span><\/code><\/pre>\n
\u52a0\u8f7dJSON<\/h2>\n
JSON\u6587\u4ef6 => \/mnt\/training\/ecommerce\/events\/events-500k.json<\/p>\n
\"undefined\"<\/div>\n
df_json<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'json'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'inferSchema'<\/span>,<\/span> True<\/span>)<\/span>\r\n  .<\/span>load<\/span>(<\/span>'\/mnt\/training\/ecommerce\/events\/events-500k.json'<\/span>)<\/span>\r\n)<\/span>\r\n\r\n# delta\u3078\u306e\u66f8\u304d\u8fbc\u307f\r\n<\/span>df_json<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/events.delta'<\/span>)<\/span>\r\n\r\n# \u78ba\u8a8d\r\n<\/span>display<\/span>(<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/events.delta'<\/span>)<\/span> )<\/span>\r\n<\/code><\/pre>\n
\u8bfb\u53d6LOG\u6570\u636e<\/h2>\n
\u53ef\u4ee5\u9002\u5e94\u4efb\u4f55\u6587\u672c\u683c\u5f0f\u3002<\/p>\n
\u8bbf\u95ee\u65e5\u5fd7\uff08Web\u670d\u52a1\u5668\u8bbf\u95ee\u65e5\u5fd7\uff09=> s3:\/\/databricks-ktmr-s3\/var\/log\/access.log.*.gz<\/p>\n
13.66.139.0 - - [19\/Dec\/2020:13:57:26 +0100] \"GET \/index.php?option=com_phocagallery&view=category&id=1:almhuette-raith&Itemid=53 HTTP\/1.1\" 200 32653 \"-\" \"Mozilla\/5.0 (compatible; bingbot\/2.0; +http:\/\/www.bing.com\/bingbot.htm)\" \"-\"\r\n157.48.153.185 - - [19\/Dec\/2020:14:08:06 +0100] \"GET \/apache-log\/access.log HTTP\/1.1\" 200 233 \"-\" \"Mozilla\/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/87.0.4280.88 Safari\/537.36\" \"-\"\r\n157.48.153.185 - - [19\/Dec\/2020:14:08:08 +0100] \"GET \/favicon.ico HTTP\/1.1\" 404 217 \"http:\/\/www.almhuette-raith.at\/apache-log\/access.log\" \"Mozilla\/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/87.0.4280.88 Safari\/537.36\" \"-\"\r\n...\r\n<\/code><\/pre>\n
\"undefined\"<\/div>\n
from<\/span> pyspark.sql.functions<\/span> import<\/span> split<\/span>,<\/span> regexp_extract<\/span>,<\/span> col<\/span>,<\/span> to_timestamp<\/span>\r\n\r\nraw_df<\/span> =<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>text<\/span>(<\/span>'s3:\/\/databricks-ktmr-s3\/var\/log\/access.log.*.gz'<\/span>)<\/span>\r\n\r\n# Regex\u3067\u30ed\u30b0\u30c7\u30fc\u30bf\u3092\u30d1\u30fc\u30b9\r\n<\/span>split_df<\/span> =<\/span> (<\/span>\r\n    raw_df<\/span>.<\/span>select<\/span>(<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})'<\/span>,<\/span> 1<\/span>).<\/span>alias<\/span>(<\/span>'src_ip'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'\\[(.+?)\\]'<\/span>,<\/span> 1<\/span>).<\/span>alias<\/span>(<\/span>'time_string'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^\\S+ \\S+ \\S+ \\S+ \\S+ \"(.+) (.+) (HTTP.+?)\"'<\/span>,<\/span> 1<\/span>).<\/span>alias<\/span>(<\/span>'method'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^\\S+ \\S+ \\S+ \\S+ \\S+ \"(.+) (.+) (HTTP.+?)\"'<\/span>,<\/span> 2<\/span>).<\/span>alias<\/span>(<\/span>'path'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^\\S+ \\S+ \\S+ \\S+ \\S+ \"(.+) (.+) (HTTP.+?)\"'<\/span>,<\/span> 3<\/span>).<\/span>alias<\/span>(<\/span>'version'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^\\S+ \\S+ \\S+ \\S+ \\S+ \".+HTTP.+?\" (\\d+) (\\d+)'<\/span>,<\/span> 1<\/span>).<\/span>cast<\/span>(<\/span>'int'<\/span>).<\/span>alias<\/span>(<\/span>'status_code'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^\\S+ \\S+ \\S+ \\S+ \\S+ \".+HTTP.+?\" (\\d+) (\\d+)'<\/span>,<\/span> 2<\/span>).<\/span>cast<\/span>(<\/span>'int'<\/span>).<\/span>alias<\/span>(<\/span>'content_size'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^\\S+ \\S+ \\S+ \\S+ \\S+ \".+HTTP.+?\" (\\d+) (\\d+) \"(.+?)\" \"(.+?)\" \"(.+?)\"'<\/span>,<\/span> 3<\/span>).<\/span>alias<\/span>(<\/span>'host2'<\/span>),<\/span>\r\n      regexp_extract<\/span>(<\/span>'value'<\/span>,<\/span> r<\/span>'^\\S+ \\S+ \\S+ \\S+ \\S+ \".+HTTP.+?\" (\\d+) (\\d+) \"(.+?)\" \"(.+?)\" \"(.+?)\"'<\/span>,<\/span> 4<\/span>).<\/span>alias<\/span>(<\/span>'user_agent'<\/span>)<\/span>\r\n    )<\/span>\r\n  .<\/span>withColumn<\/span>(<\/span> 'timestamp'<\/span>,<\/span> to_timestamp<\/span>(<\/span>  col<\/span>(<\/span>'time_string'<\/span>),<\/span> 'dd\/MMM\/yyyy:HH:mm:ss Z'<\/span>)<\/span> )<\/span>\r\n  .<\/span>drop<\/span>(<\/span>'time_string'<\/span>)<\/span>\r\n  .<\/span>filter<\/span>(<\/span> col<\/span>(<\/span>'timestamp'<\/span>).<\/span>isNotNull<\/span>()<\/span> )<\/span> \r\n)<\/span>\r\n\r\n# Delta\u306b\u66f8\u304d\u51fa\u3059\r\n<\/span>split_df<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/access_log.delta'<\/span>)<\/span>\r\n\r\n# \u78ba\u8a8d\r\n<\/span>display<\/span>(<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/access_log.delta'<\/span>)<\/span> )<\/span>\r\n<\/code><\/pre>\n
\u8bfb\u53d6\u56fe\u50cf\u6587\u4ef6\uff08\u4e8c\u8fdb\u5236\u6587\u4ef6\uff09<\/h2>\n
\u56fe\u50cf\u6587\u4ef6 => \/databricks-datasets\/cctvVideos\/train_images\/label=0\/*.jpg<\/p>\n
\"undefined\"<\/div>\n
image_df<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'binaryFile'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'mimeType'<\/span>,<\/span> 'image\/*'<\/span>)<\/span>\r\n  .<\/span>load<\/span>(<\/span>'\/databricks-datasets\/cctvVideos\/train_images\/label=0\/*.jpg'<\/span>)<\/span>\r\n)<\/span>\r\n\r\nspark<\/span>.<\/span>conf<\/span>.<\/span>set<\/span>(<\/span>\"spark.sql.parquet.compression.codec\"<\/span>,<\/span> \"uncompressed\"<\/span>)<\/span>\r\nimage_df<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/train_image.delta'<\/span>)<\/span>\r\nspark<\/span>.<\/span>conf<\/span>.<\/span>set<\/span>(<\/span>\"spark.sql.parquet.compression.codec\"<\/span>,<\/span> \"snappy\"<\/span>)<\/span>\r\n\r\n# \u78ba\u8a8d\r\n<\/span>display<\/span>(<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/train_image.delta'<\/span>)<\/span> )<\/span> \r\n<\/code><\/pre>\n
\"undefined\"<\/div>\n
\u4ece\u5173\u7cfb\u6570\u636e\u5e93\u7ba1\u7406\u7cfb\u7edf\/\u6570\u636e\u4ed3\u5e93\u4e2d\u8bfb\u53d6<\/h2>\n
\"undefined\"<\/div>\n
jdbcHostname<\/span> =<\/span> \"example.databricks.training\"<\/span>\r\njdbcPort<\/span> =<\/span> 5432<\/span>\r\njdbcDatabase<\/span> =<\/span> \"training\"<\/span>\r\njdbcUrl<\/span> =<\/span> f<\/span>\"jdbc:postgresql:\/\/<\/span>{<\/span>jdbcHostname<\/span>}<\/span>:<\/span>{<\/span>jdbcPort<\/span>}<\/span>\/<\/span>{<\/span>jdbcDatabase<\/span>}<\/span>\"<\/span>\r\n\r\n\r\nquery<\/span> =<\/span> '''\r\n(\r\n  SELECT * FROM training.people_1m  \r\n  WHERE salary > 100000\r\n) emp_alias\r\n'''<\/span>\r\n\r\ndf<\/span> =<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>jdbc<\/span>(<\/span>url<\/span>=<\/span>jdbcUrl<\/span>,<\/span> table<\/span>=<\/span>query<\/span>,<\/span> properties<\/span>=<\/span>connectionProps<\/span>)<\/span>\r\ndf<\/span>.<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/salary.delta'<\/span>)<\/span>\r\n\r\n# \u78ba\u8a8d\r\n<\/span>display<\/span>(<\/span> spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/salary.delta'<\/span>)<\/span> )<\/span>\r\n<\/code><\/pre>\n
\u52a0\u8f7d\u6d41\u5a92\u4f53<\/h2>\n
\"undefined\"<\/div>\n
from<\/span> pyspark.sql.types<\/span> import<\/span> *<\/span>\r\nfrom<\/span> pyspark.sql.functions<\/span> import<\/span> from_json<\/span>,<\/span> col<\/span>\r\nimport<\/span> re<\/span>\r\n\r\n#\u3000JSON\u30c7\u30fc\u30bf\u306e\u30b9\u30ad\u30fc\u30de\u5b9a\u7fa9\r\n<\/span>schema<\/span> =<\/span> StructType<\/span>([<\/span>\r\n  StructField<\/span>(<\/span>\"channel\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"comment\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"delta\"<\/span>,<\/span> IntegerType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"flag\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"geocoding\"<\/span>,<\/span> StructType<\/span>([<\/span> StructField<\/span>(<\/span>\"city\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span> StructField<\/span>(<\/span>\"country\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span> StructField<\/span>(<\/span>\"countryCode2\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span> StructField<\/span>(<\/span>\"countryCode3\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span> StructField<\/span>(<\/span>\"stateProvince\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span> StructField<\/span>(<\/span>\"latitude\"<\/span>,<\/span> DoubleType<\/span>(),<\/span> True<\/span>),<\/span> StructField<\/span>(<\/span>\"longitude\"<\/span>,<\/span> DoubleType<\/span>(),<\/span> True<\/span>),<\/span> ]),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"isAnonymous\"<\/span>,<\/span> BooleanType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"isNewPage\"<\/span>,<\/span> BooleanType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"isRobot\"<\/span>,<\/span> BooleanType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"isUnpatrolled\"<\/span>,<\/span> BooleanType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"namespace\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>         \r\n  StructField<\/span>(<\/span>\"page\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>              \r\n  StructField<\/span>(<\/span>\"pageURL\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>           \r\n  StructField<\/span>(<\/span>\"timestamp\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>        \r\n  StructField<\/span>(<\/span>\"url\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"user\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>              \r\n  StructField<\/span>(<\/span>\"userURL\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"wikipediaURL\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>\r\n  StructField<\/span>(<\/span>\"wikipedia\"<\/span>,<\/span> StringType<\/span>(),<\/span> True<\/span>),<\/span>\r\n])<\/span>\r\n\r\n#\u3000\u8aad\u307f\u8fbc\u307f\r\n<\/span>stream_df<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>readStream<\/span>.<\/span>format<\/span>(<\/span>'kafka'<\/span>)<\/span> # Kafka\u3092\u30bd\u30fc\u30b9\u3068\u6307\u5b9a\r\n<\/span>  .<\/span>option<\/span>(<\/span>'kafka.bootstrap.servers'<\/span>,<\/span> 'server123.databricks.training:9092'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'subscribe'<\/span>,<\/span> 'en'<\/span>)<\/span>\r\n  .<\/span>load<\/span>()<\/span>\r\n)<\/span>\r\n\r\n# ELT\u3092\u3057\u3066\u3001Delta\u306b\u66f8\u304d\u8fbc\u3080\r\n<\/span>(<\/span>\r\n  stream_df<\/span>\r\n  .<\/span>withColumn<\/span>(<\/span>'json'<\/span>,<\/span> from_json<\/span>(<\/span>col<\/span>(<\/span>'value'<\/span>).<\/span>cast<\/span>(<\/span>'string'<\/span>),<\/span> schema<\/span>))<\/span>   # Kafka\u306e\u30d0\u30a4\u30ca\u30ea\u30c7\u30fc\u30bf\u3092\u6587\u5b57\u5217\u306b\u5909\u63db\u3057\u3001from_json()\u3067JSON\u3092\u30d1\u30fc\u30b9\r\n<\/span>  .<\/span>select<\/span>(<\/span>col<\/span>(<\/span>\"json.*\"<\/span>))<\/span>                    # JSON\u306e\u5b50\u8981\u7d20\u3060\u3051\u3092\u53d6\u308a\u51fa\u3059\r\n<\/span>  .<\/span>writeStream<\/span>                              # writeStream()\u3067\u30b9\u30c8\u30ea\u30fc\u30e0\u3092\u66f8\u304d\u51fa\u3059\r\n<\/span>  .<\/span>format<\/span>(<\/span>'delta'<\/span>)<\/span>                          # Delta\u3068\u3057\u3066\u4fdd\u5b58\r\n<\/span>  .<\/span>option<\/span>(<\/span>'checkpointLocation'<\/span>,<\/span> '\/tmp\/daiwt2021\/stream.checkpoint'<\/span>)<\/span> # \u30c1\u30a7\u30c3\u30af\u30dd\u30a4\u30f3\u30c8\u4fdd\u5b58\u5148\u3092\u6307\u5b9a\r\n<\/span>  .<\/span>outputMode<\/span>(<\/span>'append'<\/span>)<\/span>                     # \u30de\u30a4\u30af\u30ed\u30d0\u30c3\u30c1\u306e\u7d50\u679c\u3092Append\u3067\u8ffd\u52a0\r\n<\/span>  .<\/span>start<\/span>(<\/span>'\/tmp\/daiwt2021\/stream.delta'<\/span>)<\/span>   # start()\u3067\u30b9\u30c8\u30ea\u30fc\u30e0\u51e6\u7406\u3092\u958b\u59cb (\u30a2\u30af\u30b7\u30e7\u30f3)\r\n<\/span>)<\/span>\r\n\r\n# \u78ba\u8a8d\r\n<\/span>df<\/span> =<\/span> spark<\/span>.<\/span>readStream<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/stream.delta'<\/span>)<\/span>\r\ndisplay<\/span>(<\/span> df<\/span> )<\/span>\r\n<\/code><\/pre>\n
\u5c06\u968f\u65f6\u66f4\u65b0\u8ffd\u52a0\u7684\u6587\u4ef6\u4f5c\u4e3a\u6d41\u8fdb\u884c\u5904\u7406\u3002<\/h2>\n
\u30d5\u30a1\u30a4\u30eb\u306e\u30c7\u30a3\u30ec\u30af\u30c8\u30ea => s3a:\/\/databricks-ktmr-s3\/stocknet-dataset\/tweet\/raw\/AAPL\/*
\n=> s3a:\/\/databricks-ktmr-s3\/stocknet-dataset\/tweet\/raw\/AAPL\/\u7684\u6587\u4ef6\u76ee\u5f55<\/p>\n
    \n
  • \n
      \u30aa\u30d6\u30b8\u30a7\u30af\u30c8\u30b9\u30c8\u30ec\u30fc\u30b8\u4e0a\u306b\u30d5\u30a1\u30a4\u30eb\u304c\u968f\u6642\u30a2\u30c3\u30d7\u30ed\u30fc\u30c9\u3055\u308c\u308b\u30d1\u30bf\u30fc\u30f3\u3082\u3088\u304f\u3042\u308a\u307e\u3059\u3002<\/ul>\n<\/li>\n<\/ul>\n
       <\/p>\n
        \n
      • \n
          Databricks\u3067\u306f\u3001\u30b9\u30c8\u30ec\u30fc\u30b8\u306b\u65b0\u305f\u306a\u8ffd\u52a0\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb\u3092\u8a8d\u8b58\u3057\u3066\u3001\u305d\u306e\u30d5\u30a1\u30a4\u30eb\u306e\u307f\u8aad\u307f\u8fbc\u3080\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/ul>\n<\/li>\n<\/ul>\n
           <\/p>\n
            \u3053\u306e\u5834\u5408\u3001\u30b9\u30c8\u30ec\u30fc\u30b8\u3092\u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u306e\u30bd\u30fc\u30b9\u3068\u3057\u3066\u5229\u7528\u3059\u308b\u3053\u3068\u306b\u306a\u308a\u307e\u3059\u3002<\/ul>\n
            \"undefined\"<\/div>\n
            # (\u30b9\u30ad\u30fc\u30de\u3092\u697d\u3057\u3066\u53d6\u5f97\u3059\u308b)\r\n<\/span>df<\/span>=<\/span>spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'json'<\/span>).<\/span>load<\/span>(<\/span>'s3a:\/\/databricks-ktmr-s3\/stocknet-dataset\/tweet\/raw\/AAPL\/2014-01-01'<\/span>)<\/span>\r\ntweet_schema<\/span> =<\/span> df<\/span>.<\/span>schema<\/span>\r\n<\/code><\/pre>\n
            \u9009\u62e91\uff09\u6301\u7eed\u968f\u65f6\u52a0\u8f7d<\/h3>\n
            df_autoloader<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>readStream<\/span>.<\/span>format<\/span>(<\/span>'cloudFiles'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'cloudFiles.format'<\/span>,<\/span> 'json'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'cloudFiles.maxBytesPerTrigger'<\/span>,<\/span> '50KB'<\/span>)<\/span> # \u4e00\u5ea6\u306b\u8aad\u3080\u30b5\u30a4\u30ba\u4e0a\u9650\r\n<\/span>  .<\/span>schema<\/span>(<\/span>tweet_schema<\/span>)<\/span>\r\n  .<\/span>load<\/span>(<\/span>'s3a:\/\/databricks-ktmr-s3\/stocknet-dataset\/tweet\/raw\/AAPL\/*'<\/span>)<\/span>\r\n)<\/span>\r\n\r\n(<\/span>\r\n  df_autoloader<\/span>.<\/span>writeStream<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'checkpointLocation'<\/span>,<\/span> '\/tmp\/daiwt2021\/tweet.checkpoint'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'maxFilesPerTrigger'<\/span>,<\/span> 25<\/span>)<\/span> # \u4e00\u5ea6\u306b\u8aad\u3080\u30d5\u30a1\u30a4\u30eb\u6570\u4e0a\u9650\r\n<\/span>  .<\/span>outputMode<\/span>(<\/span>'append'<\/span>)<\/span>\r\n  .<\/span>trigger<\/span>(<\/span>processingTime<\/span>=<\/span>'2 seconds'<\/span>)<\/span> # 2\u79d2\u306b\u4e00\u5ea6\u51e6\u7406\r\n<\/span>  #.trigger(once=True) # \u4e00\u5ea6\u3060\u3051\u51e6\u7406\r\n<\/span>  .<\/span>start<\/span>(<\/span>'\/tmp\/daiwt2021\/tweet.delta'<\/span>)<\/span>\r\n  #.awaitTermination() # async => sync\r\n<\/span>)<\/span>\r\n<\/code><\/pre>\n
            \u53ea\u5728\u6267\u884c\u4e00\u6b21\u65f6\uff0c\u5982\u679c\u6709\u65b0\u6587\u4ef6\uff0c\u5219\u8fdb\u884c\u52a0\u8f7d\u3002<\/h3>\n
            df_autoloader<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>readStream<\/span>.<\/span>format<\/span>(<\/span>'cloudFiles'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'cloudFiles.format'<\/span>,<\/span> 'json'<\/span>)<\/span>\r\n  #.option('cloudFiles.maxBytesPerTrigger', '50KB') # \u4e00\u5ea6\u306b\u8aad\u3080\u30b5\u30a4\u30ba\u4e0a\u9650 <== Trigger Once\u306e\u6642\u306b\u306f\u7121\u8996\u3055\u308c\u308b!\r\n<\/span>  .<\/span>schema<\/span>(<\/span>tweet_schema<\/span>)<\/span>\r\n  .<\/span>load<\/span>(<\/span>'s3a:\/\/databricks-ktmr-s3\/stocknet-dataset\/tweet\/raw\/AAPL\/*'<\/span>)<\/span>\r\n)<\/span>\r\n\r\n(<\/span>\r\n  df_autoloader<\/span>.<\/span>writeStream<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'checkpointLocation'<\/span>,<\/span> '\/tmp\/daiwt2021\/tweet.checkpoint'<\/span>)<\/span>\r\n  #.option('maxFilesPerTrigger', 25) # \u4e00\u5ea6\u306b\u8aad\u3080\u30d5\u30a1\u30a4\u30eb\u6570\u4e0a\u9650  <== Trigger Once\u306e\u6642\u306b\u306f\u7121\u8996\u3055\u308c\u308b!\r\n<\/span>  .<\/span>outputMode<\/span>(<\/span>'append'<\/span>)<\/span>\r\n  .<\/span>trigger<\/span>(<\/span>once<\/span>=<\/span>True<\/span>)<\/span> # \u4e00\u5ea6\u3060\u3051\u51e6\u7406\r\n<\/span>  .<\/span>start<\/span>(<\/span>'\/tmp\/daiwt2021\/tweet.delta'<\/span>)<\/span>\r\n  .<\/span>awaitTermination<\/span>()<\/span> # async => sync  <== Trigger Once\u306e\u5b9f\u884c\u304c\u7d42\u308f\u308b\u307e\u3067\u30d6\u30ed\u30c3\u30af\u3055\u305b\u308b\r\n<\/span>)<\/span>\r\n<\/code><\/pre>\n
            # \u78ba\u8a8d\r\n<\/span>spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/tweet.delta'<\/span>).<\/span>count<\/span>()<\/span>\r\n<\/code><\/pre>\n
            \u5de5\u4f5c\u30fb\u81ea\u52d5\u5316\u30fb\u6d41\u6c34\u7ebf\u5316<\/h2>\n
            \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528Databricks\u6765\u7ba1\u7406\u548c\u5b9a\u671f\u6267\u884c\u6570\u636e\u6d41\u6c34\u7ebf\u7684\u529f\u80fd\uff08Jobs\uff09\u3002
            \n\u8ba9\u6211\u4eec\u5c06\u4ee5\u4e0b\u4ee3\u7801\u8f6c\u5316\u4e3a\u4e00\u4e2aJob\u3002<\/p>\n
            ### 1. \u30b9\u30c8\u30ec\u30fc\u30b8\u304b\u3089\u66f4\u65b0\u30d5\u30a1\u30a4\u30eb\u3060\u3051\u3092\u8a8d\u8b58\u3057\u3066\u3001Delta\u30c6\u30fc\u30d6\u30eb\u306b\u8ffd\u8a18\u3059\u308b\r\n<\/span>df_autoloader<\/span> =<\/span> (<\/span>\r\n  spark<\/span>.<\/span>readStream<\/span>.<\/span>format<\/span>(<\/span>'cloudFiles'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'cloudFiles.format'<\/span>,<\/span> 'json'<\/span>)<\/span>\r\n  .<\/span>schema<\/span>(<\/span>tweet_schema<\/span>)<\/span>\r\n  .<\/span>load<\/span>(<\/span>'s3a:\/\/databricks-ktmr-s3\/stocknet-dataset\/tweet\/raw\/AAPL\/*'<\/span>)<\/span>\r\n)<\/span>\r\n\r\n(<\/span>\r\n  df_autoloader<\/span>.<\/span>writeStream<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>)<\/span>\r\n  .<\/span>option<\/span>(<\/span>'checkpointLocation'<\/span>,<\/span> '\/tmp\/daiwt2021\/job\/tweet.checkpoint'<\/span>)<\/span>\r\n  .<\/span>outputMode<\/span>(<\/span>'append'<\/span>)<\/span>\r\n  .<\/span>trigger<\/span>(<\/span>once<\/span>=<\/span>True<\/span>)<\/span> # \u4e00\u5ea6\u3060\u3051\u51e6\u7406\r\n<\/span>  .<\/span>start<\/span>(<\/span>'\/tmp\/daiwt2021\/job\/tweet.delta'<\/span>)<\/span>\r\n  .<\/span>awaitTermination<\/span>()<\/span> # async => sync\r\n<\/span>)<\/span>\r\n<\/code><\/pre>\n
            ### 2. \u4e0a\u8a18\u306eDelta\u30c6\u30fc\u30d6\u30eb\u304b\u3089\u30b5\u30de\u30ea\u306eDelta\u30c6\u30fc\u30d6\u30eb\u3092\u4f5c\u308b\r\n<\/span>\r\ndf<\/span>=<\/span>spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/job\/tweet.delta'<\/span>)<\/span>\r\n\r\n(<\/span>\r\n  df<\/span>.<\/span>groupBy<\/span>(<\/span>'lang'<\/span>).<\/span>count<\/span>()<\/span>\r\n  .<\/span>write<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>mode<\/span>(<\/span>'overwrite'<\/span>).<\/span>save<\/span>(<\/span>'\/tmp\/daiwt2021\/job\/tweet_summary.delta'<\/span>)<\/span>\r\n)<\/span>\r\n\r\nsql<\/span>(<\/span>\"CREATE TABLE IF NOT EXISTS tweet_summary USING delta LOCATION '\/tmp\/daiwt2021\/job\/tweet_summary.delta'\"<\/span>)<\/span>\r\n\r\n# \u78ba\u8a8d\r\n<\/span>display<\/span>(<\/span>\r\n  spark<\/span>.<\/span>read<\/span>.<\/span>format<\/span>(<\/span>'delta'<\/span>).<\/span>load<\/span>(<\/span>'\/tmp\/daiwt2021\/job\/tweet_summary.delta'<\/span>)<\/span>\r\n)<\/span>\r\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
            \u8fd9\u7bc7\u6587\u7ae0\u7684\u5185\u5bb9\u57fa\u4e8e\u5728Data+AI World Tour 2021\u4e3e\u884c\u7684\u6280\u672f\u7814\u8ba8\u4f1a\u201c\u6253\u9020\u7b80\u5355\u6613\u61c2\u7684\u6570\u636e\u6d41\u6c34\u7ebf\u201d […]<\/p>\n","protected":false},"author":3,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-47156","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"\nDatabricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868 - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/databricks\uff08delta-lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868\" \/>\n<meta property=\"og:description\" content=\"\u8fd9\u7bc7\u6587\u7ae0\u7684\u5185\u5bb9\u57fa\u4e8e\u5728Data+AI World Tour 2021\u4e3e\u884c\u7684\u6280\u672f\u7814\u8ba8\u4f1a\u201c\u6253\u9020\u7b80\u5355\u6613\u61c2\u7684\u6570\u636e\u6d41\u6c34\u7ebf\u201d […]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/databricks\uff08delta-lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2022-11-20T09:02:33+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-04-28T15:06:36+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d7d1b913a08637a69ecb7\/3-0.png\" \/>\n<meta name=\"author\" content=\"\u97f5, \u79d1\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u97f5, \u79d1\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"8 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/\",\"name\":\"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868 - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2022-11-20T09:02:33+00:00\",\"dateModified\":\"2024-04-28T15:06:36+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/6530331a63adef3b3443a1fab53a0e6e\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/6530331a63adef3b3443a1fab53a0e6e\",\"name\":\"\u97f5, \u79d1\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/429ccb39b3fff5188bc17986222cfb0936cbadb8cc933cff04ab5ca01bd30a08?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/429ccb39b3fff5188bc17986222cfb0936cbadb8cc933cff04ab5ca01bd30a08?s=96&d=mm&r=g\",\"caption\":\"\u97f5, \u79d1\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/yunke\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868 - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/databricks\uff08delta-lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868\/","og_locale":"zh_CN","og_type":"article","og_title":"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868","og_description":"\u8fd9\u7bc7\u6587\u7ae0\u7684\u5185\u5bb9\u57fa\u4e8e\u5728Data+AI World Tour 2021\u4e3e\u884c\u7684\u6280\u672f\u7814\u8ba8\u4f1a\u201c\u6253\u9020\u7b80\u5355\u6613\u61c2\u7684\u6570\u636e\u6d41\u6c34\u7ebf\u201d […]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/databricks\uff08delta-lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2022-11-20T09:02:33+00:00","article_modified_time":"2024-04-28T15:06:36+00:00","og_image":[{"url":"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d7d1b913a08637a69ecb7\/3-0.png"}],"author":"\u97f5, \u79d1","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u97f5, \u79d1","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"8 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/","name":"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868 - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2022-11-20T09:02:33+00:00","dateModified":"2024-04-28T15:06:36+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/6530331a63adef3b3443a1fab53a0e6e"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"Databricks\uff08Delta lake\uff09\u7684\u6570\u636e\u8f93\u5165\u8f93\u51fa\u5b9e\u73b0\u6a21\u5f0f-\u901f\u67e5\u8868"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/6530331a63adef3b3443a1fab53a0e6e","name":"\u97f5, \u79d1","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/429ccb39b3fff5188bc17986222cfb0936cbadb8cc933cff04ab5ca01bd30a08?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/429ccb39b3fff5188bc17986222cfb0936cbadb8cc933cff04ab5ca01bd30a08?s=96&d=mm&r=g","caption":"\u97f5, \u79d1"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/yunke\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/databricks%ef%bc%88delta-lake%ef%bc%89%e7%9a%84%e6%95%b0%e6%8d%ae%e8%be%93%e5%85%a5%e8%be%93%e5%87%ba%e5%ae%9e%e7%8e%b0%e6%a8%a1%e5%bc%8f-%e9%80%9f%e6%9f%a5%e8%a1%a8\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/47156","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/3"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=47156"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/47156\/revisions"}],"predecessor-version":[{"id":63816,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/47156\/revisions\/63816"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=47156"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=47156"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=47156"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}