{"id":36397,"date":"2023-09-08T04:42:49","date_gmt":"2023-02-20T21:44:28","guid":{"rendered":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/"},"modified":"2024-04-30T21:54:58","modified_gmt":"2024-04-30T13:54:58","slug":"apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9","status":"publish","type":"post","link":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/","title":{"rendered":"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9"},"content":{"rendered":"<h1>\u6574\u4e2a\u6267\u884c\u73af\u5883\u7684\u60c5\u51b5<\/h1>\n<div><img decoding=\"async\" class=\"post-images\" title=\"\" src=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d290137434c4406c423ce\/1-0.png\" alt=\"image.png\" \/><\/div>\n<h2>\u706b\u82b1\u9a71\u52a8<\/h2>\n<p>Spark\u9a71\u52a8\u7a0b\u5e8f\u5177\u6709\u591a\u4e2a\u89d2\u8272\u3002<\/p>\n<ul class=\"post-ul\">\n<li style=\"list-style-type: none;\">\n<ul class=\"post-ul\">cluster Manager\u3078\u60c5\u5831\u306e\u4f1d\u9054\u3092\u3059\u308b<\/ul>\n<\/li>\n<\/ul>\n<p>&nbsp;<\/p>\n<ul class=\"post-ul\">\n<li style=\"list-style-type: none;\">\n<ul class=\"post-ul\">Spark\u3092\u5b9f\u884c\u3059\u308bJVM\u306e\u305f\u3081\u306ecluster Managr\u304b\u3089\u306e\u3001\u30e1\u30e2\u30ea\u306a\u3069\u306e\u30ea\u30bd\u30fc\u30b9\u306e\u8981\u6c42<\/ul>\n<\/li>\n<\/ul>\n<p>&nbsp;<\/p>\n<ul class=\"post-ul\">\n<li style=\"list-style-type: none;\">\n<ul class=\"post-ul\">DAG\u30b3\u30f3\u30d4\u30e5\u30fc\u30c6\u30a3\u30f3\u30b0\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3001\u305d\u308c\u3089\u3092\u30bf\u30b9\u30af\u3068\u3057\u3066\u914d\u5e03\u3059\u308b\u3053\u3068\u3092Saprk\u306e\u64cd\u4f5c\u3067\u884c\u3046<\/ul>\n<\/li>\n<\/ul>\n<p>&nbsp;<\/p>\n<ul class=\"post-ul\">\n<li style=\"list-style-type: none;\">\n<ul class=\"post-ul\">\u518d\u3073\u30ea\u30bd\u30fc\u30b9\u3092\u914d\u5e03<\/ul>\n<\/li>\n<\/ul>\n<p>&nbsp;<\/p>\n<ul class=\"post-ul\">\u305d\u308c\u3089\u3092Spark executer\u306b\u4f1d\u9054<\/ul>\n<h2>\u706b\u82b1\u4f1a\u8bae<\/h2>\n<p>Spark\u7684\u5165\u53e3\u70b9<\/p>\n<pre class=\"post-pre\"><code>\/\/\u30e9\u30a4\u30d6\u30e9\u30ea\u306e\u8aad\u307f\u8fbc\u307f\r\nimport org.apache.spark.sql.SparkSession\r\n\r\n\/\/SparkSession\u306e\u69cb\u7bc9\r\nval spark = SparkSession\r\n    .builder\r\n    .appName(\"appName\")\r\n    .config(\"spark.sql.shuffle.partition\", 6)\r\n    .getOrCreate()\r\n\r\n\/\/Json\u306e\u8aad\u307f\u8fbc\u307f\r\nval people = spark.read.json(\"fileName\")\r\n<\/code><\/pre>\n<h2>\u96c6\u7fa4\u7ba1\u7406\u5668<\/h2>\n<p>\u89d2\u8272<\/p>\n<ul class=\"post-ul\">Spark\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u5b9f\u884c\u306e\u30ea\u30bd\u30fc\u30b9\u306e\u7ba1\u7406\u3084\u5272\u308a\u5f53\u3066\u306e\u5f79\u5272<\/ul>\n<p>\u80fd\u591f\u7ba1\u7406\u4ee5\u4e0b\u96c6\u7fa4<\/p>\n<ul class=\"post-ul\">\n<li style=\"list-style-type: none;\">\n<ul class=\"post-ul\">Apache Hadoop YARN<\/ul>\n<\/li>\n<\/ul>\n<p>&nbsp;<\/p>\n<ul class=\"post-ul\">\n<li style=\"list-style-type: none;\">\n<ul class=\"post-ul\">Apache Mesos<\/ul>\n<\/li>\n<\/ul>\n<p>&nbsp;<\/p>\n<ul class=\"post-ul\">Kubernetes<\/ul>\n<h2>\u706b\u82b1\u6267\u884c\u5668<\/h2>\n<p>Spark\u6267\u884c\u5668\u8d1f\u8d23\u5c06\u4fe1\u606f\u4f20\u9012\u7ed9\u7a0b\u5e8f\u9a71\u52a8\u7a0b\u5e8f\uff0c\u5e76\u5728\u6267\u884c\u5668\u4efb\u52a1\u4e2d\u626e\u6f14\u91cd\u8981\u89d2\u8272\u3002<\/p>\n<h2>\u90e8\u7f72\u6a21\u5f0f<\/h2>\n<p>Spark\u6709\u5404\u79cd\u4e0d\u540c\u7684\u90e8\u7f72\u6a21\u5f0f\u3002<br \/>\nCluster Manager\u662fSpark\u7684\u6267\u884c\u8005\uff0c\u53ea\u8981\u6ee1\u8db3\u8d44\u6e90\u9700\u6c42\uff0c\u5b83\u53ef\u4ee5\u72ec\u7acb\u4e8e\u73af\u5883\uff0c\u53ef\u4ee5\u90e8\u7f72\u5728Apache Hadoop YARN\u6216Kubernetes\u7b49\u5e73\u53f0\u4e0a\u3002<\/p>\n<h2>\u6570\u636e\u548c\u5206\u533a h\u00e9<\/h2>\n<p>\u6570\u636e\u5b58\u50a8\u5728\u5404\u79cd\u4e0d\u540c\u7684\u4f4d\u7f6e\u4e0a\uff0c<br \/>\n\u4f46\u5728Spark\u4e2d\uff0c\u53ef\u4ee5\u903b\u8f91\u6027\u5730\u5904\u7406\u5206\u533a\u548c\u6570\u636e\u3002<\/p>\n<p>\u8bf7\u6ce8\u610f\uff0c\u8fd9\u4e9b\u53ea\u9002\u7528\u4e8e\u7f51\u7edc\u5173\u95ed\u4e14\u5df2\u52a0\u8f7d\u5206\u533a\u7684\u4efb\u52a1\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6574\u4e2a\u6267\u884c\u73af\u5883\u7684\u60c5\u51b5 \u706b\u82b1\u9a71\u52a8 Spark\u9a71\u52a8\u7a0b\u5e8f\u5177\u6709\u591a\u4e2a\u89d2\u8272\u3002 cluster Manager\u3078\u60c5\u5831\u306e\u4f1d\u9054\u3092\u3059 [&hellip;]<\/p>\n","protected":false},"author":11,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-36397","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"<!-- This site is optimized with the Yoast SEO Premium plugin v21.5 (Yoast SEO v21.5) - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9 - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9\" \/>\n<meta property=\"og:description\" content=\"\u6574\u4e2a\u6267\u884c\u73af\u5883\u7684\u60c5\u51b5 \u706b\u82b1\u9a71\u52a8 Spark\u9a71\u52a8\u7a0b\u5e8f\u5177\u6709\u591a\u4e2a\u89d2\u8272\u3002 cluster Manager\u3078\u60c5\u5831\u306e\u4f1d\u9054\u3092\u3059 [&hellip;]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-02-20T21:44:28+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-04-30T13:54:58+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d290137434c4406c423ce\/1-0.png\" \/>\n<meta name=\"author\" content=\"\u65b0, \u97f5\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u65b0, \u97f5\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/\",\"name\":\"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9 - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-02-20T21:44:28+00:00\",\"dateModified\":\"2024-04-30T13:54:58+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9\",\"name\":\"\u65b0, \u97f5\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g\",\"caption\":\"\u65b0, \u97f5\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/yunxin\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9 - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9\/","og_locale":"zh_CN","og_type":"article","og_title":"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9","og_description":"\u6574\u4e2a\u6267\u884c\u73af\u5883\u7684\u60c5\u51b5 \u706b\u82b1\u9a71\u52a8 Spark\u9a71\u52a8\u7a0b\u5e8f\u5177\u6709\u591a\u4e2a\u89d2\u8272\u3002 cluster Manager\u3078\u60c5\u5831\u306e\u4f1d\u9054\u3092\u3059 [&hellip;]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-02-20T21:44:28+00:00","article_modified_time":"2024-04-30T13:54:58+00:00","og_image":[{"url":"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d290137434c4406c423ce\/1-0.png"}],"author":"\u65b0, \u97f5","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u65b0, \u97f5"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/","name":"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9 - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-02-20T21:44:28+00:00","dateModified":"2024-04-30T13:54:58+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"Apache Spark\u7684\u6267\u884c\u73af\u5883\u7279\u70b9"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9","name":"\u65b0, \u97f5","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g","caption":"\u65b0, \u97f5"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/yunxin\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/apache-spark%e7%9a%84%e6%89%a7%e8%a1%8c%e7%8e%af%e5%a2%83%e7%89%b9%e7%82%b9\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36397","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=36397"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36397\/revisions"}],"predecessor-version":[{"id":94524,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36397\/revisions\/94524"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=36397"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=36397"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=36397"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}