\u7ec4\u4ef6<\/h2>\n![\"image.png\"](\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png\" "\\"\\"")
<\/div>\n\u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002<\/p>\n
\n\u7528\u8a9e\u610f\u5473ApplicationSpark\u4e0a\u306b\u69cb\u7bc9\u3055\u308c\u305f\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0
\nDriver Process\u3068Executor Process\u3067\u69cb\u6210\u3055\u308c\u308b\u3002Application jar\u30e6\u30fc\u30b6\u306eSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u542b\u3093\u3060jar\u30d5\u30a1\u30a4\u30ebDriver programmain()\u95a2\u6570\u3092\u5b9f\u884c\u3059\u308b\u3002\u4e0b\u8a18\u306e\u5f79\u5272\u3092\u6301\u3064\u3002
\n\u30fb SparkContext\u3092\u4f5c\u6210\u3059\u308b
\n\u30fbSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306b\u95a2\u3059\u308b\u60c5\u5831\u7ba1\u7406
\n\u30fb\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u3084\u5165\u529b\u306e\u5fdc\u7b54
\nExecutor\u5168\u4f53\u306e\u30bf\u30b9\u30af\u306e\u5206\u6790\u3001\u5206\u6563\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3092\u884c\u3046\u3002Cluster manager\u30ea\u30bd\u30fc\u30b9\u3092\u53d6\u5f97\u3059\u308b\u305f\u3081\u306e\u5916\u90e8\u30d7\u30ed\u30b0\u30e9\u30e0\uff08YARN\u3001Mesos\u3001k8s\u7b49\uff09
\n\u7269\u7406\u30de\u30b7\u30f3\u3092\u5236\u5fa1\u3057\u3001Spark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306b\u30ea\u30bd\u30fc\u30b9\u3092\u30a2\u30b5\u30a4\u30f3\u3059\u308b\u3002Deploy mode\u30af\u30e9\u30b9\u30bf\u30fc\u30e2\u30fc\u30c9\u3068\u3001\u30af\u30e9\u30a4\u30a2\u30f3\u30c8\u30e2\u30fc\u30c9\u306e2\u3064\u304c\u3042\u308b\u3002
\n\u30fb\u30af\u30e9\u30b9\u30bf\u30e2\u30fc\u30c9\u306e\u5834\u5408\u3001Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u30af\u30e9\u30b9\u30bf\u5185\u3067\u8d77\u52d5\u3055\u308c\u308b\u3002
\n\u30fb\u30af\u30e9\u30a4\u30a2\u30f3\u30c8\u30e2\u30fc\u30c9\u306e\u5834\u5408\u3001Driver\u306f\u30af\u30e9\u30b9\u30bf\u5916\u304b\u3089\u8d77\u52d5\u3055\u308c\u308b\u3002Worker node\u30af\u30e9\u30b9\u30bf\u5185\u3067\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u5b9f\u884c\u3067\u304d\u308b\u4efb\u610f\u306e\u30ce\u30fc\u30c9Executor\u30ef\u30fc\u30ab\u30fc\u30ce\u30fc\u30c9\u4e0a\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u7528\u306b\u8d77\u52d5\u3055\u308c\u308b\u30d7\u30ed\u30bb\u30b9\u3002\u4e0b\u8a18\u306e\u5f79\u5272\u304c\u3042\u308b\u3002
\n\u30fbDriver\u304b\u3089\u5272\u308a\u5f53\u3066\u305f\u30bf\u30b9\u30af\u3092\u5b9f\u884c\u3057\u3001\u30bf\u30b9\u30af\u5168\u4f53\u306e\u30e1\u30e2\u30ea\u307e\u305f\u306f\u30c7\u30a3\u30b9\u30af\u30b9\u30c8\u30ec\u30fc\u30b8\u306b\u30c7\u30fc\u30bf\u3092\u4fdd\u6301\u3059\u308b\u3002
\n\u30fbExecutor\u306e\u8a08\u7b97\u306e\u9032\u6357\u3092Driver\u306b\u30ec\u30dd\u30fc\u30c8\u3059\u308b\u3002Task1\u3064\u306e\u30a8\u30b0\u30bc\u30ad\u30e5\u30fc\u30bf\u306b\u9001\u4fe1\u3055\u308c\u308b\u4f5c\u696d\u306e\u5358\u4f4dJobSpark\u30a2\u30af\u30b7\u30e7\u30f3 (\u4fdd\u5b58\u3001\u53ce\u96c6\u306a\u3069) \u306b\u5fdc\u7b54\u3057\u3066\u751f\u6210\u3055\u308c\u308b\u8907\u6570\u306e\u30bf\u30b9\u30af\u3067\u69cb\u6210\u3055\u308c\u308b\u4e26\u5217\u8a08\u7b97\u3002Stage\u5404\u30b8\u30e7\u30d6\u306f\u3001\u4e92\u3044\u306b\u4f9d\u5b58\u3059\u308b\u30b9\u30c6\u30fc\u30b8\u3068\u547c\u3070\u308c\u308b\u5c0f\u3055\u306a\u30bf\u30b9\u30af\u306e\u30bb\u30c3\u30c8\u306b\u5206\u5272\u3055\u308c\u308b\u3002 (MapReduce \u306e map \u30b9\u30c6\u30fc\u30b8\u3068 reduce \u30b9\u30c6\u30fc\u30b8\u306b\u8fd1\u3044\u30a4\u30e1\u30fc\u30b8)\u3002<\/div>\n<\/div>\n\u7533\u8bf7\u3001\u5de5\u4f5c\u3001\u9636\u6bb5\u3001\u4efb\u52a1\u4e4b\u95f4\u7684\u5173\u7cfb<\/h3>\n![\"image.png\"](\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/8-0.png\" "\\"\\"")
<\/div>\n\u6bcf\u4e2a\u7ec4\u4ef6\u7684\u8bf4\u660e<\/h2>\n\u53f8\u673a<\/h3>\n\n- \n
Driver\u306f\u3001\u4f5c\u6210\u3059\u308b\u30d7\u30ed\u30b0\u30e9\u30e0\u306emain\uff08\uff09\u95a2\u6570\u3092\u5b9f\u884c\u3059\u308b\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u3053\u306e\u30d7\u30ed\u30bb\u30b9\u304c\u5b9f\u884c\u3059\u308b\u30e6\u30fc\u30b6\u30b3\u30fc\u30c9\u304c\u3001SparkContext\u306e\u751f\u6210\u3001RDD\u306e\u751f\u6210\u3092\u884c\u3044\u3001\u5909\u63db\u3084\u30a2\u30af\u30b7\u30e7\u30f3\u3092\u5b9f\u884c\u3059\u308b<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306e\u30bf\u30b9\u30af\u3078\u306e\u5909\u63db\u3092\u884c\u3046<\/ul>\n<\/li>\n<\/ul>\nSpark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u30e6\u30fc\u30b6\u306e\u30d7\u30ed\u30b0\u30e9\u30e0\u3092\u5b9f\u884c\u306e\u5358\u4f4d\uff08\u30bf\u30b9\u30af\uff09\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001\u8ad6\u7406\u7684\u306aDAG\u3092\u751f\u6210\u3057\u3001\u30c9\u30e9\u30a4\u30d0\u304c\u5b9f\u884c\u3055\u308c\u308b\u3068\u3001\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u306f\u3001map\u306e\u5909\u63db\u3092\u7e8f\u3081\u3066\u30de\u30fc\u30b8\u3059\u308b\u305f\u3081\u306e\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u5316\u306e\u3088\u3046\u306a\u6700\u9069\u5316\u3092\u884c\u3044\u3001\u5b9f\u884c\u30b0\u30e9\u30d5\u3092\u4e00\u9023\u306e\u30b9\u30c6\u30fc\u30b8\u7fa4\u306b\u5909\u63db\u3059\u308b\u3002\u305d\u308c\u305e\u308c\u306e\u30b9\u30c6\u30fc\u30b8\u306b\u306f\u8907\u6570\u306e\u30bf\u30b9\u30af\u304c\u542b\u307e\u308c\u308b\u3002
\n\u30bf\u30b9\u30af\u306fSpark\u3067\u6700\u3082\u5c0f\u3055\u306a\u5358\u4f4d<\/p>\n
\u30a8\u30b0\u30bc\u30ad\u30e5\u30fc\u30bf\u4e0a\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3092\u884c\u3046<\/p>\n
\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u304c\u6c7a\u307e\u3063\u305f\u5f8c\u3001Spark\u30c9\u30e9\u30a4\u30d0\u306f\u3001Executor\u7fa4\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3092\u8abf\u6574\u3059\u308b\u3002
\nExecutor\u7fa4\u306f\u3001\u8d77\u52d5\u6642\u306b\u81ea\u5206\u81ea\u8eab\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u5bfe\u3057\u3066\u767b\u9332\u3059\u308b\u3053\u3068\u3067\u3001\u30c9\u30e9\u30a4\u30d0\u306f\u81ea\u5206\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306eExecutor\u306e\u72b6\u6cc1\u3092\u5e38\u306b\u628a\u63e1\u3067\u304d\u308b\u3002
\n\u5404Executor\u306f\u3001\u30bf\u30b9\u30af\u3092\u5b9f\u884c\u3057\u3001RDD\u306e\u30c7\u30fc\u30bf\u3092\u4fdd\u6301\u3059\u308b\u4e8b\u304c\u3067\u304d\u308b\u30d7\u30ed\u30bb\u30b9\u3002
\nSpark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u305d\u306e\u6642\u70b9\u3067\u306eExecutor\u7fa4\u3092\u898b\u3066\u3001\u30c7\u30fc\u30bf\u306e\u914d\u7f6e\u306b\u57fa\u3065\u3044\u3066\u5404\u30bf\u30b9\u30af\u3092\u9069\u5207\u306a\u5834\u6240\u306b\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3057\u3088\u3046\u3068\u3059\u308b\u3002<\/p>\n
\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u3066\u3044\u305f\u5834\u5408\u3082\u3001\u305d\u306e\u30c7\u30fc\u30bf\u306b\u30a2\u30af\u30bb\u30b9\u3059\u308b\u5c06\u6765\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u306b\u3001\u3053\u306e\u60c5\u5831\u3092\u6d3b\u7528\u3059\u308b\u3002<\/p>\n
\u30c9\u30e9\u30a4\u30d0\u306f\u3001Web\u30a4\u30f3\u30bf\u30d5\u30a7\u30a4\u30b9\u3092\u901a\u3058\u3066\u5b9f\u884c\u4e2d\u306eSpark\u30a2\u30d7\u30ea\u306b\u95a2\u3059\u308b\u60c5\u5831\u3092\u516c\u958b\u3059\u308b<\/p>\n
\u6267\u884c\u8005\u3002<\/h3>\n\n- \n
Spark\u30b8\u30e7\u30d6\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u5b9f\u884c\u3092\u53d7\u3051\u6301\u3064\u30ef\u30fc\u30ab\u30fc\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
Executor\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u8d77\u52d5\u6642\u306b1\u5ea6\u8d77\u52d5\u3055\u308c\u3066\u3001\u901a\u5e38\u306f\u305d\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u304c\u52d5\u4f5c\u3057\u3066\u3044\u308b\u9593\u3001\u52d5\u4f5c\u3057\u7d9a\u3051\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u69cb\u6210\u3059\u308b\u30bf\u30b9\u30af\u7fa4\u3092\u5b9f\u884c\u3057\u3001\u7d50\u679c\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u8fd4\u3059\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306b\u3088\u3063\u3066\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308bRDD\u3092\u306e\u305f\u3081\u306e\u30a4\u30f3\u30e1\u30e2\u30ea\u30b9\u30c8\u30ec\u30fc\u30b8\u3092\u5404Executor\u5185\u3067\u52d5\u4f5c\u3059\u308b\u30d6\u30ed\u30c3\u30af\u30de\u30cd\u30fc\u30b8\u30e3\u30fc\u3068\u547c\u3070\u308c\u308b\u30b5\u30fc\u30d3\u30b9\u3092\u901a\u3058\u3066\u63d0\u4f9b\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\nRDD\u306fExecutor\u5185\u306b\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308b\u306e\u3067\u3001\u30bf\u30b9\u30af\u306f\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u305f\u30c7\u30fc\u30bf\u3068\u540c\u5c45\u3057\u3066\u52d5\u4f5c\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n
\u96c6\u7fa4\u7ecf\u7406<\/h3>\n\n- \n
Executor\u3084Driver\u306e\u8d77\u52d5\u3092Cluster Manager\u304c\u62c5\u3046\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Spark\u306eCluster Manager\u306f\u30d7\u30e9\u30ac\u30d6\u30eb\u306a\u69cb\u6210\u3068\u306a\u3063\u3066\u304a\u308a\u3001YARN\u3084Mesos\u306a\u3069\u304c\u5229\u7528\u3067\u304d\u308b\u3002<\/ul>\nHadoop YARN \u5206\u5e03\u5f0f\u7cfb\u7edf<\/h4>\n\n- \n
\u5171\u6709\u30ea\u30bd\u30fc\u30b9\u30d7\u30fc\u30eb\u4e0a\u3067\u69d8\u3005\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u30d5\u30ec\u30fc\u30e0\u30ef\u30fc\u30af\u3092\u52d5\u4f5c\u3055\u305b\u308b\u3053\u3068\u304c\u3067\u304d\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
YARN\u3067\u52d5\u4f5c\u3055\u305b\u308b\u5834\u5408\u306b\u3001Spark\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306f\u6c7a\u3081\u3089\u308c\u305f\u6570\u306eExecutor\u3092\u4f7f\u3046\u3002\u5404\u7a2e\u30d1\u30e9\u30e1\u30fc\u30bf\u306f\u4e0b\u8a18\u306e\u901a\u308a<\/ul>\n\n\u8a2d\u5b9a\u9805\u76ee\u8aac\u660e–num-executorsExecutor\u306e\u6570–executor-memoryExecutor\u304c\u4f7f\u7528\u3059\u308b\u30e1\u30e2\u30ea–executor-corsYARN\u306b\u8981\u6c42\u3059\u308b\u30b3\u30a2\u6570<\/div>\n<\/div>\nSpark\u5e94\u7528\u7a0b\u5e8f<\/h3>\n\n- \n
Spark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u4e2d\u6838\u3068\u306a\u308b\u306e\u306f\u3001Spark Driver\u3068\u306a\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Spark Driver\u3067\u306fSparkSession\u3092\u4f5c\u6210\u3059\u308b<\/ul>\nSpark \u5de5\u4f5c<\/h3>\n\n- \n
Driver\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u30921\u3064\u4ee5\u4e0a\u306eSpark\u30b8\u30e7\u30d6\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\u6b21\u306b\u5404Spark \u30b8\u30e7\u30d6\u3092DAG\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n\u706b\u82b1\u9636\u6bb5<\/h3>\n\n- \n
DAG\u30ce\u30fc\u30c9\u306e\u4e00\u90e8\u3068\u3057\u3066\u3001\u76f4\u5217\u307e\u305f\u306f\u4e26\u5217\u306b\u5b9f\u884c\u3067\u304d\u308b\u64cd\u4f5c\u306b\u57fa\u3065\u3044\u3066\u30b9\u30c6\u30fc\u30b8\u304c\u4f5c\u6210\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\u3059\u3079\u3066\u306e\u64cd\u4f5c\u304c1\u3064\u306eStage\u306b\u306a\u308b\u308f\u3051\u3067\u306f\u306a\u304f\u3001\u8907\u6570\u306eStage\u3067\u5206\u5272\u3067\u304d\u308b\u3002<\/ul>\n\u706b\u82b1\u4efb\u52a1<\/h3>\n\n- \n
\u5404\u30b9\u30c6\u30fc\u30b8\u306f\u3001Spark\u30bf\u30b9\u30af\u3067\u69cb\u6210\u3055\u308c\u3001\u5404Spark Executor\u9593\u3067\u30d5\u30a7\u30c7\u30ec\u30fc\u30b7\u30e7\u30f3\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u5404\u30bf\u30b9\u30af\u306f1\u3064\u306e\u30b3\u30a2\u306b\u30de\u30c3\u30d7\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
16\u30b3\u30a2\u306eExecutor\u3067\u306f\u300116\u500b\u4ee5\u4e0a\u306e\u30bf\u30b9\u30af\u309216\u500b\u4ee5\u4e0a\u306e\u30d1\u30fc\u30c6\u30a3\u30b7\u30e7\u30f3\u3067\u4e26\u5217\u3067\u51e6\u7406\u3059\u308b\u3002<\/ul>\n\u6267\u884c\u65f6\u5e8f<\/h2>\n
\u5728\u9a71\u52a8\u7a0b\u5e8f\u4e2d\uff0cSparkContext\u7a0b\u5e8f\u8fdb\u884c\u8c03\u6574\u548c\u63a7\u5236\u3002\u5927\u81f4\u6d41\u7a0b\u5982\u4e0b\u3002<\/p>\n
\n- \n
SparkContext\u901a\u8fc7\u8bbf\u95eeCluster Manager\u6765\u5206\u914d\u8d44\u6e90\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
\n- \n
\u8fde\u63a5\u540e\uff0c\u5728\u96c6\u7fa4\u7684\u8282\u70b9\u4e0a\u83b7\u53d6Executor\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
\n- \n
SparkContext\u5c06\u5e94\u7528\u7a0b\u5e8f\uff08Jar\u6587\u4ef6\u6216Python\u6587\u4ef6\uff09\u5206\u53d1\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
SparkContext\u5c06\u4efb\u52a1\u53d1\u9001\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n\u8bf7\u6ce8\u610f\u4e8b\u9879\u3002<\/h2>\n\n- \n
Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Executor\u304b\u3089\u306e\u901a\u4fe1\u3092Listen\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u3001\u30ef\u30fc\u30ab\u30fc\u30ce\u30fc\u30c9\u304b\u3089NW\u3067\u30a2\u30c9\u30ec\u30b9\u3092\u6307\u5b9a\u3067\u304d\u308b\u5fc5\u8981\u304c\u3042\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Worker\u30ce\u30fc\u30c9\u3092\u5236\u5fa1\u3059\u308b\u305f\u3081\u53ef\u80fd\u306a\u9650\u308a\u8fd1\u304f\u3067\u5b9f\u884c\u3059\u308b\u3053\u3068\u304c\u63a8\u5968\u3055\u308c\u308b\u3002<\/ul>\n\u8bf7\u8003\u5bdf<\/h1>\n
\u7531\u4e8e\u57fa\u672c\u6982\u5ff5\u5df2\u7ecf\u6f84\u6e05\uff0c\u6211\u8ba1\u5212\u5c06\u91cd\u70b9\u653e\u5728\u6574\u7406Spark\u7279\u8272\u4e2d\u7684\u5ef6\u8fdf\u8bc4\u4f30\u6267\u884c\u7b49\u7279\u6027\u548c\u53c2\u6570\u4e0a\u3002<\/p>\n
\u8bf7\u53c2\u8003<\/h1>\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]<\/p>\n","protected":false},"author":5,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-36589","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"\n
\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\" \/>\n<meta property=\"og:description\" content=\"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-12-16T03:55:01+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-04-29T17:03:25+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png\" \/>\n<meta name=\"author\" content=\"\u6e05, \u5b87\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u6e05, \u5b87\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"1 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-12-16T03:55:01+00:00\",\"dateModified\":\"2024-04-29T17:03:25+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\",\"name\":\"\u6e05, \u5b87\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"caption\":\"\u6e05, \u5b87\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_locale":"zh_CN","og_type":"article","og_title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406","og_description":"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-12-16T03:55:01+00:00","article_modified_time":"2024-04-29T17:03:25+00:00","og_image":[{"url":"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png"}],"author":"\u6e05, \u5b87","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u6e05, \u5b87","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"1 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-12-16T03:55:01+00:00","dateModified":"2024-04-29T17:03:25+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e","name":"\u6e05, \u5b87","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","caption":"\u6e05, \u5b87"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/5"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=36589"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions"}],"predecessor-version":[{"id":88735,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions\/88735"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=36589"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=36589"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=36589"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
<\/div>\n\u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002<\/p>\n
\n
\u7528\u8a9e\u610f\u5473ApplicationSpark\u4e0a\u306b\u69cb\u7bc9\u3055\u308c\u305f\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0
\nDriver Process\u3068Executor Process\u3067\u69cb\u6210\u3055\u308c\u308b\u3002Application jar\u30e6\u30fc\u30b6\u306eSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u542b\u3093\u3060jar\u30d5\u30a1\u30a4\u30ebDriver programmain()\u95a2\u6570\u3092\u5b9f\u884c\u3059\u308b\u3002\u4e0b\u8a18\u306e\u5f79\u5272\u3092\u6301\u3064\u3002
\n\u30fb SparkContext\u3092\u4f5c\u6210\u3059\u308b
\n\u30fbSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306b\u95a2\u3059\u308b\u60c5\u5831\u7ba1\u7406
\n\u30fb\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u3084\u5165\u529b\u306e\u5fdc\u7b54
\nExecutor\u5168\u4f53\u306e\u30bf\u30b9\u30af\u306e\u5206\u6790\u3001\u5206\u6563\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3092\u884c\u3046\u3002Cluster manager\u30ea\u30bd\u30fc\u30b9\u3092\u53d6\u5f97\u3059\u308b\u305f\u3081\u306e\u5916\u90e8\u30d7\u30ed\u30b0\u30e9\u30e0\uff08YARN\u3001Mesos\u3001k8s\u7b49\uff09
\n\u7269\u7406\u30de\u30b7\u30f3\u3092\u5236\u5fa1\u3057\u3001Spark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306b\u30ea\u30bd\u30fc\u30b9\u3092\u30a2\u30b5\u30a4\u30f3\u3059\u308b\u3002Deploy mode\u30af\u30e9\u30b9\u30bf\u30fc\u30e2\u30fc\u30c9\u3068\u3001\u30af\u30e9\u30a4\u30a2\u30f3\u30c8\u30e2\u30fc\u30c9\u306e2\u3064\u304c\u3042\u308b\u3002
\n\u30fb\u30af\u30e9\u30b9\u30bf\u30e2\u30fc\u30c9\u306e\u5834\u5408\u3001Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u30af\u30e9\u30b9\u30bf\u5185\u3067\u8d77\u52d5\u3055\u308c\u308b\u3002
\n\u30fb\u30af\u30e9\u30a4\u30a2\u30f3\u30c8\u30e2\u30fc\u30c9\u306e\u5834\u5408\u3001Driver\u306f\u30af\u30e9\u30b9\u30bf\u5916\u304b\u3089\u8d77\u52d5\u3055\u308c\u308b\u3002Worker node\u30af\u30e9\u30b9\u30bf\u5185\u3067\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u5b9f\u884c\u3067\u304d\u308b\u4efb\u610f\u306e\u30ce\u30fc\u30c9Executor\u30ef\u30fc\u30ab\u30fc\u30ce\u30fc\u30c9\u4e0a\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u7528\u306b\u8d77\u52d5\u3055\u308c\u308b\u30d7\u30ed\u30bb\u30b9\u3002\u4e0b\u8a18\u306e\u5f79\u5272\u304c\u3042\u308b\u3002
\n\u30fbDriver\u304b\u3089\u5272\u308a\u5f53\u3066\u305f\u30bf\u30b9\u30af\u3092\u5b9f\u884c\u3057\u3001\u30bf\u30b9\u30af\u5168\u4f53\u306e\u30e1\u30e2\u30ea\u307e\u305f\u306f\u30c7\u30a3\u30b9\u30af\u30b9\u30c8\u30ec\u30fc\u30b8\u306b\u30c7\u30fc\u30bf\u3092\u4fdd\u6301\u3059\u308b\u3002
\n\u30fbExecutor\u306e\u8a08\u7b97\u306e\u9032\u6357\u3092Driver\u306b\u30ec\u30dd\u30fc\u30c8\u3059\u308b\u3002Task1\u3064\u306e\u30a8\u30b0\u30bc\u30ad\u30e5\u30fc\u30bf\u306b\u9001\u4fe1\u3055\u308c\u308b\u4f5c\u696d\u306e\u5358\u4f4dJobSpark\u30a2\u30af\u30b7\u30e7\u30f3 (\u4fdd\u5b58\u3001\u53ce\u96c6\u306a\u3069) \u306b\u5fdc\u7b54\u3057\u3066\u751f\u6210\u3055\u308c\u308b\u8907\u6570\u306e\u30bf\u30b9\u30af\u3067\u69cb\u6210\u3055\u308c\u308b\u4e26\u5217\u8a08\u7b97\u3002Stage\u5404\u30b8\u30e7\u30d6\u306f\u3001\u4e92\u3044\u306b\u4f9d\u5b58\u3059\u308b\u30b9\u30c6\u30fc\u30b8\u3068\u547c\u3070\u308c\u308b\u5c0f\u3055\u306a\u30bf\u30b9\u30af\u306e\u30bb\u30c3\u30c8\u306b\u5206\u5272\u3055\u308c\u308b\u3002 (MapReduce \u306e map \u30b9\u30c6\u30fc\u30b8\u3068 reduce \u30b9\u30c6\u30fc\u30b8\u306b\u8fd1\u3044\u30a4\u30e1\u30fc\u30b8)\u3002<\/div>\n<\/div>\n\u7533\u8bf7\u3001\u5de5\u4f5c\u3001\u9636\u6bb5\u3001\u4efb\u52a1\u4e4b\u95f4\u7684\u5173\u7cfb<\/h3>\n
\nDriver Process\u3068Executor Process\u3067\u69cb\u6210\u3055\u308c\u308b\u3002Application jar\u30e6\u30fc\u30b6\u306eSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u542b\u3093\u3060jar\u30d5\u30a1\u30a4\u30ebDriver programmain()\u95a2\u6570\u3092\u5b9f\u884c\u3059\u308b\u3002\u4e0b\u8a18\u306e\u5f79\u5272\u3092\u6301\u3064\u3002
\n\u30fb SparkContext\u3092\u4f5c\u6210\u3059\u308b
\n\u30fbSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306b\u95a2\u3059\u308b\u60c5\u5831\u7ba1\u7406
\n\u30fb\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u3084\u5165\u529b\u306e\u5fdc\u7b54
\nExecutor\u5168\u4f53\u306e\u30bf\u30b9\u30af\u306e\u5206\u6790\u3001\u5206\u6563\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3092\u884c\u3046\u3002Cluster manager\u30ea\u30bd\u30fc\u30b9\u3092\u53d6\u5f97\u3059\u308b\u305f\u3081\u306e\u5916\u90e8\u30d7\u30ed\u30b0\u30e9\u30e0\uff08YARN\u3001Mesos\u3001k8s\u7b49\uff09
\n\u7269\u7406\u30de\u30b7\u30f3\u3092\u5236\u5fa1\u3057\u3001Spark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306b\u30ea\u30bd\u30fc\u30b9\u3092\u30a2\u30b5\u30a4\u30f3\u3059\u308b\u3002Deploy mode\u30af\u30e9\u30b9\u30bf\u30fc\u30e2\u30fc\u30c9\u3068\u3001\u30af\u30e9\u30a4\u30a2\u30f3\u30c8\u30e2\u30fc\u30c9\u306e2\u3064\u304c\u3042\u308b\u3002
\n\u30fb\u30af\u30e9\u30b9\u30bf\u30e2\u30fc\u30c9\u306e\u5834\u5408\u3001Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u30af\u30e9\u30b9\u30bf\u5185\u3067\u8d77\u52d5\u3055\u308c\u308b\u3002
\n\u30fb\u30af\u30e9\u30a4\u30a2\u30f3\u30c8\u30e2\u30fc\u30c9\u306e\u5834\u5408\u3001Driver\u306f\u30af\u30e9\u30b9\u30bf\u5916\u304b\u3089\u8d77\u52d5\u3055\u308c\u308b\u3002Worker node\u30af\u30e9\u30b9\u30bf\u5185\u3067\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u5b9f\u884c\u3067\u304d\u308b\u4efb\u610f\u306e\u30ce\u30fc\u30c9Executor\u30ef\u30fc\u30ab\u30fc\u30ce\u30fc\u30c9\u4e0a\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u7528\u306b\u8d77\u52d5\u3055\u308c\u308b\u30d7\u30ed\u30bb\u30b9\u3002\u4e0b\u8a18\u306e\u5f79\u5272\u304c\u3042\u308b\u3002
\n\u30fbDriver\u304b\u3089\u5272\u308a\u5f53\u3066\u305f\u30bf\u30b9\u30af\u3092\u5b9f\u884c\u3057\u3001\u30bf\u30b9\u30af\u5168\u4f53\u306e\u30e1\u30e2\u30ea\u307e\u305f\u306f\u30c7\u30a3\u30b9\u30af\u30b9\u30c8\u30ec\u30fc\u30b8\u306b\u30c7\u30fc\u30bf\u3092\u4fdd\u6301\u3059\u308b\u3002
\n\u30fbExecutor\u306e\u8a08\u7b97\u306e\u9032\u6357\u3092Driver\u306b\u30ec\u30dd\u30fc\u30c8\u3059\u308b\u3002Task1\u3064\u306e\u30a8\u30b0\u30bc\u30ad\u30e5\u30fc\u30bf\u306b\u9001\u4fe1\u3055\u308c\u308b\u4f5c\u696d\u306e\u5358\u4f4dJobSpark\u30a2\u30af\u30b7\u30e7\u30f3 (\u4fdd\u5b58\u3001\u53ce\u96c6\u306a\u3069) \u306b\u5fdc\u7b54\u3057\u3066\u751f\u6210\u3055\u308c\u308b\u8907\u6570\u306e\u30bf\u30b9\u30af\u3067\u69cb\u6210\u3055\u308c\u308b\u4e26\u5217\u8a08\u7b97\u3002Stage\u5404\u30b8\u30e7\u30d6\u306f\u3001\u4e92\u3044\u306b\u4f9d\u5b58\u3059\u308b\u30b9\u30c6\u30fc\u30b8\u3068\u547c\u3070\u308c\u308b\u5c0f\u3055\u306a\u30bf\u30b9\u30af\u306e\u30bb\u30c3\u30c8\u306b\u5206\u5272\u3055\u308c\u308b\u3002 (MapReduce \u306e map \u30b9\u30c6\u30fc\u30b8\u3068 reduce \u30b9\u30c6\u30fc\u30b8\u306b\u8fd1\u3044\u30a4\u30e1\u30fc\u30b8)\u3002<\/div>\n<\/div>\n
\u7533\u8bf7\u3001\u5de5\u4f5c\u3001\u9636\u6bb5\u3001\u4efb\u52a1\u4e4b\u95f4\u7684\u5173\u7cfb<\/h3>\n<\/div>\n\u6bcf\u4e2a\u7ec4\u4ef6\u7684\u8bf4\u660e<\/h2>\n\u53f8\u673a<\/h3>\n\n- \n
Driver\u306f\u3001\u4f5c\u6210\u3059\u308b\u30d7\u30ed\u30b0\u30e9\u30e0\u306emain\uff08\uff09\u95a2\u6570\u3092\u5b9f\u884c\u3059\u308b\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u3053\u306e\u30d7\u30ed\u30bb\u30b9\u304c\u5b9f\u884c\u3059\u308b\u30e6\u30fc\u30b6\u30b3\u30fc\u30c9\u304c\u3001SparkContext\u306e\u751f\u6210\u3001RDD\u306e\u751f\u6210\u3092\u884c\u3044\u3001\u5909\u63db\u3084\u30a2\u30af\u30b7\u30e7\u30f3\u3092\u5b9f\u884c\u3059\u308b<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306e\u30bf\u30b9\u30af\u3078\u306e\u5909\u63db\u3092\u884c\u3046<\/ul>\n<\/li>\n<\/ul>\nSpark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u30e6\u30fc\u30b6\u306e\u30d7\u30ed\u30b0\u30e9\u30e0\u3092\u5b9f\u884c\u306e\u5358\u4f4d\uff08\u30bf\u30b9\u30af\uff09\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001\u8ad6\u7406\u7684\u306aDAG\u3092\u751f\u6210\u3057\u3001\u30c9\u30e9\u30a4\u30d0\u304c\u5b9f\u884c\u3055\u308c\u308b\u3068\u3001\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u306f\u3001map\u306e\u5909\u63db\u3092\u7e8f\u3081\u3066\u30de\u30fc\u30b8\u3059\u308b\u305f\u3081\u306e\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u5316\u306e\u3088\u3046\u306a\u6700\u9069\u5316\u3092\u884c\u3044\u3001\u5b9f\u884c\u30b0\u30e9\u30d5\u3092\u4e00\u9023\u306e\u30b9\u30c6\u30fc\u30b8\u7fa4\u306b\u5909\u63db\u3059\u308b\u3002\u305d\u308c\u305e\u308c\u306e\u30b9\u30c6\u30fc\u30b8\u306b\u306f\u8907\u6570\u306e\u30bf\u30b9\u30af\u304c\u542b\u307e\u308c\u308b\u3002
\n\u30bf\u30b9\u30af\u306fSpark\u3067\u6700\u3082\u5c0f\u3055\u306a\u5358\u4f4d<\/p>\n
\u30a8\u30b0\u30bc\u30ad\u30e5\u30fc\u30bf\u4e0a\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3092\u884c\u3046<\/p>\n
\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u304c\u6c7a\u307e\u3063\u305f\u5f8c\u3001Spark\u30c9\u30e9\u30a4\u30d0\u306f\u3001Executor\u7fa4\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3092\u8abf\u6574\u3059\u308b\u3002
\nExecutor\u7fa4\u306f\u3001\u8d77\u52d5\u6642\u306b\u81ea\u5206\u81ea\u8eab\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u5bfe\u3057\u3066\u767b\u9332\u3059\u308b\u3053\u3068\u3067\u3001\u30c9\u30e9\u30a4\u30d0\u306f\u81ea\u5206\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306eExecutor\u306e\u72b6\u6cc1\u3092\u5e38\u306b\u628a\u63e1\u3067\u304d\u308b\u3002
\n\u5404Executor\u306f\u3001\u30bf\u30b9\u30af\u3092\u5b9f\u884c\u3057\u3001RDD\u306e\u30c7\u30fc\u30bf\u3092\u4fdd\u6301\u3059\u308b\u4e8b\u304c\u3067\u304d\u308b\u30d7\u30ed\u30bb\u30b9\u3002
\nSpark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u305d\u306e\u6642\u70b9\u3067\u306eExecutor\u7fa4\u3092\u898b\u3066\u3001\u30c7\u30fc\u30bf\u306e\u914d\u7f6e\u306b\u57fa\u3065\u3044\u3066\u5404\u30bf\u30b9\u30af\u3092\u9069\u5207\u306a\u5834\u6240\u306b\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3057\u3088\u3046\u3068\u3059\u308b\u3002<\/p>\n
\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u3066\u3044\u305f\u5834\u5408\u3082\u3001\u305d\u306e\u30c7\u30fc\u30bf\u306b\u30a2\u30af\u30bb\u30b9\u3059\u308b\u5c06\u6765\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u306b\u3001\u3053\u306e\u60c5\u5831\u3092\u6d3b\u7528\u3059\u308b\u3002<\/p>\n
\u30c9\u30e9\u30a4\u30d0\u306f\u3001Web\u30a4\u30f3\u30bf\u30d5\u30a7\u30a4\u30b9\u3092\u901a\u3058\u3066\u5b9f\u884c\u4e2d\u306eSpark\u30a2\u30d7\u30ea\u306b\u95a2\u3059\u308b\u60c5\u5831\u3092\u516c\u958b\u3059\u308b<\/p>\n
\u6267\u884c\u8005\u3002<\/h3>\n\n- \n
Spark\u30b8\u30e7\u30d6\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u5b9f\u884c\u3092\u53d7\u3051\u6301\u3064\u30ef\u30fc\u30ab\u30fc\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
Executor\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u8d77\u52d5\u6642\u306b1\u5ea6\u8d77\u52d5\u3055\u308c\u3066\u3001\u901a\u5e38\u306f\u305d\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u304c\u52d5\u4f5c\u3057\u3066\u3044\u308b\u9593\u3001\u52d5\u4f5c\u3057\u7d9a\u3051\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u69cb\u6210\u3059\u308b\u30bf\u30b9\u30af\u7fa4\u3092\u5b9f\u884c\u3057\u3001\u7d50\u679c\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u8fd4\u3059\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306b\u3088\u3063\u3066\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308bRDD\u3092\u306e\u305f\u3081\u306e\u30a4\u30f3\u30e1\u30e2\u30ea\u30b9\u30c8\u30ec\u30fc\u30b8\u3092\u5404Executor\u5185\u3067\u52d5\u4f5c\u3059\u308b\u30d6\u30ed\u30c3\u30af\u30de\u30cd\u30fc\u30b8\u30e3\u30fc\u3068\u547c\u3070\u308c\u308b\u30b5\u30fc\u30d3\u30b9\u3092\u901a\u3058\u3066\u63d0\u4f9b\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\nRDD\u306fExecutor\u5185\u306b\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308b\u306e\u3067\u3001\u30bf\u30b9\u30af\u306f\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u305f\u30c7\u30fc\u30bf\u3068\u540c\u5c45\u3057\u3066\u52d5\u4f5c\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n
\u96c6\u7fa4\u7ecf\u7406<\/h3>\n\n- \n
Executor\u3084Driver\u306e\u8d77\u52d5\u3092Cluster Manager\u304c\u62c5\u3046\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Spark\u306eCluster Manager\u306f\u30d7\u30e9\u30ac\u30d6\u30eb\u306a\u69cb\u6210\u3068\u306a\u3063\u3066\u304a\u308a\u3001YARN\u3084Mesos\u306a\u3069\u304c\u5229\u7528\u3067\u304d\u308b\u3002<\/ul>\nHadoop YARN \u5206\u5e03\u5f0f\u7cfb\u7edf<\/h4>\n\n- \n
\u5171\u6709\u30ea\u30bd\u30fc\u30b9\u30d7\u30fc\u30eb\u4e0a\u3067\u69d8\u3005\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u30d5\u30ec\u30fc\u30e0\u30ef\u30fc\u30af\u3092\u52d5\u4f5c\u3055\u305b\u308b\u3053\u3068\u304c\u3067\u304d\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
YARN\u3067\u52d5\u4f5c\u3055\u305b\u308b\u5834\u5408\u306b\u3001Spark\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306f\u6c7a\u3081\u3089\u308c\u305f\u6570\u306eExecutor\u3092\u4f7f\u3046\u3002\u5404\u7a2e\u30d1\u30e9\u30e1\u30fc\u30bf\u306f\u4e0b\u8a18\u306e\u901a\u308a<\/ul>\n\n\u8a2d\u5b9a\u9805\u76ee\u8aac\u660e–num-executorsExecutor\u306e\u6570–executor-memoryExecutor\u304c\u4f7f\u7528\u3059\u308b\u30e1\u30e2\u30ea–executor-corsYARN\u306b\u8981\u6c42\u3059\u308b\u30b3\u30a2\u6570<\/div>\n<\/div>\nSpark\u5e94\u7528\u7a0b\u5e8f<\/h3>\n\n- \n
Spark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u4e2d\u6838\u3068\u306a\u308b\u306e\u306f\u3001Spark Driver\u3068\u306a\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Spark Driver\u3067\u306fSparkSession\u3092\u4f5c\u6210\u3059\u308b<\/ul>\nSpark \u5de5\u4f5c<\/h3>\n\n- \n
Driver\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u30921\u3064\u4ee5\u4e0a\u306eSpark\u30b8\u30e7\u30d6\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\u6b21\u306b\u5404Spark \u30b8\u30e7\u30d6\u3092DAG\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n\u706b\u82b1\u9636\u6bb5<\/h3>\n\n- \n
DAG\u30ce\u30fc\u30c9\u306e\u4e00\u90e8\u3068\u3057\u3066\u3001\u76f4\u5217\u307e\u305f\u306f\u4e26\u5217\u306b\u5b9f\u884c\u3067\u304d\u308b\u64cd\u4f5c\u306b\u57fa\u3065\u3044\u3066\u30b9\u30c6\u30fc\u30b8\u304c\u4f5c\u6210\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\u3059\u3079\u3066\u306e\u64cd\u4f5c\u304c1\u3064\u306eStage\u306b\u306a\u308b\u308f\u3051\u3067\u306f\u306a\u304f\u3001\u8907\u6570\u306eStage\u3067\u5206\u5272\u3067\u304d\u308b\u3002<\/ul>\n\u706b\u82b1\u4efb\u52a1<\/h3>\n\n- \n
\u5404\u30b9\u30c6\u30fc\u30b8\u306f\u3001Spark\u30bf\u30b9\u30af\u3067\u69cb\u6210\u3055\u308c\u3001\u5404Spark Executor\u9593\u3067\u30d5\u30a7\u30c7\u30ec\u30fc\u30b7\u30e7\u30f3\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u5404\u30bf\u30b9\u30af\u306f1\u3064\u306e\u30b3\u30a2\u306b\u30de\u30c3\u30d7\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
16\u30b3\u30a2\u306eExecutor\u3067\u306f\u300116\u500b\u4ee5\u4e0a\u306e\u30bf\u30b9\u30af\u309216\u500b\u4ee5\u4e0a\u306e\u30d1\u30fc\u30c6\u30a3\u30b7\u30e7\u30f3\u3067\u4e26\u5217\u3067\u51e6\u7406\u3059\u308b\u3002<\/ul>\n\u6267\u884c\u65f6\u5e8f<\/h2>\n
\u5728\u9a71\u52a8\u7a0b\u5e8f\u4e2d\uff0cSparkContext\u7a0b\u5e8f\u8fdb\u884c\u8c03\u6574\u548c\u63a7\u5236\u3002\u5927\u81f4\u6d41\u7a0b\u5982\u4e0b\u3002<\/p>\n
\n- \n
SparkContext\u901a\u8fc7\u8bbf\u95eeCluster Manager\u6765\u5206\u914d\u8d44\u6e90\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
\n- \n
\u8fde\u63a5\u540e\uff0c\u5728\u96c6\u7fa4\u7684\u8282\u70b9\u4e0a\u83b7\u53d6Executor\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
\n- \n
SparkContext\u5c06\u5e94\u7528\u7a0b\u5e8f\uff08Jar\u6587\u4ef6\u6216Python\u6587\u4ef6\uff09\u5206\u53d1\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
SparkContext\u5c06\u4efb\u52a1\u53d1\u9001\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n\u8bf7\u6ce8\u610f\u4e8b\u9879\u3002<\/h2>\n\n- \n
Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Executor\u304b\u3089\u306e\u901a\u4fe1\u3092Listen\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u3001\u30ef\u30fc\u30ab\u30fc\u30ce\u30fc\u30c9\u304b\u3089NW\u3067\u30a2\u30c9\u30ec\u30b9\u3092\u6307\u5b9a\u3067\u304d\u308b\u5fc5\u8981\u304c\u3042\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Worker\u30ce\u30fc\u30c9\u3092\u5236\u5fa1\u3059\u308b\u305f\u3081\u53ef\u80fd\u306a\u9650\u308a\u8fd1\u304f\u3067\u5b9f\u884c\u3059\u308b\u3053\u3068\u304c\u63a8\u5968\u3055\u308c\u308b\u3002<\/ul>\n\u8bf7\u8003\u5bdf<\/h1>\n
\u7531\u4e8e\u57fa\u672c\u6982\u5ff5\u5df2\u7ecf\u6f84\u6e05\uff0c\u6211\u8ba1\u5212\u5c06\u91cd\u70b9\u653e\u5728\u6574\u7406Spark\u7279\u8272\u4e2d\u7684\u5ef6\u8fdf\u8bc4\u4f30\u6267\u884c\u7b49\u7279\u6027\u548c\u53c2\u6570\u4e0a\u3002<\/p>\n
\u8bf7\u53c2\u8003<\/h1>\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]<\/p>\n","protected":false},"author":5,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-36589","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"\n
\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\" \/>\n<meta property=\"og:description\" content=\"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-12-16T03:55:01+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-04-29T17:03:25+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png\" \/>\n<meta name=\"author\" content=\"\u6e05, \u5b87\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u6e05, \u5b87\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"1 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-12-16T03:55:01+00:00\",\"dateModified\":\"2024-04-29T17:03:25+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\",\"name\":\"\u6e05, \u5b87\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"caption\":\"\u6e05, \u5b87\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_locale":"zh_CN","og_type":"article","og_title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406","og_description":"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-12-16T03:55:01+00:00","article_modified_time":"2024-04-29T17:03:25+00:00","og_image":[{"url":"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png"}],"author":"\u6e05, \u5b87","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u6e05, \u5b87","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"1 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-12-16T03:55:01+00:00","dateModified":"2024-04-29T17:03:25+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e","name":"\u6e05, \u5b87","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","caption":"\u6e05, \u5b87"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/5"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=36589"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions"}],"predecessor-version":[{"id":88735,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions\/88735"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=36589"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=36589"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=36589"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
<\/div>\n\u6bcf\u4e2a\u7ec4\u4ef6\u7684\u8bf4\u660e<\/h2>\n\u53f8\u673a<\/h3>\n\n- \n
Driver\u306f\u3001\u4f5c\u6210\u3059\u308b\u30d7\u30ed\u30b0\u30e9\u30e0\u306emain\uff08\uff09\u95a2\u6570\u3092\u5b9f\u884c\u3059\u308b\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u3053\u306e\u30d7\u30ed\u30bb\u30b9\u304c\u5b9f\u884c\u3059\u308b\u30e6\u30fc\u30b6\u30b3\u30fc\u30c9\u304c\u3001SparkContext\u306e\u751f\u6210\u3001RDD\u306e\u751f\u6210\u3092\u884c\u3044\u3001\u5909\u63db\u3084\u30a2\u30af\u30b7\u30e7\u30f3\u3092\u5b9f\u884c\u3059\u308b<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306e\u30bf\u30b9\u30af\u3078\u306e\u5909\u63db\u3092\u884c\u3046<\/ul>\n<\/li>\n<\/ul>\nSpark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u30e6\u30fc\u30b6\u306e\u30d7\u30ed\u30b0\u30e9\u30e0\u3092\u5b9f\u884c\u306e\u5358\u4f4d\uff08\u30bf\u30b9\u30af\uff09\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001\u8ad6\u7406\u7684\u306aDAG\u3092\u751f\u6210\u3057\u3001\u30c9\u30e9\u30a4\u30d0\u304c\u5b9f\u884c\u3055\u308c\u308b\u3068\u3001\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u306f\u3001map\u306e\u5909\u63db\u3092\u7e8f\u3081\u3066\u30de\u30fc\u30b8\u3059\u308b\u305f\u3081\u306e\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u5316\u306e\u3088\u3046\u306a\u6700\u9069\u5316\u3092\u884c\u3044\u3001\u5b9f\u884c\u30b0\u30e9\u30d5\u3092\u4e00\u9023\u306e\u30b9\u30c6\u30fc\u30b8\u7fa4\u306b\u5909\u63db\u3059\u308b\u3002\u305d\u308c\u305e\u308c\u306e\u30b9\u30c6\u30fc\u30b8\u306b\u306f\u8907\u6570\u306e\u30bf\u30b9\u30af\u304c\u542b\u307e\u308c\u308b\u3002
\n\u30bf\u30b9\u30af\u306fSpark\u3067\u6700\u3082\u5c0f\u3055\u306a\u5358\u4f4d<\/p>\n
\u30a8\u30b0\u30bc\u30ad\u30e5\u30fc\u30bf\u4e0a\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3092\u884c\u3046<\/p>\n
\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u304c\u6c7a\u307e\u3063\u305f\u5f8c\u3001Spark\u30c9\u30e9\u30a4\u30d0\u306f\u3001Executor\u7fa4\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3092\u8abf\u6574\u3059\u308b\u3002
\nExecutor\u7fa4\u306f\u3001\u8d77\u52d5\u6642\u306b\u81ea\u5206\u81ea\u8eab\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u5bfe\u3057\u3066\u767b\u9332\u3059\u308b\u3053\u3068\u3067\u3001\u30c9\u30e9\u30a4\u30d0\u306f\u81ea\u5206\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306eExecutor\u306e\u72b6\u6cc1\u3092\u5e38\u306b\u628a\u63e1\u3067\u304d\u308b\u3002
\n\u5404Executor\u306f\u3001\u30bf\u30b9\u30af\u3092\u5b9f\u884c\u3057\u3001RDD\u306e\u30c7\u30fc\u30bf\u3092\u4fdd\u6301\u3059\u308b\u4e8b\u304c\u3067\u304d\u308b\u30d7\u30ed\u30bb\u30b9\u3002
\nSpark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u305d\u306e\u6642\u70b9\u3067\u306eExecutor\u7fa4\u3092\u898b\u3066\u3001\u30c7\u30fc\u30bf\u306e\u914d\u7f6e\u306b\u57fa\u3065\u3044\u3066\u5404\u30bf\u30b9\u30af\u3092\u9069\u5207\u306a\u5834\u6240\u306b\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3057\u3088\u3046\u3068\u3059\u308b\u3002<\/p>\n
\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u3066\u3044\u305f\u5834\u5408\u3082\u3001\u305d\u306e\u30c7\u30fc\u30bf\u306b\u30a2\u30af\u30bb\u30b9\u3059\u308b\u5c06\u6765\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u306b\u3001\u3053\u306e\u60c5\u5831\u3092\u6d3b\u7528\u3059\u308b\u3002<\/p>\n
\u30c9\u30e9\u30a4\u30d0\u306f\u3001Web\u30a4\u30f3\u30bf\u30d5\u30a7\u30a4\u30b9\u3092\u901a\u3058\u3066\u5b9f\u884c\u4e2d\u306eSpark\u30a2\u30d7\u30ea\u306b\u95a2\u3059\u308b\u60c5\u5831\u3092\u516c\u958b\u3059\u308b<\/p>\n
\u6267\u884c\u8005\u3002<\/h3>\n\n- \n
Spark\u30b8\u30e7\u30d6\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u5b9f\u884c\u3092\u53d7\u3051\u6301\u3064\u30ef\u30fc\u30ab\u30fc\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
Executor\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u8d77\u52d5\u6642\u306b1\u5ea6\u8d77\u52d5\u3055\u308c\u3066\u3001\u901a\u5e38\u306f\u305d\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u304c\u52d5\u4f5c\u3057\u3066\u3044\u308b\u9593\u3001\u52d5\u4f5c\u3057\u7d9a\u3051\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u69cb\u6210\u3059\u308b\u30bf\u30b9\u30af\u7fa4\u3092\u5b9f\u884c\u3057\u3001\u7d50\u679c\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u8fd4\u3059\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306b\u3088\u3063\u3066\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308bRDD\u3092\u306e\u305f\u3081\u306e\u30a4\u30f3\u30e1\u30e2\u30ea\u30b9\u30c8\u30ec\u30fc\u30b8\u3092\u5404Executor\u5185\u3067\u52d5\u4f5c\u3059\u308b\u30d6\u30ed\u30c3\u30af\u30de\u30cd\u30fc\u30b8\u30e3\u30fc\u3068\u547c\u3070\u308c\u308b\u30b5\u30fc\u30d3\u30b9\u3092\u901a\u3058\u3066\u63d0\u4f9b\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\nRDD\u306fExecutor\u5185\u306b\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308b\u306e\u3067\u3001\u30bf\u30b9\u30af\u306f\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u305f\u30c7\u30fc\u30bf\u3068\u540c\u5c45\u3057\u3066\u52d5\u4f5c\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n
\u96c6\u7fa4\u7ecf\u7406<\/h3>\n\n- \n
Executor\u3084Driver\u306e\u8d77\u52d5\u3092Cluster Manager\u304c\u62c5\u3046\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Spark\u306eCluster Manager\u306f\u30d7\u30e9\u30ac\u30d6\u30eb\u306a\u69cb\u6210\u3068\u306a\u3063\u3066\u304a\u308a\u3001YARN\u3084Mesos\u306a\u3069\u304c\u5229\u7528\u3067\u304d\u308b\u3002<\/ul>\nHadoop YARN \u5206\u5e03\u5f0f\u7cfb\u7edf<\/h4>\n\n- \n
\u5171\u6709\u30ea\u30bd\u30fc\u30b9\u30d7\u30fc\u30eb\u4e0a\u3067\u69d8\u3005\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u30d5\u30ec\u30fc\u30e0\u30ef\u30fc\u30af\u3092\u52d5\u4f5c\u3055\u305b\u308b\u3053\u3068\u304c\u3067\u304d\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
YARN\u3067\u52d5\u4f5c\u3055\u305b\u308b\u5834\u5408\u306b\u3001Spark\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306f\u6c7a\u3081\u3089\u308c\u305f\u6570\u306eExecutor\u3092\u4f7f\u3046\u3002\u5404\u7a2e\u30d1\u30e9\u30e1\u30fc\u30bf\u306f\u4e0b\u8a18\u306e\u901a\u308a<\/ul>\n\n\u8a2d\u5b9a\u9805\u76ee\u8aac\u660e–num-executorsExecutor\u306e\u6570–executor-memoryExecutor\u304c\u4f7f\u7528\u3059\u308b\u30e1\u30e2\u30ea–executor-corsYARN\u306b\u8981\u6c42\u3059\u308b\u30b3\u30a2\u6570<\/div>\n<\/div>\nSpark\u5e94\u7528\u7a0b\u5e8f<\/h3>\n\n- \n
Spark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u4e2d\u6838\u3068\u306a\u308b\u306e\u306f\u3001Spark Driver\u3068\u306a\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Spark Driver\u3067\u306fSparkSession\u3092\u4f5c\u6210\u3059\u308b<\/ul>\nSpark \u5de5\u4f5c<\/h3>\n\n- \n
Driver\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u30921\u3064\u4ee5\u4e0a\u306eSpark\u30b8\u30e7\u30d6\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\u6b21\u306b\u5404Spark \u30b8\u30e7\u30d6\u3092DAG\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n\u706b\u82b1\u9636\u6bb5<\/h3>\n\n- \n
DAG\u30ce\u30fc\u30c9\u306e\u4e00\u90e8\u3068\u3057\u3066\u3001\u76f4\u5217\u307e\u305f\u306f\u4e26\u5217\u306b\u5b9f\u884c\u3067\u304d\u308b\u64cd\u4f5c\u306b\u57fa\u3065\u3044\u3066\u30b9\u30c6\u30fc\u30b8\u304c\u4f5c\u6210\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\u3059\u3079\u3066\u306e\u64cd\u4f5c\u304c1\u3064\u306eStage\u306b\u306a\u308b\u308f\u3051\u3067\u306f\u306a\u304f\u3001\u8907\u6570\u306eStage\u3067\u5206\u5272\u3067\u304d\u308b\u3002<\/ul>\n\u706b\u82b1\u4efb\u52a1<\/h3>\n\n- \n
\u5404\u30b9\u30c6\u30fc\u30b8\u306f\u3001Spark\u30bf\u30b9\u30af\u3067\u69cb\u6210\u3055\u308c\u3001\u5404Spark Executor\u9593\u3067\u30d5\u30a7\u30c7\u30ec\u30fc\u30b7\u30e7\u30f3\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
\n- \n
\u5404\u30bf\u30b9\u30af\u306f1\u3064\u306e\u30b3\u30a2\u306b\u30de\u30c3\u30d7\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
16\u30b3\u30a2\u306eExecutor\u3067\u306f\u300116\u500b\u4ee5\u4e0a\u306e\u30bf\u30b9\u30af\u309216\u500b\u4ee5\u4e0a\u306e\u30d1\u30fc\u30c6\u30a3\u30b7\u30e7\u30f3\u3067\u4e26\u5217\u3067\u51e6\u7406\u3059\u308b\u3002<\/ul>\n\u6267\u884c\u65f6\u5e8f<\/h2>\n
\u5728\u9a71\u52a8\u7a0b\u5e8f\u4e2d\uff0cSparkContext\u7a0b\u5e8f\u8fdb\u884c\u8c03\u6574\u548c\u63a7\u5236\u3002\u5927\u81f4\u6d41\u7a0b\u5982\u4e0b\u3002<\/p>\n
\n- \n
SparkContext\u901a\u8fc7\u8bbf\u95eeCluster Manager\u6765\u5206\u914d\u8d44\u6e90\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
\n- \n
\u8fde\u63a5\u540e\uff0c\u5728\u96c6\u7fa4\u7684\u8282\u70b9\u4e0a\u83b7\u53d6Executor\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
\n- \n
SparkContext\u5c06\u5e94\u7528\u7a0b\u5e8f\uff08Jar\u6587\u4ef6\u6216Python\u6587\u4ef6\uff09\u5206\u53d1\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n<\/li>\n<\/ol>\n <\/p>\n
SparkContext\u5c06\u4efb\u52a1\u53d1\u9001\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n\u8bf7\u6ce8\u610f\u4e8b\u9879\u3002<\/h2>\n\n- \n
Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Executor\u304b\u3089\u306e\u901a\u4fe1\u3092Listen\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u3001\u30ef\u30fc\u30ab\u30fc\u30ce\u30fc\u30c9\u304b\u3089NW\u3067\u30a2\u30c9\u30ec\u30b9\u3092\u6307\u5b9a\u3067\u304d\u308b\u5fc5\u8981\u304c\u3042\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n <\/p>\n
Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Worker\u30ce\u30fc\u30c9\u3092\u5236\u5fa1\u3059\u308b\u305f\u3081\u53ef\u80fd\u306a\u9650\u308a\u8fd1\u304f\u3067\u5b9f\u884c\u3059\u308b\u3053\u3068\u304c\u63a8\u5968\u3055\u308c\u308b\u3002<\/ul>\n\u8bf7\u8003\u5bdf<\/h1>\n
\u7531\u4e8e\u57fa\u672c\u6982\u5ff5\u5df2\u7ecf\u6f84\u6e05\uff0c\u6211\u8ba1\u5212\u5c06\u91cd\u70b9\u653e\u5728\u6574\u7406Spark\u7279\u8272\u4e2d\u7684\u5ef6\u8fdf\u8bc4\u4f30\u6267\u884c\u7b49\u7279\u6027\u548c\u53c2\u6570\u4e0a\u3002<\/p>\n
\u8bf7\u53c2\u8003<\/h1>\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]<\/p>\n","protected":false},"author":5,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-36589","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"\n
\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\" \/>\n<meta property=\"og:description\" content=\"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-12-16T03:55:01+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-04-29T17:03:25+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png\" \/>\n<meta name=\"author\" content=\"\u6e05, \u5b87\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u6e05, \u5b87\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"1 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-12-16T03:55:01+00:00\",\"dateModified\":\"2024-04-29T17:03:25+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\",\"name\":\"\u6e05, \u5b87\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"caption\":\"\u6e05, \u5b87\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_locale":"zh_CN","og_type":"article","og_title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406","og_description":"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-12-16T03:55:01+00:00","article_modified_time":"2024-04-29T17:03:25+00:00","og_image":[{"url":"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png"}],"author":"\u6e05, \u5b87","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u6e05, \u5b87","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"1 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-12-16T03:55:01+00:00","dateModified":"2024-04-29T17:03:25+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e","name":"\u6e05, \u5b87","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","caption":"\u6e05, \u5b87"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/5"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=36589"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions"}],"predecessor-version":[{"id":88735,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions\/88735"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=36589"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=36589"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=36589"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
- \n
- \n
- Driver\u306f\u3001\u4f5c\u6210\u3059\u308b\u30d7\u30ed\u30b0\u30e9\u30e0\u306emain\uff08\uff09\u95a2\u6570\u3092\u5b9f\u884c\u3059\u308b\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n
- \n
- \u3053\u306e\u30d7\u30ed\u30bb\u30b9\u304c\u5b9f\u884c\u3059\u308b\u30e6\u30fc\u30b6\u30b3\u30fc\u30c9\u304c\u3001SparkContext\u306e\u751f\u6210\u3001RDD\u306e\u751f\u6210\u3092\u884c\u3044\u3001\u5909\u63db\u3084\u30a2\u30af\u30b7\u30e7\u30f3\u3092\u5b9f\u884c\u3059\u308b<\/ul>\n<\/li>\n<\/ul>\n
- \n
- \u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306e\u30bf\u30b9\u30af\u3078\u306e\u5909\u63db\u3092\u884c\u3046<\/ul>\n<\/li>\n<\/ul>\n
- \n
- Spark\u30b8\u30e7\u30d6\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u5b9f\u884c\u3092\u53d7\u3051\u6301\u3064\u30ef\u30fc\u30ab\u30fc\u30d7\u30ed\u30bb\u30b9<\/ul>\n<\/li>\n<\/ul>\n
- \n
- Executor\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u8d77\u52d5\u6642\u306b1\u5ea6\u8d77\u52d5\u3055\u308c\u3066\u3001\u901a\u5e38\u306f\u305d\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u304c\u52d5\u4f5c\u3057\u3066\u3044\u308b\u9593\u3001\u52d5\u4f5c\u3057\u7d9a\u3051\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- \u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u69cb\u6210\u3059\u308b\u30bf\u30b9\u30af\u7fa4\u3092\u5b9f\u884c\u3057\u3001\u7d50\u679c\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u8fd4\u3059\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- \u30e6\u30fc\u30b6\u30d7\u30ed\u30b0\u30e9\u30e0\u306b\u3088\u3063\u3066\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308bRDD\u3092\u306e\u305f\u3081\u306e\u30a4\u30f3\u30e1\u30e2\u30ea\u30b9\u30c8\u30ec\u30fc\u30b8\u3092\u5404Executor\u5185\u3067\u52d5\u4f5c\u3059\u308b\u30d6\u30ed\u30c3\u30af\u30de\u30cd\u30fc\u30b8\u30e3\u30fc\u3068\u547c\u3070\u308c\u308b\u30b5\u30fc\u30d3\u30b9\u3092\u901a\u3058\u3066\u63d0\u4f9b\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- Executor\u3084Driver\u306e\u8d77\u52d5\u3092Cluster Manager\u304c\u62c5\u3046\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- \u5171\u6709\u30ea\u30bd\u30fc\u30b9\u30d7\u30fc\u30eb\u4e0a\u3067\u69d8\u3005\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u30d5\u30ec\u30fc\u30e0\u30ef\u30fc\u30af\u3092\u52d5\u4f5c\u3055\u305b\u308b\u3053\u3068\u304c\u3067\u304d\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- Spark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306e\u4e2d\u6838\u3068\u306a\u308b\u306e\u306f\u3001Spark Driver\u3068\u306a\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- Driver\u306fSpark\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u30921\u3064\u4ee5\u4e0a\u306eSpark\u30b8\u30e7\u30d6\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- DAG\u30ce\u30fc\u30c9\u306e\u4e00\u90e8\u3068\u3057\u3066\u3001\u76f4\u5217\u307e\u305f\u306f\u4e26\u5217\u306b\u5b9f\u884c\u3067\u304d\u308b\u64cd\u4f5c\u306b\u57fa\u3065\u3044\u3066\u30b9\u30c6\u30fc\u30b8\u304c\u4f5c\u6210\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- \u5404\u30b9\u30c6\u30fc\u30b8\u306f\u3001Spark\u30bf\u30b9\u30af\u3067\u69cb\u6210\u3055\u308c\u3001\u5404Spark Executor\u9593\u3067\u30d5\u30a7\u30c7\u30ec\u30fc\u30b7\u30e7\u30f3\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- \u5404\u30bf\u30b9\u30af\u306f1\u3064\u306e\u30b3\u30a2\u306b\u30de\u30c3\u30d7\u3055\u308c\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
- \n
- SparkContext\u901a\u8fc7\u8bbf\u95eeCluster Manager\u6765\u5206\u914d\u8d44\u6e90\u3002<\/ol>\n<\/li>\n<\/ol>\n
- \n
- \u8fde\u63a5\u540e\uff0c\u5728\u96c6\u7fa4\u7684\u8282\u70b9\u4e0a\u83b7\u53d6Executor\u3002<\/ol>\n<\/li>\n<\/ol>\n
- \n
- SparkContext\u5c06\u5e94\u7528\u7a0b\u5e8f\uff08Jar\u6587\u4ef6\u6216Python\u6587\u4ef6\uff09\u5206\u53d1\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n<\/li>\n<\/ol>\n
- \n
- Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Executor\u304b\u3089\u306e\u901a\u4fe1\u3092Listen\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u3001\u30ef\u30fc\u30ab\u30fc\u30ce\u30fc\u30c9\u304b\u3089NW\u3067\u30a2\u30c9\u30ec\u30b9\u3092\u6307\u5b9a\u3067\u304d\u308b\u5fc5\u8981\u304c\u3042\u308b\u3002<\/ul>\n<\/li>\n<\/ul>\n
<\/p>\n
- Driver\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001Worker\u30ce\u30fc\u30c9\u3092\u5236\u5fa1\u3059\u308b\u305f\u3081\u53ef\u80fd\u306a\u9650\u308a\u8fd1\u304f\u3067\u5b9f\u884c\u3059\u308b\u3053\u3068\u304c\u63a8\u5968\u3055\u308c\u308b\u3002<\/ul>\n
\u8bf7\u8003\u5bdf<\/h1>\n
\u7531\u4e8e\u57fa\u672c\u6982\u5ff5\u5df2\u7ecf\u6f84\u6e05\uff0c\u6211\u8ba1\u5212\u5c06\u91cd\u70b9\u653e\u5728\u6574\u7406Spark\u7279\u8272\u4e2d\u7684\u5ef6\u8fdf\u8bc4\u4f30\u6267\u884c\u7b49\u7279\u6027\u548c\u53c2\u6570\u4e0a\u3002<\/p>\n
\u8bf7\u53c2\u8003<\/h1>\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]<\/p>\n","protected":false},"author":5,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-36589","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"\n
\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\" \/>\n<meta property=\"og:description\" content=\"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-12-16T03:55:01+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-04-29T17:03:25+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png\" \/>\n<meta name=\"author\" content=\"\u6e05, \u5b87\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u6e05, \u5b87\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"1 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\",\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-12-16T03:55:01+00:00\",\"dateModified\":\"2024-04-29T17:03:25+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e\",\"name\":\"\u6e05, \u5b87\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g\",\"caption\":\"\u6e05, \u5b87\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_locale":"zh_CN","og_type":"article","og_title":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406","og_description":"\u5373\u80cc\u666f\u548c\u76ee\u6807\u3002 \u518d\u6574\u7406\u4e00\u4e0bApache Spark\u7684\u67b6\u6784\u548c\u7ec4\u4ef6\u3002 \u7b80\u8ff0 \u7ec4\u4ef6 \u4ee5\u4e0b\u662f\u5728Spark\u4e2d\u4f7f\u7528\u7684\u672f\u8bed\u3002 […]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/\u5bf9apache-spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-12-16T03:55:01+00:00","article_modified_time":"2024-04-29T17:03:25+00:00","og_image":[{"url":"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d2a0f37434c4406c460b3\/4-0.png"}],"author":"\u6e05, \u5b87","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u6e05, \u5b87","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"1 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/","name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406 - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-12-16T03:55:01+00:00","dateModified":"2024-04-29T17:03:25+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"\u5bf9Apache Spark\u7684\u7ec4\u4ef6\u8fdb\u884c\u6574\u7406"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/1a6ecd3d914d22a5ac32791ffc1fbd8e","name":"\u6e05, \u5b87","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/4b2016c18459a605fc469c7566608f5686491baa112d0871ee613f61b7210565?s=96&d=mm&r=g","caption":"\u6e05, \u5b87"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/qingyu\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e5%af%b9apache-spark%e7%9a%84%e7%bb%84%e4%bb%b6%e8%bf%9b%e8%a1%8c%e6%95%b4%e7%90%86\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/5"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=36589"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions"}],"predecessor-version":[{"id":88735,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36589\/revisions\/88735"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=36589"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=36589"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=36589"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
<\/p>\n
- SparkContext\u5c06\u4efb\u52a1\u53d1\u9001\u7ed9\u6bcf\u4e2aExecutor\u3002<\/ol>\n
\u8bf7\u6ce8\u610f\u4e8b\u9879\u3002<\/h2>\n
- \n
- \n
<\/p>\n
- \n
- \n
<\/p>\n
- \n
- \n
<\/p>\n
- 16\u30b3\u30a2\u306eExecutor\u3067\u306f\u300116\u500b\u4ee5\u4e0a\u306e\u30bf\u30b9\u30af\u309216\u500b\u4ee5\u4e0a\u306e\u30d1\u30fc\u30c6\u30a3\u30b7\u30e7\u30f3\u3067\u4e26\u5217\u3067\u51e6\u7406\u3059\u308b\u3002<\/ul>\n
\u6267\u884c\u65f6\u5e8f<\/h2>\n
\u5728\u9a71\u52a8\u7a0b\u5e8f\u4e2d\uff0cSparkContext\u7a0b\u5e8f\u8fdb\u884c\u8c03\u6574\u548c\u63a7\u5236\u3002\u5927\u81f4\u6d41\u7a0b\u5982\u4e0b\u3002<\/p>\n
- \n
- \n
<\/p>\n
- \n
- \n
<\/p>\n
- \u3059\u3079\u3066\u306e\u64cd\u4f5c\u304c1\u3064\u306eStage\u306b\u306a\u308b\u308f\u3051\u3067\u306f\u306a\u304f\u3001\u8907\u6570\u306eStage\u3067\u5206\u5272\u3067\u304d\u308b\u3002<\/ul>\n
\u706b\u82b1\u4efb\u52a1<\/h3>\n
- \n
- \n
<\/p>\n
- \u6b21\u306b\u5404Spark \u30b8\u30e7\u30d6\u3092DAG\u306b\u5909\u63db\u3059\u308b\u3002<\/ul>\n
\u706b\u82b1\u9636\u6bb5<\/h3>\n
- \n
- \n
<\/p>\n
- Spark Driver\u3067\u306fSparkSession\u3092\u4f5c\u6210\u3059\u308b<\/ul>\n
Spark \u5de5\u4f5c<\/h3>\n
- \n
- \n
<\/p>\n
- YARN\u3067\u52d5\u4f5c\u3055\u305b\u308b\u5834\u5408\u306b\u3001Spark\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306f\u6c7a\u3081\u3089\u308c\u305f\u6570\u306eExecutor\u3092\u4f7f\u3046\u3002\u5404\u7a2e\u30d1\u30e9\u30e1\u30fc\u30bf\u306f\u4e0b\u8a18\u306e\u901a\u308a<\/ul>\n
\n\u8a2d\u5b9a\u9805\u76ee\u8aac\u660e–num-executorsExecutor\u306e\u6570–executor-memoryExecutor\u304c\u4f7f\u7528\u3059\u308b\u30e1\u30e2\u30ea–executor-corsYARN\u306b\u8981\u6c42\u3059\u308b\u30b3\u30a2\u6570<\/div>\n<\/div>\nSpark\u5e94\u7528\u7a0b\u5e8f<\/h3>\n
- \n
- \n
<\/p>\n
- Spark\u306eCluster Manager\u306f\u30d7\u30e9\u30ac\u30d6\u30eb\u306a\u69cb\u6210\u3068\u306a\u3063\u3066\u304a\u308a\u3001YARN\u3084Mesos\u306a\u3069\u304c\u5229\u7528\u3067\u304d\u308b\u3002<\/ul>\n
Hadoop YARN \u5206\u5e03\u5f0f\u7cfb\u7edf<\/h4>\n
- \n
- \n
RDD\u306fExecutor\u5185\u306b\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u308b\u306e\u3067\u3001\u30bf\u30b9\u30af\u306f\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u305f\u30c7\u30fc\u30bf\u3068\u540c\u5c45\u3057\u3066\u52d5\u4f5c\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n
\u96c6\u7fa4\u7ecf\u7406<\/h3>\n
- \n
- \n
<\/p>\n
- \n
- \n
<\/p>\n
- \n
- \n
<\/p>\n
- \n
- \n
Spark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u30e6\u30fc\u30b6\u306e\u30d7\u30ed\u30b0\u30e9\u30e0\u3092\u5b9f\u884c\u306e\u5358\u4f4d\uff08\u30bf\u30b9\u30af\uff09\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001\u8ad6\u7406\u7684\u306aDAG\u3092\u751f\u6210\u3057\u3001\u30c9\u30e9\u30a4\u30d0\u304c\u5b9f\u884c\u3055\u308c\u308b\u3068\u3001\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u306b\u5909\u63db\u3059\u308b\u3002
\nSpark\u306f\u3001map\u306e\u5909\u63db\u3092\u7e8f\u3081\u3066\u30de\u30fc\u30b8\u3059\u308b\u305f\u3081\u306e\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u5316\u306e\u3088\u3046\u306a\u6700\u9069\u5316\u3092\u884c\u3044\u3001\u5b9f\u884c\u30b0\u30e9\u30d5\u3092\u4e00\u9023\u306e\u30b9\u30c6\u30fc\u30b8\u7fa4\u306b\u5909\u63db\u3059\u308b\u3002\u305d\u308c\u305e\u308c\u306e\u30b9\u30c6\u30fc\u30b8\u306b\u306f\u8907\u6570\u306e\u30bf\u30b9\u30af\u304c\u542b\u307e\u308c\u308b\u3002
\n\u30bf\u30b9\u30af\u306fSpark\u3067\u6700\u3082\u5c0f\u3055\u306a\u5358\u4f4d<\/p>\n\u30a8\u30b0\u30bc\u30ad\u30e5\u30fc\u30bf\u4e0a\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3092\u884c\u3046<\/p>\n
\u7269\u7406\u7684\u306a\u5b9f\u884c\u8a08\u753b\u304c\u6c7a\u307e\u3063\u305f\u5f8c\u3001Spark\u30c9\u30e9\u30a4\u30d0\u306f\u3001Executor\u7fa4\u306e\u500b\u3005\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3092\u8abf\u6574\u3059\u308b\u3002
\nExecutor\u7fa4\u306f\u3001\u8d77\u52d5\u6642\u306b\u81ea\u5206\u81ea\u8eab\u3092\u30c9\u30e9\u30a4\u30d0\u306b\u5bfe\u3057\u3066\u767b\u9332\u3059\u308b\u3053\u3068\u3067\u3001\u30c9\u30e9\u30a4\u30d0\u306f\u81ea\u5206\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u306eExecutor\u306e\u72b6\u6cc1\u3092\u5e38\u306b\u628a\u63e1\u3067\u304d\u308b\u3002
\n\u5404Executor\u306f\u3001\u30bf\u30b9\u30af\u3092\u5b9f\u884c\u3057\u3001RDD\u306e\u30c7\u30fc\u30bf\u3092\u4fdd\u6301\u3059\u308b\u4e8b\u304c\u3067\u304d\u308b\u30d7\u30ed\u30bb\u30b9\u3002
\nSpark\u30c9\u30e9\u30a4\u30d0\u306f\u3001\u305d\u306e\u6642\u70b9\u3067\u306eExecutor\u7fa4\u3092\u898b\u3066\u3001\u30c7\u30fc\u30bf\u306e\u914d\u7f6e\u306b\u57fa\u3065\u3044\u3066\u5404\u30bf\u30b9\u30af\u3092\u9069\u5207\u306a\u5834\u6240\u306b\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u3057\u3088\u3046\u3068\u3059\u308b\u3002<\/p>\n\u30ad\u30e3\u30c3\u30b7\u30e5\u3055\u308c\u3066\u3044\u305f\u5834\u5408\u3082\u3001\u305d\u306e\u30c7\u30fc\u30bf\u306b\u30a2\u30af\u30bb\u30b9\u3059\u308b\u5c06\u6765\u306e\u30bf\u30b9\u30af\u306e\u30b9\u30b1\u30b8\u30e5\u30fc\u30ea\u30f3\u30b0\u306b\u3001\u3053\u306e\u60c5\u5831\u3092\u6d3b\u7528\u3059\u308b\u3002<\/p>\n
\u30c9\u30e9\u30a4\u30d0\u306f\u3001Web\u30a4\u30f3\u30bf\u30d5\u30a7\u30a4\u30b9\u3092\u901a\u3058\u3066\u5b9f\u884c\u4e2d\u306eSpark\u30a2\u30d7\u30ea\u306b\u95a2\u3059\u308b\u60c5\u5831\u3092\u516c\u958b\u3059\u308b<\/p>\n
\u6267\u884c\u8005\u3002<\/h3>\n
- \n
- \n
<\/p>\n
- \n
- \n
<\/p>\n
- \n
- \n