{"id":46850,"date":"2023-05-18T20:51:23","date_gmt":"2023-10-17T20:19:49","guid":{"rendered":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/"},"modified":"2024-04-30T04:50:36","modified_gmt":"2024-04-29T20:50:36","slug":"%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage","status":"publish","type":"post","link":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/","title":{"rendered":"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage"},"content":{"rendered":"<pre class=\"post-pre\"><code>    <span class=\"k\">val<\/span> <span class=\"nv\">path<\/span> <span class=\"k\">=<\/span> <span class=\"nv\">paths<\/span><span class=\"o\">.<\/span><span class=\"py\">map<\/span><span class=\"o\">(<\/span><span class=\"nv\">_<\/span><span class=\"o\">.<\/span><span class=\"py\">path<\/span><span class=\"o\">)<\/span> <span class=\"n\">mkString<\/span> <span class=\"s\">\", \"<\/span>\r\n    <span class=\"k\">val<\/span> <span class=\"nv\">clustername<\/span> <span class=\"k\">=<\/span> <span class=\"nv\">System<\/span><span class=\"o\">.<\/span><span class=\"py\">getProperty<\/span><span class=\"o\">(<\/span><span class=\"s\">\"cluster.name\"<\/span><span class=\"o\">)<\/span>\r\n    <span class=\"k\">val<\/span> <span class=\"nv\">absolutebasepath<\/span> <span class=\"k\">=<\/span> <span class=\"nv\">System<\/span><span class=\"o\">.<\/span><span class=\"py\">getProperty<\/span><span class=\"o\">(<\/span><span class=\"s\">\"absolute.base.path\"<\/span><span class=\"o\">)<\/span>\r\n    <span class=\"k\">val<\/span> <span class=\"nv\">upath<\/span> <span class=\"k\">=<\/span> <span class=\"nv\">path<\/span><span class=\"o\">.<\/span><span class=\"py\">replace<\/span><span class=\"o\">(<\/span><span class=\"n\">absolutebasepath<\/span><span class=\"o\">,<\/span><span class=\"s\">\"\"<\/span><span class=\"o\">)<\/span>\r\n    <span class=\"k\">new<\/span> <span class=\"nc\">EndpointDataset<\/span><span class=\"o\">(<\/span><span class=\"n\">name<\/span><span class=\"o\">,<\/span> <span class=\"n\">qualifiedName<\/span><span class=\"o\">,<\/span> <span class=\"n\">attributes<\/span><span class=\"o\">,<\/span> <span class=\"k\">new<\/span> <span class=\"nc\">FileEndpoint<\/span><span class=\"o\">(<\/span><span class=\"n\">upath<\/span><span class=\"o\">,<\/span> <span class=\"n\">upath<\/span><span class=\"o\">+<\/span><span class=\"s\">\"@\"<\/span><span class=\"o\">+<\/span> <span class=\"n\">clustername<\/span><span class=\"o\">),<\/span> <span class=\"nv\">EndpointType<\/span><span class=\"o\">.<\/span><span class=\"py\">file<\/span><span class=\"o\">,<\/span> <span class=\"nv\">EndpointDirection<\/span><span class=\"o\">.<\/span><span class=\"py\">input<\/span><span class=\"o\">,<\/span> <span class=\"n\">st<\/span><span class=\"o\">)<\/span>\r\n<\/code><\/pre>\n<pre class=\"post-pre\"><code>    <span class=\"n\">spark<\/span><span class=\"o\">-<\/span><span class=\"n\">shell<\/span>  <span class=\"o\">--<\/span><span class=\"n\">master<\/span> <span class=\"n\">yarn<\/span>  <span class=\"o\">--<\/span><span class=\"n\">driver<\/span><span class=\"o\">-<\/span><span class=\"n\">java<\/span><span class=\"o\">-<\/span><span class=\"n\">options<\/span><span class=\"o\">='-<\/span><span class=\"nv\">Dspline<\/span><span class=\"o\">.<\/span><span class=\"py\">persistence<\/span><span class=\"o\">.<\/span><span class=\"py\">factory<\/span><span class=\"k\">=<\/span><span class=\"nv\">za<\/span><span class=\"o\">.<\/span><span class=\"py\">co<\/span><span class=\"o\">.<\/span><span class=\"py\">absa<\/span><span class=\"o\">.<\/span><span class=\"py\">spline<\/span><span class=\"o\">.<\/span><span class=\"py\">persistence<\/span><span class=\"o\">.<\/span><span class=\"py\">atlas<\/span><span class=\"o\">.<\/span><span class=\"py\">AtlasPersistenceFactory<\/span><span class=\"o\">'<\/span> <span class=\"o\">--<\/span><span class=\"n\">files<\/span> <span class=\"o\">\/<\/span><span class=\"n\">usr<\/span><span class=\"o\">\/<\/span><span class=\"n\">hdp<\/span><span class=\"o\">\/<\/span><span class=\"mf\">2.6<\/span><span class=\"o\">.<\/span><span class=\"mf\">4.0<\/span><span class=\"o\">-<\/span><span class=\"mi\">91<\/span><span class=\"o\">\/<\/span><span class=\"n\">kafka<\/span><span class=\"o\">\/<\/span><span class=\"n\">conf<\/span><span class=\"o\">\/<\/span><span class=\"nv\">producer<\/span><span class=\"o\">.<\/span><span class=\"py\">properties<\/span> <span class=\"o\">--<\/span><span class=\"n\">conf<\/span> <span class=\"ss\">'spark<\/span><span class=\"o\">.<\/span><span class=\"py\">driver<\/span><span class=\"o\">.<\/span><span class=\"py\">extraJavaOptions<\/span><span class=\"o\">=-<\/span><span class=\"nv\">Datlas<\/span><span class=\"o\">.<\/span><span class=\"py\">kafka<\/span><span class=\"o\">.<\/span><span class=\"py\">bootstrap<\/span><span class=\"o\">.<\/span><span class=\"py\">servers<\/span><span class=\"k\">=<\/span><span class=\"n\">hdp264<\/span><span class=\"o\">-<\/span><span class=\"mf\">0.f<\/span><span class=\"nv\">ield<\/span><span class=\"o\">.<\/span><span class=\"py\">hortonworks<\/span><span class=\"o\">.<\/span><span class=\"py\">com<\/span><span class=\"k\">:<\/span><span class=\"err\">6667<\/span> <span class=\"kt\">-Dbootstrap.servers<\/span><span class=\"o\">=<\/span><span class=\"n\">hdp264<\/span><span class=\"o\">-<\/span><span class=\"mf\">0.f<\/span><span class=\"nv\">ield<\/span><span class=\"o\">.<\/span><span class=\"py\">hortonworks<\/span><span class=\"o\">.<\/span><span class=\"py\">com<\/span><span class=\"k\">:<\/span><span class=\"err\">6667<\/span> <span class=\"kt\">-Dspline.persistence.factory<\/span><span class=\"o\">=<\/span><span class=\"nv\">za<\/span><span class=\"o\">.<\/span><span class=\"py\">co<\/span><span class=\"o\">.<\/span><span class=\"py\">absa<\/span><span class=\"o\">.<\/span><span class=\"py\">spline<\/span><span class=\"o\">.<\/span><span class=\"py\">persistence<\/span><span class=\"o\">.<\/span><span class=\"py\">atlas<\/span><span class=\"o\">.<\/span><span class=\"py\">AtlasPersistenceFactory<\/span> <span class=\"o\">-<\/span><span class=\"nv\">Datlas<\/span><span class=\"o\">.<\/span><span class=\"py\">kafka<\/span><span class=\"o\">.<\/span><span class=\"py\">auto<\/span><span class=\"o\">.<\/span><span class=\"py\">commit<\/span><span class=\"o\">.<\/span><span class=\"py\">enable<\/span><span class=\"k\">=<\/span><span class=\"kc\">false<\/span> <span class=\"o\">-<\/span><span class=\"nv\">Datlas<\/span><span class=\"o\">.<\/span><span class=\"py\">kafka<\/span><span class=\"o\">.<\/span><span class=\"py\">hook<\/span><span class=\"o\">.<\/span><span class=\"py\">group<\/span><span class=\"o\">.<\/span><span class=\"py\">id<\/span><span class=\"k\">=<\/span><span class=\"n\">atlas<\/span> <span class=\"o\">-<\/span><span class=\"nv\">Datlas<\/span><span class=\"o\">.<\/span><span class=\"py\">kafka<\/span><span class=\"o\">.<\/span><span class=\"py\">zookeeper<\/span><span class=\"o\">.<\/span><span class=\"py\">connect<\/span><span class=\"k\">=<\/span><span class=\"n\">hdp264<\/span><span class=\"o\">-<\/span><span class=\"mf\">0.f<\/span><span class=\"nv\">ield<\/span><span class=\"o\">.<\/span><span class=\"py\">hortonworks<\/span><span class=\"o\">.<\/span><span class=\"py\">com<\/span><span class=\"k\">:<\/span><span class=\"err\">2181<\/span> <span class=\"kt\">-Datlas.kafka.zookeeper.connection.timeout.ms<\/span><span class=\"o\">=<\/span><span class=\"mi\">30000<\/span> <span class=\"o\">-<\/span><span class=\"nv\">Datlas<\/span><span class=\"o\">.<\/span><span class=\"py\">kafka<\/span><span class=\"o\">.<\/span><span class=\"py\">zookeeper<\/span><span class=\"o\">.<\/span><span class=\"py\">session<\/span><span class=\"o\">.<\/span><span class=\"py\">timeout<\/span><span class=\"o\">.<\/span><span class=\"py\">ms<\/span><span class=\"k\">=<\/span><span class=\"mi\">60000<\/span> <span class=\"o\">-<\/span><span class=\"nv\">Datlas<\/span><span class=\"o\">.<\/span><span class=\"py\">kafka<\/span><span class=\"o\">.<\/span><span class=\"py\">zookeeper<\/span><span class=\"o\">.<\/span><span class=\"py\">sync<\/span><span class=\"o\">.<\/span><span class=\"py\">time<\/span><span class=\"o\">.<\/span><span class=\"py\">ms<\/span><span class=\"k\">=<\/span><span class=\"mi\">20<\/span> <span class=\"o\">-<\/span><span class=\"nv\">Dcluster<\/span><span class=\"o\">.<\/span><span class=\"py\">name<\/span><span class=\"k\">=<\/span><span class=\"n\">hdp264<\/span> <span class=\"o\">-<\/span><span class=\"nv\">Dabsolute<\/span><span class=\"o\">.<\/span><span class=\"py\">base<\/span><span class=\"o\">.<\/span><span class=\"py\">path<\/span><span class=\"k\">=<\/span><span class=\"n\">hdfs<\/span><span class=\"o\">:\/\/<\/span><span class=\"n\">hdp264<\/span><span class=\"o\">-<\/span><span class=\"mf\">0.f<\/span><span class=\"nv\">ield<\/span><span class=\"o\">.<\/span><span class=\"py\">hortonworks<\/span><span class=\"o\">.<\/span><span class=\"py\">com<\/span><span class=\"k\">:<\/span><span class=\"err\">8020<\/span><span class=\"kt\">'<\/span>\r\n    <span class=\"kt\">import<\/span> <span class=\"kt\">za.co.absa.spline.core.SparkLineageInitializer._<\/span>\r\n    <span class=\"nv\">spark<\/span><span class=\"o\">.<\/span><span class=\"py\">enableLineageTracking<\/span><span class=\"o\">()<\/span>\r\n    <span class=\"k\">import<\/span> <span class=\"nn\">org.apache.spark.sql.SaveMode<\/span>\r\n    <span class=\"k\">val<\/span> <span class=\"nv\">sourceDS<\/span> <span class=\"k\">=<\/span> <span class=\"nv\">spark<\/span><span class=\"o\">.<\/span><span class=\"py\">read<\/span><span class=\"o\">.<\/span><span class=\"py\">option<\/span><span class=\"o\">(<\/span><span class=\"s\">\"header\"<\/span><span class=\"o\">,<\/span><span class=\"s\">\"true\"<\/span><span class=\"o\">).<\/span><span class=\"py\">option<\/span><span class=\"o\">(<\/span><span class=\"s\">\"inferSchema\"<\/span><span class=\"o\">,<\/span><span class=\"s\">\"true\"<\/span><span class=\"o\">).<\/span><span class=\"py\">csv<\/span><span class=\"o\">(<\/span><span class=\"s\">\"\/user\/nifi\/data\/wikidata.csv\"<\/span><span class=\"o\">).<\/span><span class=\"py\">as<\/span><span class=\"o\">(<\/span><span class=\"s\">\"source\"<\/span><span class=\"o\">).<\/span><span class=\"py\">filter<\/span><span class=\"o\">(<\/span><span class=\"n\">$<\/span><span class=\"s\">\"total_response_size\"<\/span> <span class=\"o\">&gt;<\/span> <span class=\"mi\">1000<\/span><span class=\"o\">).<\/span><span class=\"py\">filter<\/span><span class=\"o\">(<\/span><span class=\"n\">$<\/span><span class=\"s\">\"count_views\"<\/span> <span class=\"o\">&gt;<\/span> <span class=\"mi\">10<\/span><span class=\"o\">)<\/span>\r\n    <span class=\"k\">val<\/span> <span class=\"nv\">domainMappingDS<\/span> <span class=\"k\">=<\/span><span class=\"nv\">spark<\/span><span class=\"o\">.<\/span><span class=\"py\">read<\/span><span class=\"o\">.<\/span><span class=\"py\">option<\/span><span class=\"o\">(<\/span><span class=\"s\">\"header\"<\/span><span class=\"o\">,<\/span><span class=\"s\">\"true\"<\/span><span class=\"o\">).<\/span><span class=\"py\">option<\/span><span class=\"o\">(<\/span><span class=\"s\">\"inferSchema\"<\/span><span class=\"o\">,<\/span><span class=\"s\">\"true\"<\/span><span class=\"o\">).<\/span><span class=\"py\">csv<\/span><span class=\"o\">(<\/span><span class=\"s\">\"\/user\/nifi\/data\/domain.csv\"<\/span><span class=\"o\">).<\/span><span class=\"py\">as<\/span><span class=\"o\">(<\/span><span class=\"s\">\"mapping\"<\/span><span class=\"o\">)<\/span>\r\n    <span class=\"k\">val<\/span> <span class=\"nv\">joinedDS<\/span> <span class=\"k\">=<\/span> <span class=\"nv\">sourceDS<\/span><span class=\"o\">.<\/span><span class=\"py\">join<\/span><span class=\"o\">(<\/span><span class=\"n\">domainMappingDS<\/span><span class=\"o\">,<\/span> <span class=\"n\">$<\/span><span class=\"s\">\"domain_code\"<\/span> <span class=\"o\">===<\/span><span class=\"n\">$<\/span><span class=\"s\">\"d_code\"<\/span><span class=\"o\">,<\/span><span class=\"s\">\"left_outer\"<\/span><span class=\"o\">).<\/span><span class=\"py\">select<\/span><span class=\"o\">(<\/span><span class=\"n\">$<\/span><span class=\"s\">\"page_title\"<\/span><span class=\"o\">.<\/span><span class=\"py\">as<\/span><span class=\"o\">(<\/span><span class=\"s\">\"page\"<\/span><span class=\"o\">),<\/span><span class=\"n\">$<\/span><span class=\"s\">\"d_name\"<\/span><span class=\"o\">.<\/span><span class=\"py\">as<\/span><span class=\"o\">(<\/span><span class=\"s\">\"domain\"<\/span><span class=\"o\">),<\/span> <span class=\"n\">$<\/span><span class=\"s\">\"count_views\"<\/span><span class=\"o\">)<\/span>\r\n    <span class=\"nv\">joinedDS<\/span><span class=\"o\">.<\/span><span class=\"py\">write<\/span><span class=\"o\">.<\/span><span class=\"py\">mode<\/span><span class=\"o\">(<\/span><span class=\"nv\">SaveMode<\/span><span class=\"o\">.<\/span><span class=\"py\">Overwrite<\/span><span class=\"o\">).<\/span><span class=\"py\">format<\/span><span class=\"o\">(<\/span><span class=\"s\">\"orc\"<\/span><span class=\"o\">).<\/span><span class=\"py\">save<\/span><span class=\"o\">(<\/span><span class=\"s\">\"\/user\/nifi\/sparkoutput\"<\/span><span class=\"o\">)<\/span>\r\n<\/code><\/pre>\n<div><img decoding=\"async\" class=\"post-images\" title=\"\" src=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d7af3913a08637a697b19\/2-0.png\" alt=\"image.png\" \/><\/div>\n<div><img decoding=\"async\" class=\"post-images\" title=\"\" src=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d7af3913a08637a697b19\/3-0.png\" alt=\"image.png\" \/><\/div>\n","protected":false},"excerpt":{"rendered":"<p>val path = paths.map(_.path) mkString &#8220;, &#8221; val clustern [&hellip;]<\/p>\n","protected":false},"author":10,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-46850","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"<!-- This site is optimized with the Yoast SEO Premium plugin v21.5 (Yoast SEO v21.5) - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528atlas\u663e\u793aspark\u7684lineage\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage\" \/>\n<meta property=\"og:description\" content=\"val path = paths.map(_.path) mkString &quot;, &quot; val clustern [&hellip;]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528atlas\u663e\u793aspark\u7684lineage\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-10-17T20:19:49+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-04-29T20:50:36+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d7af3913a08637a697b19\/2-0.png\" \/>\n<meta name=\"author\" content=\"\u5b87, \u534e\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u5b87, \u534e\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"1 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/\",\"name\":\"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-10-17T20:19:49+00:00\",\"dateModified\":\"2024-04-29T20:50:36+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/513018e4e121d3add1b7c5de8be21458\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/513018e4e121d3add1b7c5de8be21458\",\"name\":\"\u5b87, \u534e\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/63cd45cbc05a35fc4ff7637a163c83c4962ef58d27472726c3a3e0c9c5194f0f?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/63cd45cbc05a35fc4ff7637a163c83c4962ef58d27472726c3a3e0c9c5194f0f?s=96&d=mm&r=g\",\"caption\":\"\u5b87, \u534e\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/yuhua\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528atlas\u663e\u793aspark\u7684lineage\/","og_locale":"zh_CN","og_type":"article","og_title":"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage","og_description":"val path = paths.map(_.path) mkString \", \" val clustern [&hellip;]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528atlas\u663e\u793aspark\u7684lineage\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-10-17T20:19:49+00:00","article_modified_time":"2024-04-29T20:50:36+00:00","og_image":[{"url":"https:\/\/cdn.silicloud.com\/blog-img\/blog\/img\/657d7af3913a08637a697b19\/2-0.png"}],"author":"\u5b87, \u534e","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u5b87, \u534e","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"1 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/","name":"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-10-17T20:19:49+00:00","dateModified":"2024-04-29T20:50:36+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/513018e4e121d3add1b7c5de8be21458"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"\u4f7f\u7528Atlas\u663e\u793aSpark\u7684Lineage"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/513018e4e121d3add1b7c5de8be21458","name":"\u5b87, \u534e","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/63cd45cbc05a35fc4ff7637a163c83c4962ef58d27472726c3a3e0c9c5194f0f?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/63cd45cbc05a35fc4ff7637a163c83c4962ef58d27472726c3a3e0c9c5194f0f?s=96&d=mm&r=g","caption":"\u5b87, \u534e"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/yuhua\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8atlas%e6%98%be%e7%a4%baspark%e7%9a%84lineage\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/46850","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/10"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=46850"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/46850\/revisions"}],"predecessor-version":[{"id":89793,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/46850\/revisions\/89793"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=46850"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=46850"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=46850"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}