{"id":41227,"date":"2023-09-18T11:25:41","date_gmt":"2023-10-27T10:41:58","guid":{"rendered":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/"},"modified":"2024-01-15T10:53:21","modified_gmt":"2024-01-15T02:53:21","slug":"%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82","status":"publish","type":"post","link":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/","title":{"rendered":"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch"},"content":{"rendered":"<p>\u5728Amazon RDS\u4e2d\uff0c\u9ed8\u8ba4\u914d\u7f6e\u4e0b\uff0c\u6162\u67e5\u8be2\u65e5\u5fd7\u88ab\u4fdd\u5b58\u5728\u540d\u4e3amysql.slow_log\u7684\u8868\u4e2d\u3002\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528Embulk\u5c06\u5b58\u50a8\u5728mysql.slow_log\u4e2d\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165\u5230Elasticsearch\u4e2d\u8fdb\u884c\u6295\u5165\u3002<\/p>\n<h1>\u5b89\u88c5\u6240\u9700\u7684\u63d2\u4ef6\u3002<\/h1>\n<p>\u4f7f\u7528embulk mkbundle\uff0c\u53ef\u4ee5\u5c06\u6240\u9700\u7684\u63d2\u4ef6\u901a\u8fc7Gemfile\u8fdb\u884c\u7ba1\u7406\u3002<\/p>\n<pre class=\"post-pre\"><code>$ embulk mkbundle embulk-aurora-move-slowlog\r\n$ cd embulk-move-slowlog\r\n<\/code><\/pre>\n<p>\u5728Gemfile\u4e2d\u6dfb\u52a0\u6240\u9700\u7684\u63d2\u4ef6\u3002\u7531\u4e8e\u672c\u6b21\u64cd\u4f5c\u662f\u5c06RDS\u6570\u636e\u5e93\u4e2d\u7684slowlog\u6295\u5165\u5230Elasticsearch\u4e2d\uff0c\u56e0\u6b64\u5728Gemfile\u4e2d\u52a0\u5165\u4ee5\u4e0b\u7684\u63cf\u8ff0\u3002<\/p>\n<pre class=\"post-pre\"><code>source 'https:\/\/rubygems.org\/'\r\ngem 'embulk', '~&gt; 0.8.0'\r\ngem 'embulk-input-mysql', '~&gt; 0.8.2'\r\ngem 'embulk-output-elasticsearch_ruby', '~&gt; 0.1.4'\r\ngem 'embulk-filter-typecast', '~&gt; 0.1.5'\r\ngem 'embulk-filter-column', '~&gt; 0.6.0'\r\n<\/code><\/pre>\n<p>\u5b8c\u6210Gemfile\u540e\uff0c\u4f7f\u7528\u4ee5\u4e0b\u547d\u4ee4\u6765\u5b89\u88c5\u63d2\u4ef6\u3002<\/p>\n<pre class=\"post-pre\"><code>$ embulk bundle\r\n<\/code><\/pre>\n<h1>\u521b\u5efa Embulk \u7684\u914d\u7f6e\u6587\u4ef6<\/h1>\n<p>\u521b\u5efa\u4e00\u4e2a\u7c7b\u4f3c\u4ee5\u4e0b\u7684\u914d\u7f6e\u6587\u4ef6\u3002<\/p>\n<pre class=\"post-pre\"><code><span class=\"na\">in<\/span><span class=\"pi\">:<\/span>\r\n  <span class=\"na\">type<\/span><span class=\"pi\">:<\/span> <span class=\"s\">mysql<\/span>\r\n  <span class=\"na\">user<\/span><span class=\"pi\">:<\/span> <span class=\"s\">xxxxxx<\/span>\r\n  <span class=\"na\">password<\/span><span class=\"pi\">:<\/span> <span class=\"s\">yyyyy<\/span>\r\n  <span class=\"na\">database<\/span><span class=\"pi\">:<\/span> <span class=\"s\">mysql<\/span>\r\n  <span class=\"na\">table<\/span><span class=\"pi\">:<\/span> <span class=\"s\">slow_log<\/span>\r\n  <span class=\"na\">host<\/span><span class=\"pi\">:<\/span> <span class=\"s\">zzzzzz.ap-northeast-1.rds.amazonaws.com<\/span>\r\n  <span class=\"na\">select<\/span><span class=\"pi\">:<\/span> <span class=\"s2\">\"<\/span><span class=\"s\">*\"<\/span>\r\n  <span class=\"na\">column_options<\/span><span class=\"pi\">:<\/span>\r\n    <span class=\"na\">start_time<\/span><span class=\"pi\">:<\/span> <span class=\"pi\">{<\/span><span class=\"nv\">type<\/span><span class=\"pi\">:<\/span> <span class=\"nv\">string<\/span><span class=\"pi\">,<\/span> <span class=\"nv\">timestamp_format<\/span><span class=\"pi\">:<\/span> <span class=\"s2\">\"<\/span><span class=\"s\">%Y-%m-%d<\/span> <span class=\"s\">%H:%M:%S+09:00\"<\/span><span class=\"pi\">}<\/span>\r\n  <span class=\"na\">incremental<\/span><span class=\"pi\">:<\/span> <span class=\"no\">true<\/span>\r\n  <span class=\"na\">incremental_columns<\/span><span class=\"pi\">:<\/span> <span class=\"pi\">[<\/span><span class=\"nv\">start_time<\/span><span class=\"pi\">]<\/span>\r\n  <span class=\"na\">options<\/span><span class=\"pi\">:<\/span>\r\n    <span class=\"na\">useLegacyDatetimeCode<\/span><span class=\"pi\">:<\/span> <span class=\"no\">false<\/span>\r\n    <span class=\"na\">serverTimezone<\/span><span class=\"pi\">:<\/span> <span class=\"s\">UTC<\/span>\r\n<span class=\"na\">filters<\/span><span class=\"pi\">:<\/span>\r\n  <span class=\"pi\">-<\/span> <span class=\"na\">type<\/span><span class=\"pi\">:<\/span> <span class=\"s\">typecast<\/span>\r\n    <span class=\"na\">columns<\/span><span class=\"pi\">:<\/span>\r\n      <span class=\"pi\">-<\/span> <span class=\"pi\">{<\/span><span class=\"nv\">name<\/span><span class=\"pi\">:<\/span> <span class=\"nv\">query_time<\/span><span class=\"pi\">,<\/span> <span class=\"nv\">type<\/span><span class=\"pi\">:<\/span> <span class=\"nv\">long<\/span><span class=\"pi\">}<\/span>\r\n      <span class=\"pi\">-<\/span> <span class=\"pi\">{<\/span><span class=\"nv\">name<\/span><span class=\"pi\">:<\/span> <span class=\"nv\">lock_time<\/span><span class=\"pi\">,<\/span> <span class=\"nv\">type<\/span><span class=\"pi\">:<\/span> <span class=\"nv\">long<\/span><span class=\"pi\">}<\/span>\r\n<span class=\"na\">out<\/span><span class=\"pi\">:<\/span>\r\n  <span class=\"na\">type<\/span><span class=\"pi\">:<\/span> <span class=\"s\">elasticsearch_ruby<\/span>\r\n  <span class=\"na\">mode<\/span><span class=\"pi\">:<\/span> <span class=\"s\">normal<\/span>\r\n  <span class=\"na\">nodes<\/span><span class=\"pi\">:<\/span>\r\n    <span class=\"pi\">-<\/span> <span class=\"pi\">{<\/span><span class=\"nv\">host<\/span><span class=\"pi\">:<\/span> <span class=\"s2\">\"<\/span><span class=\"s\">elasticsearch.example.com\"<\/span><span class=\"pi\">,<\/span> <span class=\"nv\">port<\/span><span class=\"pi\">:<\/span> <span class=\"nv\">9200<\/span> <span class=\"pi\">}<\/span>\r\n  <span class=\"na\">index<\/span><span class=\"pi\">:<\/span> <span class=\"s\">slow_log<\/span>\r\n  <span class=\"na\">index_type<\/span><span class=\"pi\">:<\/span> <span class=\"s\">log<\/span>\r\n  <span class=\"na\">request_timeout<\/span><span class=\"pi\">:<\/span> <span class=\"m\">60<\/span>\r\n<\/code><\/pre>\n<p>\u5173\u4e8e\u67e5\u8be2\u65f6\u95f4(query_time)\u548c\u9501\u5b9a\u65f6\u95f4(lock_time)\uff0c\u5728\u786e\u8ba4\u4e86\u6570\u636e\u5e93\u6a21\u5f0f\u540e\uff0c\u53d1\u73b0\u5b83\u4eec\u88ab\u5b58\u50a8\u4e3a\u65f6\u95f4\u7c7b\u578b(time)\u3002\u5982\u679c\u4fdd\u6301\u539f\u6837\u5bfc\u5165\u5230Elasticsearch\u4e2d\uff0c\u5c06\u65e0\u6cd5\u51c6\u786e\u4f20\u9012\u6570\u503c\uff0c\u56e0\u6b64\u6211\u4f7f\u7528embulk-filter-typecast\u5c06\u5176\u8f6c\u6362\u4e3a\u957f\u6574\u578b(long)\u3002\u53e6\u5916\uff0c\u5982\u679cRDS\u7684\u65f6\u533a\u4e3aJST\uff0c\u800cembulk\u91c7\u7528\u7684\u662fUTC\u65f6\u533a\uff0c\u5219\u53ef\u80fd\u5bfc\u81f4\u641c\u7d22\u5931\u8d25\uff0c\u6216\u8005\u5728\u4f7f\u7528Resume\u529f\u80fd\u65f6\u521b\u5efa\u7684\u6587\u4ef6\u65f6\u95f4\u88ab\u5012\u9000\u3002\u4e3a\u4e86\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\uff0c\u6211\u5c06\u5176\u8f6c\u6362\u4e3a\u5b57\u7b26\u4e32\uff0c\u5e76\u5728column_options\u4e2d\u6dfb\u52a0\u65f6\u533a+09:00\uff0c\u5c06\u5176\u5f53\u4f5cUTC\u5904\u7406\uff0c\u5e76\u5728Elasticsearch\u7684\u6620\u5c04\u4e2d\u8fdb\u884c\u65f6\u533a\u8c03\u6574\u3002<\/p>\n<h1>Elasticsearch\u7684\u6620\u5c04\u8bbe\u7f6e<\/h1>\n<p>\u7531\u4e8e\u524d\u8ff0\u7684\u539f\u56e0\uff0c\u6211\u4eec\u9700\u8981\u6307\u5b9astart_time\u7684format\u4e0etimestamp_format\u7684\u8f93\u51fa\u683c\u5f0f\u76f8\u5339\u914d\u3002<\/p>\n<pre class=\"post-pre\"><code><span class=\"p\">{<\/span>\r\n  <span class=\"nl\">\"template\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"slow_log-*\"<\/span><span class=\"p\">,<\/span>\r\n  <span class=\"nl\">\"mappings\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n    <span class=\"nl\">\"log\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n      <span class=\"nl\">\"properties\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n        <span class=\"nl\">\"start_time\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"date\"<\/span><span class=\"p\">,<\/span>\r\n          <span class=\"nl\">\"format\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"yyyy-MM-dd HH:mm:ssZZ\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"user_host\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"string\"<\/span><span class=\"p\">,<\/span>\r\n          <span class=\"nl\">\"index\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"not_analyzed\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"query_time\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"lock_time\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"row_sent\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"rows_examined\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"db\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"string\"<\/span><span class=\"p\">,<\/span>\r\n          <span class=\"nl\">\"index\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"not_analyzed\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"last_insert_id\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"insert_id\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"server_id\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"sql_text\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"string\"<\/span><span class=\"p\">,<\/span>\r\n          <span class=\"nl\">\"index\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"not_analyzed\"<\/span>\r\n        <span class=\"p\">},<\/span>\r\n        <span class=\"nl\">\"thread_id\"<\/span><span class=\"p\">:<\/span> <span class=\"p\">{<\/span>\r\n          <span class=\"nl\">\"type\"<\/span><span class=\"p\">:<\/span> <span class=\"s2\">\"long\"<\/span>\r\n        <span class=\"p\">}<\/span>\r\n      <span class=\"p\">}<\/span>\r\n    <span class=\"p\">}<\/span>\r\n  <span class=\"p\">}<\/span>\r\n<span class=\"p\">}<\/span>\r\n<\/code><\/pre>\n<p>\u4f7f\u7528\u4e0a\u8ff0\u7684slow_log.json\u4f5c\u4e3atemplate\u5c06\u5176\u6ce8\u518c\u5230Elasticsearch\u4e2d\u3002<\/p>\n<pre class=\"post-pre\"><code>$ curl -XPUT elasticsearch.example.com:9200\/_template\/slow_log -d \"$(cat slow_log.json)\"\r\n<\/code><\/pre>\n<p>\u4ee5\u4e0b\u662fslow_log\u7684\u6570\u636e\u5e93\u7ed3\u6784\uff0c\u9664\u4e86\u5c06type\u4e2d\u7684query_time\u548clock_time\u8bbe\u7f6e\u4e3along\u7c7b\u578b\u4e4b\u5916\uff0c\u5176\u4ed6\u5c3d\u53ef\u80fd\u6309\u7167mysql\u7684\u6a21\u5f0f\u8fdb\u884c\u4e86\u8bbe\u7f6e\u3002thread_id\u548cserver_id\u8fd9\u4e9b\u5b57\u6bb5\u53ef\u80fd\u6ca1\u6709\u4ec0\u4e48\u7528\u9014\uff0c\u6240\u4ee5\u4e5f\u8bb8\u53ef\u4ee5\u5c06\u5b83\u4eec\u8bbe\u7f6e\u4e3astring\u7c7b\u578b\u3002<\/p>\n<pre class=\"post-pre\"><code>mysql&gt; desc mysql.slow_log;\r\n+----------------+---------------------+------+-----+-------------------+-----------------------------+\r\n| Field          | Type                | Null | Key | Default           | Extra                       |\r\n+----------------+---------------------+------+-----+-------------------+-----------------------------+\r\n| start_time     | timestamp           | NO   |     | CURRENT_TIMESTAMP | on update CURRENT_TIMESTAMP |\r\n| user_host      | mediumtext          | NO   |     | NULL              |                             |\r\n| query_time     | time                | NO   |     | NULL              |                             |\r\n| lock_time      | time                | NO   |     | NULL              |                             |\r\n| rows_sent      | int(11)             | NO   |     | NULL              |                             |\r\n| rows_examined  | int(11)             | NO   |     | NULL              |                             |\r\n| db             | varchar(512)        | NO   |     | NULL              |                             |\r\n| last_insert_id | int(11)             | NO   |     | NULL              |                             |\r\n| insert_id      | int(11)             | NO   |     | NULL              |                             |\r\n| server_id      | int(10) unsigned    | NO   |     | NULL              |                             |\r\n| sql_text       | mediumtext          | NO   |     | NULL              |                             |\r\n| thread_id      | bigint(21) unsigned | NO   |     | NULL              |                             |\r\n+----------------+---------------------+------+-----+-------------------+-----------------------------+\r\n12 rows in set (0.00 sec)\r\n<\/code><\/pre>\n<h1>\u6267\u884cEmbulk<\/h1>\n<p>\u4f7f\u7528\u4ee5\u4e0b\u547d\u4ee4\u8fdb\u884c\u6267\u884c\uff1a<\/p>\n<p>$ embulk run config.yml -b .\/<\/p>\n<p>\u53e6\u5916\uff0c\u7531\u4e8eembulk-input-mysql\u652f\u6301Resume\u529f\u80fd\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u6765\u5bfc\u5165\u4e0a\u6b21\u6267\u884c\u540e\u7684\u8bb0\u5f55\u3002<\/p>\n<pre class=\"post-pre\"><code>$ embulk run config.yml -c diff.yml -b .\/\r\n<\/code><\/pre>\n<p>\u901a\u8fc7config.yml\u4e2d\u7684\u914d\u7f6eincremental: true\u548cincremental_columns: [start_time]\uff0c\u53ef\u4ee5\u4f7f\u5f97\u4e0a\u6b21\u5bfc\u5165\u7684start_time\u4ee5\u540e\u7684\u6570\u636e\u5728\u7b2c\u4e8c\u6b21\u53ca\u4ee5\u540e\u7684\u5bfc\u5165\u4e2d\u88ab\u63a5\u6536\u3002<br \/>\ndiff.yml\u5c06\u8f93\u51fa\u4ee5\u4e0b\u5185\u5bb9\uff0c\u5e76\u4e14\u6bcf\u6b21\u6267\u884cembulk\u65f6\uff0clast_record\u90e8\u5206\u90fd\u5c06\u88ab\u66f4\u65b0\u3002<\/p>\n<pre class=\"post-pre\"><code><span class=\"na\">in<\/span><span class=\"pi\">:<\/span>\r\n  <span class=\"na\">last_record<\/span><span class=\"pi\">:<\/span> <span class=\"pi\">[<\/span><span class=\"s1\">'<\/span><span class=\"s\">2017-03-29T21:00:02.000000Z'<\/span><span class=\"pi\">]<\/span>\r\n<span class=\"na\">out<\/span><span class=\"pi\">:<\/span> <span class=\"pi\">{}<\/span>\r\n<\/code><\/pre>\n<h1>\u7ba1\u7406Elasticsearch\u7684\u65e5\u5fd7\u65f6\u95f4<\/h1>\n<p>\u5728embulk-output-elasticsearch_ruby\u4e2d\uff0c\u901a\u8fc7\u5728\u7d22\u5f15\u540d\u79f0\u4e2d\u6dfb\u52a0%Y%m%d\uff0c\u53ef\u4ee5\u5b9e\u73b0\u6bcf\u5929\u66f4\u6539Elasticsearch\u7d22\u5f15\u540d\u79f0\u7684\u529f\u80fd\u3002<\/p>\n<pre class=\"post-pre\"><code><span class=\"na\">in<\/span><span class=\"pi\">:<\/span>\r\n <span class=\"na\">type<\/span><span class=\"pi\">:<\/span> <span class=\"s\">mysql<\/span>\r\n<span class=\"nn\">...<\/span>\r\n<span class=\"s\">\u4e2d\u7565<\/span>\r\n<span class=\"nn\">...<\/span>\r\n<span class=\"na\">out<\/span><span class=\"pi\">:<\/span>\r\n <span class=\"na\">type<\/span><span class=\"pi\">:<\/span> <span class=\"s\">elasticsearch_ruby<\/span>\r\n <span class=\"na\">mode<\/span><span class=\"pi\">:<\/span> <span class=\"s\">normal<\/span>\r\n <span class=\"na\">nodes<\/span><span class=\"pi\">:<\/span>\r\n <span class=\"pi\">-<\/span> <span class=\"pi\">{<\/span><span class=\"nv\">host<\/span><span class=\"pi\">:<\/span> <span class=\"s2\">\"<\/span><span class=\"s\">elasticsearch.example.com\"<\/span><span class=\"pi\">,<\/span> <span class=\"nv\">port<\/span><span class=\"pi\">:<\/span> <span class=\"nv\">9200<\/span> <span class=\"pi\">}<\/span>\r\n <span class=\"na\">index<\/span><span class=\"pi\">:<\/span> <span class=\"s\">slow_log-%Y%m%d<\/span>\r\n <span class=\"na\">index_type<\/span><span class=\"pi\">:<\/span> <span class=\"s\">log<\/span>\r\n <span class=\"na\">request_timeout<\/span><span class=\"pi\">:<\/span> <span class=\"m\">60<\/span>\r\n<\/code><\/pre>\n<p>\u901a\u8fc7\u5c06\u63cf\u8ff0\u8bbe\u7f6e\u4e3a\u7c7b\u4f3cslow_log-20170301\u7684\u7d22\u5f15\u540d\u79f0\uff0c\u53ef\u4ee5\u5c06\u6570\u636e\u63d2\u5165\uff0c\u5e76\u6309\u7167\u6620\u5c04\u6a21\u677f\u8bbe\u7f6e\u7684\u7d22\u5f15\u540d\u79f0\u8fdb\u884c\u5206\u9694\u3002\u8fd9\u6837\uff0c\u53ef\u4ee5\u6839\u636e\u6bcf\u5929\u7684\u7d22\u5f15\u540d\u79f0\u5c06\u7d22\u5f15\u5206\u5f00\u3002\u901a\u8fc7\u8fd9\u6837\u505a\uff0c\u6211\u8ba4\u4e3a\u53ef\u4ee5\u8f7b\u677e\u5220\u9664\u4e00\u5468\u524d\u63d2\u5165\u7684\u7d22\u5f15\u3002<\/p>\n<h1>\u6587\u4e2d\u63d0\u53ca<\/h1>\n<p>\u6211\u53c2\u8003\u4e86\u4ee5\u4e0b\u6587\u7ae0\u3002\u975e\u5e38\u611f\u8c22\u3002<br \/>\n1. \u4f7f\u7528Embulk\u5c06ELB\u7684\u65e5\u5fd7\u5bfc\u5165Elasticsearch\u7684\u8bb0\u5f55<br \/>\n2. \u4f7f\u7528embulk mkbundle\u7684\u65b9\u6cd5<br \/>\n3. \u6574\u7406\u4e86Fluentd\u7684\u6279\u5904\u7406\u7248\u672cEmbulk\u7684\u603b\u7ed3<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u5728Amazon RDS\u4e2d\uff0c\u9ed8\u8ba4\u914d\u7f6e\u4e0b\uff0c\u6162\u67e5\u8be2\u65e5\u5fd7\u88ab\u4fdd\u5b58\u5728\u540d\u4e3amysql.slow_log\u7684\u8868\u4e2d\u3002\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528E [&hellip;]<\/p>\n","protected":false},"author":11,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-41227","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"<!-- This site is optimized with the Yoast SEO Premium plugin v21.5 (Yoast SEO v21.5) - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528embulk\u5c06rds\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165elasticsearch\u3002\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch\" \/>\n<meta property=\"og:description\" content=\"\u5728Amazon RDS\u4e2d\uff0c\u9ed8\u8ba4\u914d\u7f6e\u4e0b\uff0c\u6162\u67e5\u8be2\u65e5\u5fd7\u88ab\u4fdd\u5b58\u5728\u540d\u4e3amysql.slow_log\u7684\u8868\u4e2d\u3002\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528E [&hellip;]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528embulk\u5c06rds\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165elasticsearch\u3002\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-10-27T10:41:58+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-01-15T02:53:21+00:00\" \/>\n<meta name=\"author\" content=\"\u65b0, \u97f5\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u65b0, \u97f5\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"2 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/\",\"name\":\"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-10-27T10:41:58+00:00\",\"dateModified\":\"2024-01-15T02:53:21+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9\",\"name\":\"\u65b0, \u97f5\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g\",\"caption\":\"\u65b0, \u97f5\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/yunxin\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528embulk\u5c06rds\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165elasticsearch\u3002\/","og_locale":"zh_CN","og_type":"article","og_title":"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch","og_description":"\u5728Amazon RDS\u4e2d\uff0c\u9ed8\u8ba4\u914d\u7f6e\u4e0b\uff0c\u6162\u67e5\u8be2\u65e5\u5fd7\u88ab\u4fdd\u5b58\u5728\u540d\u4e3amysql.slow_log\u7684\u8868\u4e2d\u3002\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528E [&hellip;]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528embulk\u5c06rds\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165elasticsearch\u3002\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-10-27T10:41:58+00:00","article_modified_time":"2024-01-15T02:53:21+00:00","author":"\u65b0, \u97f5","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u65b0, \u97f5","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"2 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/","name":"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-10-27T10:41:58+00:00","dateModified":"2024-01-15T02:53:21+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"\u4f7f\u7528Embulk\u5c06RDS\u7684\u6162\u67e5\u8be2\u65e5\u5fd7\u5bfc\u5165Elasticsearch"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/4ba4019495123db3038fd0809e6959c9","name":"\u65b0, \u97f5","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/d484b6c6e4ae82e8a9efea989e1d2af46d9b6ef128101e63b18f559fca0ae627?s=96&d=mm&r=g","caption":"\u65b0, \u97f5"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/yunxin\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8embulk%e5%b0%86rds%e7%9a%84%e6%85%a2%e6%9f%a5%e8%af%a2%e6%97%a5%e5%bf%97%e5%af%bc%e5%85%a5elasticsearch%e3%80%82\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/41227","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=41227"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/41227\/revisions"}],"predecessor-version":[{"id":58801,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/41227\/revisions\/58801"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=41227"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=41227"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=41227"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}