{"id":38,"date":"2016-03-16T15:48:43","date_gmt":"2016-03-16T07:48:43","guid":{"rendered":"http:\/\/jason19659.pw\/?p=38"},"modified":"2016-03-16T15:49:38","modified_gmt":"2016-03-16T07:49:38","slug":"%e5%a4%87%e4%bb%bd-hadoop-hdfs%e5%b8%b8%e7%94%a8%e6%96%87%e4%bb%b6%e6%93%8d%e4%bd%9c%e5%91%bd%e4%bb%a4","status":"publish","type":"post","link":"https:\/\/t2t2.cc\/?p=38","title":{"rendered":"\u5907\u4efd-hadoop HDFS\u5e38\u7528\u6587\u4ef6\u64cd\u4f5c\u547d\u4ee4"},"content":{"rendered":"

\u547d\u4ee4\u57fa\u672c\u683c\u5f0f:<\/h2>\n
hadoop<\/span> fs -cmd < args >\r\n<\/code><\/pre>\n
\n
1.ls<\/h3>\n
hadoop<\/span> fs -ls  \/\r\n<\/code><\/pre>\n
\u5217\u51fahdfs\u6587\u4ef6\u7cfb\u7edf\u6839\u76ee\u5f55\u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6<\/p>\n
hadoop<\/span> fs -ls -R \/\r\n<\/code><\/pre>\n
\u5217\u51fahdfs\u6587\u4ef6\u7cfb\u7edf\u6240\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6<\/p>\n
\n
2.put<\/h3>\n
hadoop fs -put < local<\/span> file<\/span> ><\/span> < hdfs<\/span> file<\/span> ><\/span>\r\n<\/code><\/pre>\n
hdfs file\u7684\u7236\u76ee\u5f55\u4e00\u5b9a\u8981\u5b58\u5728\uff0c\u5426\u5219\u547d\u4ee4\u4e0d\u4f1a\u6267\u884c<\/p>\n
hadoop fs -put  < local<\/span> file<\/span> or<\/span> dir<\/span> ><\/span>...< hdfs<\/span> dir<\/span> ><\/span>\r\n<\/code><\/pre>\n
hdfs dir \u4e00\u5b9a\u8981\u5b58\u5728\uff0c\u5426\u5219\u547d\u4ee4\u4e0d\u4f1a\u6267\u884c<\/p>\n
hadoop<\/span> fs -put - < hdsf  file>\r\n<\/code><\/pre>\n
\u4ece\u952e\u76d8\u8bfb\u53d6\u8f93\u5165\u5230hdfs file\u4e2d\uff0c\u6309Ctrl+D\u7ed3\u675f\u8f93\u5165\uff0chdfs file\u4e0d\u80fd\u5b58\u5728\uff0c\u5426\u5219\u547d\u4ee4\u4e0d\u4f1a\u6267\u884c<\/p>\n
2.1.moveFromLocal<\/h4>\n
hadoop fs -moveFromLocal  < local<\/span> src<\/span> ><\/span> ... < hdfs<\/span> dst<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u4e0eput\u76f8\u7c7b\u4f3c\uff0c\u547d\u4ee4\u6267\u884c\u540e\u6e90\u6587\u4ef6 local src \u88ab\u5220\u9664\uff0c\u4e5f\u53ef\u4ee5\u4ece\u4ece\u952e\u76d8\u8bfb\u53d6\u8f93\u5165\u5230hdfs file\u4e2d<\/p>\n
2.2.copyFromLocal<\/h4>\n
hadoop fs -copyFromLocal  < local<\/span> src<\/span> ><\/span> ... < hdfs<\/span> dst<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u4e0eput\u76f8\u7c7b\u4f3c\uff0c\u4e5f\u53ef\u4ee5\u4ece\u4ece\u952e\u76d8\u8bfb\u53d6\u8f93\u5165\u5230hdfs file\u4e2d<\/p>\n
\n
3.get<\/h3>\n
hadoop fs -get < hdfs<\/span> file<\/span> ><\/span> < local<\/span> file<\/span> or<\/span> dir<\/span>><\/span>\r\n<\/code><\/pre>\n
local file\u4e0d\u80fd\u548c hdfs file\u540d\u5b57\u4e0d\u80fd\u76f8\u540c\uff0c\u5426\u5219\u4f1a\u63d0\u793a\u6587\u4ef6\u5df2\u5b58\u5728\uff0c\u6ca1\u6709\u91cd\u540d\u7684\u6587\u4ef6\u4f1a\u590d\u5236\u5230\u672c\u5730<\/p>\n
hadoop fs -get < hdfs<\/span> file<\/span> or<\/span> dir<\/span> ><\/span> ... < local<\/span>  dir<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u62f7\u8d1d\u591a\u4e2a\u6587\u4ef6\u6216\u76ee\u5f55\u5230\u672c\u5730\u65f6\uff0c\u672c\u5730\u8981\u4e3a\u6587\u4ef6\u5939\u8def\u5f84
\n\u6ce8\u610f\uff1a<\/strong>\u5982\u679c\u7528\u6237\u4e0d\u662froot\uff0c local \u8def\u5f84\u8981\u4e3a\u7528\u6237\u6587\u4ef6\u5939\u4e0b\u7684\u8def\u5f84\uff0c\u5426\u5219\u4f1a\u51fa\u73b0\u6743\u9650\u95ee\u9898\uff0c<\/p>\n
3.1.moveToLocal<\/h4>\n
\u5f53\u524d\u7248\u672c\u4e2d\u8fd8\u672a\u5b9e\u73b0\u6b64\u547d\u4ee4<\/p>\n
3.2.copyToLocal<\/h4>\n
hadoop fs -copyToLocal < local<\/span> src<\/span> ><\/span> ... < hdfs<\/span> dst<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u4e0eget\u76f8\u7c7b\u4f3c<\/p>\n
\n
4.rm<\/h3>\n
hadoop fs -rm < hdfs<\/span> file<\/span> ><\/span> ...\r\nhadoop fs -rm -r < hdfs<\/span> dir<\/span>><\/span>...\r\n<\/code><\/pre>\n
\u6bcf\u6b21\u53ef\u4ee5\u5220\u9664\u591a\u4e2a\u6587\u4ef6\u6216\u76ee\u5f55<\/p>\n
\n
5.mkdir<\/h3>\n
hadoop<\/span> fs -mkdir < hdfs path>\r\n<\/code><\/pre>\n
\u53ea\u80fd\u4e00\u7ea7\u4e00\u7ea7\u7684\u5efa\u76ee\u5f55\uff0c\u7236\u76ee\u5f55\u4e0d\u5b58\u5728\u7684\u8bdd\u4f7f\u7528\u8fd9\u4e2a\u547d\u4ee4\u4f1a\u62a5\u9519<\/p>\n
hadoop<\/span> fs -mkdir -p < hdfs path> \r\n<\/code><\/pre>\n
\u6240\u521b\u5efa\u7684\u76ee\u5f55\u5982\u679c\u7236\u76ee\u5f55\u4e0d\u5b58\u5728\u5c31\u521b\u5efa\u8be5\u7236\u76ee\u5f55<\/p>\n
\n
6.getmerge<\/h3>\n
hadoop fs -getmerge < hdfs<\/span> dir<\/span> ><\/span>  < local<\/span> file<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u5c06hdfs\u6307\u5b9a\u76ee\u5f55\u4e0b\u6240\u6709\u6587\u4ef6\u6392\u5e8f\u540e\u5408\u5e76\u5230local\u6307\u5b9a\u7684\u6587\u4ef6<\/strong>\u4e2d\uff0c\u6587\u4ef6\u4e0d\u5b58\u5728\u65f6\u4f1a\u81ea\u52a8\u521b\u5efa\uff0c\u6587\u4ef6\u5b58\u5728\u65f6\u4f1a\u8986\u76d6\u91cc\u9762\u7684\u5185\u5bb9<\/p>\n
hadoop fs -getmerge -nl  < hdfs<\/span> dir<\/span> ><\/span>  < local<\/span> file<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u52a0\u4e0anl\u540e\uff0c\u5408\u5e76\u5230local file\u4e2d\u7684hdfs\u6587\u4ef6\u4e4b\u95f4\u4f1a\u7a7a\u51fa\u4e00\u884c<\/p>\n
\n
7.cp<\/h3>\n
hadoop fs -cp  < hdfs<\/span> file<\/span> ><\/span>  < hdfs<\/span> file<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u76ee\u6807\u6587\u4ef6\u4e0d\u80fd\u5b58\u5728\uff0c\u5426\u5219\u547d\u4ee4\u4e0d\u80fd\u6267\u884c\uff0c\u76f8\u5f53\u4e8e\u7ed9\u6587\u4ef6\u91cd\u547d\u540d\u5e76\u4fdd\u5b58\uff0c\u6e90\u6587\u4ef6\u8fd8\u5b58\u5728
\nhadoop fs -cp < hdfs file or dir >… < hdfs dir ><\/p>\n
\u76ee\u6807\u6587\u4ef6\u5939\u8981\u5b58\u5728\uff0c\u5426\u5219\u547d\u4ee4\u4e0d\u80fd\u6267\u884c<\/p>\n
\n
8.mv<\/h3>\n
hadoop fs -mv < hdfs<\/span> file<\/span> ><\/span>  < hdfs<\/span> file<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u76ee\u6807\u6587\u4ef6\u4e0d\u80fd\u5b58\u5728\uff0c\u5426\u5219\u547d\u4ee4\u4e0d\u80fd\u6267\u884c\uff0c\u76f8\u5f53\u4e8e\u7ed9\u6587\u4ef6\u91cd\u547d\u540d\u5e76\u4fdd\u5b58\uff0c\u6e90\u6587\u4ef6\u4e0d\u5b58\u5728<\/p>\n
hadoop fs -mv  < hdfs<\/span> file<\/span> or<\/span> dir<\/span> ><\/span>...  < hdfs<\/span> dir<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u6e90\u8def\u5f84\u6709\u591a\u4e2a\u65f6\uff0c\u76ee\u6807\u8def\u5f84\u5fc5\u987b\u4e3a\u76ee\u5f55\uff0c\u4e14\u5fc5\u987b\u5b58\u5728\u3002
\n\u6ce8\u610f\uff1a<\/strong>\u8de8\u6587\u4ef6\u7cfb\u7edf\u7684\u79fb\u52a8\uff08local\u5230hdfs\u6216\u8005\u53cd\u8fc7\u6765\uff09\u90fd\u662f\u4e0d\u5141\u8bb8\u7684<\/p>\n
\n
9.count<\/h3>\n
hadoop<\/span> fs -count < hdfs path >\r\n<\/code><\/pre>\n
\u7edf\u8ba1hdfs\u5bf9\u5e94\u8def\u5f84\u4e0b\u7684\u76ee\u5f55\u4e2a\u6570\uff0c\u6587\u4ef6\u4e2a\u6570\uff0c\u6587\u4ef6\u603b\u8ba1\u5927\u5c0f
\n\u663e\u793a\u4e3a\u76ee\u5f55\u4e2a\u6570\uff0c\u6587\u4ef6\u4e2a\u6570\uff0c\u6587\u4ef6\u603b\u8ba1\u5927\u5c0f\uff0c\u8f93\u5165\u8def\u5f84<\/p>\n
\n
10.du<\/h3>\n
hadoop<\/span> fs -du < hdsf path> \r\n<\/code><\/pre>\n
\u663e\u793ahdfs\u5bf9\u5e94\u8def\u5f84\u4e0b\u6bcf\u4e2a\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u7684\u5927\u5c0f<\/p>\n
hadoop<\/span> fs -du -s < hdsf path> \r\n<\/code><\/pre>\n
\u663e\u793ahdfs\u5bf9\u5e94\u8def\u5f84\u4e0b\u6240\u6709\u6587\u4ef6\u548c\u7684\u5927\u5c0f<\/p>\n
hadoop<\/span> fs -du - h < hdsf path> \r\n<\/code><\/pre>\n
\u663e\u793ahdfs\u5bf9\u5e94\u8def\u5f84\u4e0b\u6bcf\u4e2a\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u7684\u5927\u5c0f,\u6587\u4ef6\u7684\u5927\u5c0f\u7528\u65b9\u4fbf\u9605\u8bfb\u7684\u5f62\u5f0f\u8868\u793a\uff0c\u4f8b\u5982\u752864M\u4ee3\u66ff67108864<\/p>\n
\n
11.text<\/h3>\n
hadoop<\/span> fs -text < hdsf file>\r\n<\/code><\/pre>\n
\u5c06\u6587\u672c\u6587\u4ef6\u6216\u67d0\u4e9b\u683c\u5f0f\u7684\u975e\u6587\u672c\u6587\u4ef6\u901a\u8fc7\u6587\u672c\u683c\u5f0f\u8f93\u51fa<\/p>\n
\n
12.setrep<\/h3>\n
hadoop<\/span> fs -setrep -R 3<\/span> < hdfs path >\r\n<\/code><\/pre>\n
\u6539\u53d8\u4e00\u4e2a\u6587\u4ef6\u5728hdfs\u4e2d\u7684\u526f\u672c\u4e2a\u6570\uff0c\u4e0a\u8ff0\u547d\u4ee4\u4e2d\u6570\u5b573\u4e3a\u6240\u8bbe\u7f6e\u7684\u526f\u672c\u4e2a\u6570\uff0c-R\u9009\u9879\u53ef\u4ee5\u5bf9\u4e00\u4e2a\u4eba\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u76ee\u5f55+\u6587\u4ef6\u9012\u5f52\u6267\u884c\u6539\u53d8\u526f\u672c\u4e2a\u6570\u7684\u64cd\u4f5c<\/p>\n
\n
13.stat<\/h3>\n
hdoop fs -stat<\/span> [format<\/span>] < hdfs path >\r\n<\/code><\/pre>\n
\u8fd4\u56de\u5bf9\u5e94\u8def\u5f84\u7684\u72b6\u6001\u4fe1\u606f
\n[format]\u53ef\u9009\u53c2\u6570\u6709\uff1a%b\uff08\u6587\u4ef6\u5927\u5c0f\uff09\uff0c%o\uff08Block\u5927\u5c0f\uff09\uff0c%n\uff08\u6587\u4ef6\u540d\uff09\uff0c%r\uff08\u526f\u672c\u4e2a\u6570\uff09\uff0c%y\uff08\u6700\u540e\u4e00\u6b21\u4fee\u6539\u65e5\u671f\u548c\u65f6\u95f4\uff09
\n\u53ef\u4ee5\u8fd9\u6837\u4e66\u5199hadoop fs -stat %b%o%n < hdfs path ><\/em>\uff0c\u4e0d\u8fc7\u4e0d\u5efa\u8bae\uff0c\u8fd9\u6837\u6bcf\u4e2a\u5b57\u7b26\u8f93\u51fa\u7684\u7ed3\u679c\u4e0d\u662f\u592a\u5bb9\u6613\u5206\u6e05\u695a<\/p>\n
\n
14.tail<\/h3>\n
hadoop<\/span> fs -tail < hdfs file >\r\n<\/code><\/pre>\n
\u5728\u6807\u51c6\u8f93\u51fa\u4e2d\u663e\u793a\u6587\u4ef6\u672b\u5c3e\u76841KB\u6570\u636e<\/p>\n
\n
15.archive<\/h3>\n
hadoop archive -archiveName name.har -p < hdfs<\/span> parent<\/span> dir<\/span> ><\/span> < src<\/span> ><\/span>* < hdfs<\/span> dst<\/span> ><\/span>\r\n<\/code><\/pre>\n
\u547d\u4ee4\u4e2d\u53c2\u6570name\uff1a\u538b\u7f29\u6587\u4ef6\u540d\uff0c\u81ea\u5df1\u4efb\u610f\u53d6\uff1b< hdfs parent dir > \uff1a\u538b\u7f29\u6587\u4ef6\u6240\u5728\u7684\u7236\u76ee\u5f55\uff1b< src >\uff1a\u8981\u538b\u7f29\u7684\u6587\u4ef6\u540d\uff1b< hdfs dst >\uff1a\u538b\u7f29\u6587\u4ef6\u5b58\u653e\u8def\u5f84
\n*\u793a\u4f8b\uff1ahadoop archive -archiveName hadoop.har -p \/user 1.txt 2.txt \/des<\/em>
\n\u793a\u4f8b\u4e2d\u5c06hdfs\u4e2d\/user\u76ee\u5f55\u4e0b\u7684\u6587\u4ef61.txt\uff0c2.txt\u538b\u7f29\u6210\u4e00\u4e2a\u540d\u53ebhadoop.har\u7684\u6587\u4ef6\u5b58\u653e\u5728hdfs\u4e2d\/des\u76ee\u5f55\u4e0b\uff0c\u5982\u679c1.txt\uff0c2.txt\u4e0d\u5199\u5c31\u662f\u5c06\/user\u76ee\u5f55\u4e0b\u6240\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u538b\u7f29\u6210\u4e00\u4e2a\u540d\u53ebhadoop.har\u7684\u6587\u4ef6\u5b58\u653e\u5728hdfs\u4e2d\/des\u76ee\u5f55\u4e0b
\n\u663e\u793ahar\u7684\u5185\u5bb9\u53ef\u4ee5\u7528\u5982\u4e0b\u547d\u4ee4\uff1a<\/p>\n
hadoop<\/span> fs -ls \/des\/hadoop.jar\r\n<\/code><\/pre>\n
\u663e\u793ahar\u538b\u7f29\u7684\u662f\u90a3\u4e9b\u6587\u4ef6\u53ef\u4ee5\u7528\u5982\u4e0b\u547d\u4ee4<\/p>\n
hadoop<\/span> fs -ls -R har:\/\/\/des\/hadoop.har<\/span>\r\n<\/code><\/pre>\n
\u6ce8\u610f\uff1a<\/strong>har\u6587\u4ef6\u4e0d\u80fd\u8fdb\u884c\u4e8c\u6b21\u538b\u7f29\u3002\u5982\u679c\u60f3\u7ed9.har\u52a0\u6587\u4ef6\uff0c\u53ea\u80fd\u627e\u5230\u539f\u6765\u7684\u6587\u4ef6\uff0c\u91cd\u65b0\u521b\u5efa\u4e00\u4e2a\u3002har\u6587\u4ef6\u4e2d\u539f\u6765\u6587\u4ef6\u7684\u6570\u636e\u5e76\u6ca1\u6709\u53d8\u5316\uff0char\u6587\u4ef6\u771f\u6b63\u7684\u4f5c\u7528\u662f\u51cf\u5c11NameNode\u548cDataNode\u8fc7\u591a\u7684\u7a7a\u95f4\u6d6a\u8d39\u3002<\/p>\n
\n
16.balancer<\/h3>\n
hdfs<\/span> balancer\r\n<\/code><\/pre>\n
\u5982\u679c\u7ba1\u7406\u5458\u53d1\u73b0\u67d0\u4e9bDataNode\u4fdd\u5b58\u6570\u636e\u8fc7\u591a\uff0c\u67d0\u4e9bDataNode\u4fdd\u5b58\u6570\u636e\u76f8\u5bf9\u8f83\u5c11\uff0c\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u547d\u4ee4\u624b\u52a8\u542f\u52a8\u5185\u90e8\u7684\u5747\u8861\u8fc7\u7a0b<\/p>\n
\n
17.dfsadmin<\/h3>\n
hdfs dfsadmin -help<\/span>\r\n<\/span><\/code><\/pre>\n
\u7ba1\u7406\u5458\u53ef\u4ee5\u901a\u8fc7dfsadmin\u7ba1\u7406HDFS\uff0c\u7528\u6cd5\u53ef\u4ee5\u901a\u8fc7\u4e0a\u8ff0\u547d\u4ee4\u67e5\u770b
\nhdfs dfsadmin -report<\/p>\n
\u663e\u793a\u6587\u4ef6\u7cfb\u7edf\u7684\u57fa\u672c\u6570\u636e<\/p>\n
hdfs dfsadmin -safemode < enter | leave | get<\/span> | wait >\r\n<\/code><\/pre>\n
enter\uff1a\u8fdb\u5165\u5b89\u5168\u6a21\u5f0f\uff1bleave\uff1a\u79bb\u5f00\u5b89\u5168\u6a21\u5f0f\uff1bget\uff1a\u83b7\u77e5\u662f\u5426\u5f00\u542f\u5b89\u5168\u6a21\u5f0f\uff1b
\nwait\uff1a\u7b49\u5f85\u79bb\u5f00\u5b89\u5168\u6a21\u5f0f<\/p>\n
\n
18.distcp<\/h3>\n
\u7528\u6765\u5728\u4e24\u4e2aHDFS\u4e4b\u95f4\u62f7\u8d1d\u6570\u636e<\/p>\n","protected":false},"excerpt":{"rendered":"
\u547d\u4ee4\u57fa\u672c\u683c\u5f0f: hadoop fs -cmd < args > 1.ls hadoop fs -l… Continue reading →<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"aside","meta":{"footnotes":""},"categories":[4],"tags":[],"aioseo_notices":[],"_links":{"self":[{"href":"https:\/\/t2t2.cc\/index.php?rest_route=\/wp\/v2\/posts\/38"}],"collection":[{"href":"https:\/\/t2t2.cc\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/t2t2.cc\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/t2t2.cc\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/t2t2.cc\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=38"}],"version-history":[{"count":1,"href":"https:\/\/t2t2.cc\/index.php?rest_route=\/wp\/v2\/posts\/38\/revisions"}],"predecessor-version":[{"id":39,"href":"https:\/\/t2t2.cc\/index.php?rest_route=\/wp\/v2\/posts\/38\/revisions\/39"}],"wp:attachment":[{"href":"https:\/\/t2t2.cc\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=38"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/t2t2.cc\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=38"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/t2t2.cc\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=38"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}