1.通過eclise或者idea的maven工具創(chuàng)建普通java項目
2.在pox.xml文件中添加依賴，下載hadoop client api需要依賴的jar包：
3.創(chuàng)建測試類，通過junit進行通過API操作hdfs系統(tǒng)文件的測試
package com.hadooptest;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* hdfs client api 通過hdfs 客戶端api操作hdfs服務(wù)器
* @author Administrator
*/
public class HdfsAPITest {
FileSystem fileSystem = null;
Configuration configuration = null;
@Before
public void init() throws Exception {
configuration = new Configuration();
// 設(shè)置副本數(shù) ： 取決于你的datanodes節(jié)點數(shù)
configuration.set("dfs.replication", "1");
fileSystem = FileSystem.get(new URI("hdfs://hadoop00:8020"), configuration, "root");
}
/**
* 通過hdsf client api創(chuàng)建文件目錄
* @throws Exception
*/
@Test
public void mkdir() throws Exception {
boolean result = fileSystem.mkdirs(new Path("/hdfsapi/test"));
System.out.println(result);
}
/**
* 創(chuàng)建文件到hdfs服務(wù)器
* @throws Exception
*/
@Test
public void create() throws Exception {
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/test1.java"));
out.writeUTF("hello hadoop's hdfs!");
out.flush();
out.close();
}
/**
* 重命名
* @throws Exception
*/
@Test
public void rename() throws Exception {
boolean result = fileSystem.rename(new Path("/hdfsapi/test/test1.java"), new Path("/hdfsapi/test/test.py"));
System.out.println(result);
}
/**
* 本地文件上傳到hdfs服務(wù)器
* @throws Exception
*/
@Test
public void copyfromlocalfile() throws Exception {
fileSystem.copyFromLocalFile(new Path("d://word.txt"), new Path("/hdfsapi/test"));
}
/**
* 帶進度條的文件上傳
* @throws Exception
*/
@Test
public void copyfromlocalfileprocess() throws Exception {
InputStream in = new BufferedInputStream(new FileInputStream(new File("D://software//navicat_premium12.zip")));
FSDataOutputStream out = fileSystem.create(new Path("/hdsfapi/test/navicat_premium12.zip"), new Progressable() {
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, out, 1024);
}
/**
* 復(fù)制文件到本地
* @throws Exception
*/
@Test
public void copytolocalfile() throws Exception {
fileSystem.copyToLocalFile(new Path("/hdfsapi/test/test.py"), new Path("D://hdfsapi//test.py"));
}
/**
* 查詢文件列表
* @throws Exception
*/
@Test
public void fileList() throws Exception {
FileStatus[] listStatus = fileSystem.listStatus(new Path("/hdfsapi/test"));
for (FileStatus file : listStatus) {
if (file.isDirectory()) {
System.out.println("文件夾");
} else {
System.out.println("文件：" + file.getPath());
}
}
}
/**
* 遞歸查詢子文件夾中的文件
* @throws Exception
*/
@Test
public void fileListRecursive() throws Exception {
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true);
while (listFiles.hasNext()) {
LocatedFileStatus file = listFiles.next();
System.out.println(file.getPath().toString() + file.getLen() + file.getOwner() + file.getPermission()
+ file.getReplication());
}
}
/**
* 獲取文件的塊信息
* @throws Exception
*/
@Test
public void getblockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/test.java"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation block : blocks) {
for (String name : block.getNames()) {
System.out.println(name + block.getLength() + ":" + block.getOffset());
}
}
}
/**
* 遞歸刪除文件目錄及文件
*
* @throws Exception
*/
@Test
public void deletefile() throws Exception {
boolean result = fileSystem.delete(new Path("/hdfsapi/test/1"), true);
System.out.println(result);
}
@After
public void destory() throws Exception {
configuration = null;
fileSystem.close();
}
}
