1. Building Development Environment

1) copy the compiled hadoop jar package to the non Chinese path.

2) Configure HADOOP_HOME environment variables

3) Configuring Path environment variables

4) Create a Maven Project HDFSClientDemo

5) Import the corresponding dependent coordinates + log additions

pom file dependency

<dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.11.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.4</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.4</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.4</version>
        </dependency>
    </dependencies>

log4j log file settings, placed in the resource directory

log4j.rootLogger=INFO,stdout,logfile
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/Spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

6) Create packages

7) Create test class HDFSClientDemo

/**
 * Created by caimh on 2019/9/9.
 */
public class HDFSClientDemo {

    public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {

        //1. Getting HDFS Client
        Configuration conf = new Configuration();
        //Mode 1
        conf.set("fs.defaultFS", "hdfs://master-node:9000");
        System.setProperty("HADOOP_USER_HOME", "caimh");
        FileSystem fileSystem = FileSystem.get(conf);
        //Mode 2
        //FileSystem fileSystem = FileSystem.get(new URI("hdfs://master-node:9000"),conf,"caimh");

        //2. Create directories
        fileSystem.mkdirs(new Path("/hdfsClientTest"));
        //3. Closing resources
        fileSystem.close();
        System.out.println("HDFS Directory Creation Successful!");
    }
}

8) Implementation procedures

When a client operates on hdfs, it has a user identity. By default, the HDFS client api takes a parameter from the jvm as its user identity:

- DHADOOP_USER_NAME = caimh, which is the user name.

2. JAVA API Operation of HDFS

public class HDFSClientDemo {

    FileSystem fs = null;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        //Get the HDFS client
        Configuration conf = new Configuration();
        fs = FileSystem.get(new URI("hdfs://master-node:9000"), conf, "caimh");
    }

    /**
     * Upload files
     *
     * @throws IOException
     */
    @Test
    public void uploadFileToHdfs() throws IOException {
        //The local path of the file to be uploaded
        Path srcPath = new Path("e:/cmh.txt");
        //Upload HDFS Path
        Path destPath = new Path("/HDFSClientTest");

        fs.copyFromLocalFile(srcPath,destPath);

        fs.close();
    }

    /**
     * Copy files from HDFS to local
     *
     * @throws IOException
     */
    @Test
    public void copyFileToLocalFromHdfs() throws IOException {
        fs.copyToLocalFile(false,new Path("/hdfsClientTest/cmh.txt"),new Path("e:/hdfsClient"));
        fs.close();
    }

    /**
     * Directory operation
     *
     * @throws IOException
     */
    @Test
    public void mkdirAndDeleteAndRename() throws IOException {
        //Create directory
        fs.mkdirs(new Path("/hdfsClientTest/cmh"));
        //Rename a file or folder
        fs.rename(new Path("/hdfsClientTest1"),new Path("/hdfsTest1"));
        //remove folders
        fs.delete(new Path("/hdfsTest1"),true);
        fs.close();
    }

    /**
     * List file directories
     *
     * @throws IOException
     */
    @Test
    public void listFiles() throws IOException {
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);

        while (listFiles.hasNext()) {
            LocatedFileStatus fileStatus = listFiles.next();
            System.out.println(fileStatus.getPath());
            System.out.println(fileStatus.getPath().getName());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getLen());
            final BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for (BlockLocation blk : blockLocations) {
                System.out.println("block-length:"+blk.getLength()+"--"+"block-offset:"+blk.getOffset());
                final String[] hosts = blk.getHosts();
                for (String host : hosts) {
                    System.out.println(host);
                }
            }
        }
        fs.close();
    }

    /**
     * View File and Folder Information
     *
     * @throws IOException
     */
    @Test
    public void listAll() throws IOException {
        final FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        String flag = "";
        for (FileStatus fileStatuse : fileStatuses) {
            if(fileStatuse.isFile())
            {
                flag = "f--";
            }
            if(fileStatuse.isDirectory())
            {
                flag = "d--";
            }
            System.out.println(flag+fileStatuse.getPath().getName());
            System.out.println(fileStatuse.getPermission());
        }
        fs.close();
    }

    @Test
    public void getClient() throws IOException {
        //1. Getting HDFS Client
        Configuration conf = new Configuration();
        //Mode 1
        conf.set("fs.defaultFS", "hdfs://master-node:9000");
        //Client Identity Settings
        System.setProperty("HADOOP_USER_HOME", "caimh");
        FileSystem fs = FileSystem.get(conf);
        //Mode 2
        //FileSystem fs = FileSystem.get(new URI("hdfs://master-node:9000"),conf,"caimh");

        //2. Create directories
        fs.mkdirs(new Path("/hdfsClientTest"));

        //3. Upload files
        fs.copyFromLocalFile(new Path("e:/cmh.txt"), new Path("/hdfsClientTest"));

        //4. Closing resources
        fs.close();
        System.out.println("HDFS Directory Creation Successful!");
    }
}

Stream Flow Formal Operations (to be updated)