額茬贵，最近折騰機(jī)器學(xué)習(xí)和大數(shù)據(jù)，之前一直對(duì)hadoop有所了解沼头，但是覺得用不上爷绘，現(xiàn)在有機(jī)遇可以接觸一下。遂研究了一下網(wǎng)上的搭建方式进倍。弄明白原理后找了些docker的搭建方案揉阎。發(fā)現(xiàn)了https://hub.docker.com/r/uhopper/hadoop/ 這個(gè)項(xiàng)目還比較火”嘲疲可以一件部署hadoop生態(tài)環(huán)境。自帶的容器有namenode 洞斯、 datanode毡庆、 resourcemanager、nodemanager烙如、spark么抗。如果整合其他的也可以通過環(huán)境變量引入配置文件的方式來處理。不過目前這幾個(gè)容器也夠用了亚铁。

話不多說蝇刀。上代碼

version: '3'
services:
    namenode:
      image: uhopper/hadoop-namenode:2.8.1
      hostname: namenode
      container_name: namenode
      domainname: hadoop
      net: hadoop
      volumes:
        - /namenode:/hadoop/dfs/name
      environment:
        - CLUSTER_NAME=datanode1
        - CLUSTER_NAME=datanode2
        - CLUSTER_NAME=datanode3

    datanode1:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode1
      container_name: datanode1
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode1:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        
    datanode2:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode2
      container_name: datanode2
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode2:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020

    datanode3:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode3
      container_name: datanode3
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode3:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    
    resourcemanager:
      image: uhopper/hadoop-resourcemanager:2.8.1
      hostname: resourcemanager
      container_name: resourcemanager
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_log___aggregation___enable=true
        
    nodemanager:
      image: uhopper/hadoop-nodemanager:2.8.1
      hostname: nodemanager
      container_name: nodemanager
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
        - YARN_CONF_yarn_log___aggregation___enable=true
        - YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
        
    spark:
      image: uhopper/hadoop-spark:2.8.1
      hostname: spark
      container_name: spark
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
      command: tail -f /var/log/dmesg

    
    ```