hive, tez 설치

1. hive, tez 관련 설치프로그램 다운로드
   hive: https://downloads.apache.org/hive/hive-3.1.2/apache-hive-3.1.2-bin.tar.gz
   tez: https://dlcdn.apache.org/tez/0.9.2/apache-tez-0.9.2-bin.tar.gz
   
   
   
2. hive 기본 설치
   

   # 압축해제
   [wasadm@dw01 ~]$ cd ~/usr/local
   [wasadm@dw01 local]$ tar -zxf tmp/apache-hive-3.1.2-bin.tar.gz
   [wasadm@dw01 local]$ mv apache-hive-3.1.2-bin hive-3.1.2
   [wasadm@dw01 local]$ cd hive-3.1.2
   
   # 환경변수 추가
   [wasadm@dw01 local]$ vi ~/.bash_profile
   
   export HIVE_HOME=$HOME/usr/local/hive-3.1.2
   
   # path 맨 뒤에 붙이기
   PATH=~~~:$HIVE_HOME/bin
   
   # 반영
   [wasadm@dw01 local]$ source ~/.bash_profile
   
   # config 등록
   [wasadm@dw01 local]$ cd hive-3.1.2/conf
   
   # 1.hive-env.sh 
   [wasadm@dw01 conf]$ cp hive-env.sh.template hive-env.sh
   [wasadm@dw01 conf]$ vi hive-env.sh
   
   export HADOOP_HOME=/home/wasadm/usr/local/hadoop-3.3.1
   export HIVE_CONF_DIR=/home/wasadm/usr/local/hive-3.1.2/conf
   
   
   # 2.hive-site.xml, template을 copy해서 사용해도 되지만 너무 많은 내용이 있어 수기로 작성
   [wasadm@dw01 conf]$ vi hive-site.xml
   
   <configuration>
           <property> <!-- 로컬 메타스토어 여부 -->
                   <name>hive.metastore.local</name>
                   <value>false</value>
           </property>
           <property> <!-- hive는 thrift 서비스를 통해 외부와 통신 -->
                   <name>hive.metastore.uris</name>
                   <value>thrift://dwhdfad01:9083</value>
           </property>
           <property> <!-- hive에서 테이블 생성시 기본 저장 위치 -->
                   <name>hive.metastore.warehouse.dir</name>
                   <value>/user/wasadm/warehouse</value>
           </property>
           <!-- default인 derby 이 외 외부 DBMS와 연결시의 정보 -->
           <property> 
                   <name>hive.metastore.db.type</name>
                   <value>mariadb</value>
           </property>
           <property>
                   <name>javax.jdo.option.ConnectionDriverName</name>
                   <value>org.mariadb.jdbc.Driver</value>
           </property>
           <property>
                   <name>javax.jdo.option.ConnectionURL</name>
                   <value>jdbc:mariadb://dwhdfad01:3306/hive_meta?createDatabaseIfNotExist=true&amp;charactorEncoding=UTF-8&amp;serverTimezone=UTC</value>
           </property>
           <property>
                   <name>javax.jdo.option.ConnectionUserName</name>
                   <value>hive</value>
           </property>
           <property>
                   <name>javax.jdo.option.ConnectionPassword</name>
                   <value>password</value>
           </property>
           <property>
                   <name>datanucleus.autoStartMechanismMode</name>
                   <value>ignored</value>
           </property>
           <property>
                   <name>hive.server2.enable.doAs</name>
                   <value>false</value>
           </property>
   </configuration>
   
   
   # mysql jdbc driver 다운로드 하기
   https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.2.3/mariadb-java-client-2.2.3.jar
   
   # hive/lib 밑에 드라이버 넣어두기
   [wasadm@dw01 conf]$ mv ~/usr/local/tmp/mariadb-java-client-2.2.3.jar $HIVE_HOME/lib
   
   # hive metastore 초기화
   [wasadm@dw01 conf]$ cd ../lib
   [wasadm@dw01 lib]$ schematool -initSchema -dbType mysql
   SLF4J: Class path contains multiple SLF4J bindings.
   SLF4J: Found binding in [jar:file:/home/wasadm/usr/local/hive-3.1.2/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
   SLF4J: Found binding in [jar:file:/home/wasadm/usr/local/hadoop-3.3.1/share/hadoop/common/lib/slf4j-log4j12-1.7.30.jar!/org/slf4j/impl/StaticLoggerBinder.class]
   SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
   SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
   Metastore connection URL:	 jdbc:mariadb://dw01:3306/hive_meta?createDatabaseIfNotExist=true&charactorEncoding=UTF-8&serverTimezone=UTC
   Metastore Connection Driver :	 org.mariadb.jdbc.Driver
   Metastore connection User:	 hive
   Starting metastore schema initialization to 3.1.0
   Initialization script hive-schema-3.1.0.mysql.sql
   Initialization script completed
   schemaTool completed
   
   
   # 하이브 서비스 실행
   [wasadm@dw01 lib]$ cd ..
   [wasadm@dw01 hive-3.1.2]$ nohup bin/hive --service metastore > /dev/null 2>&1 &
   [wasadm@dw01 hive-3.1.2]$ nohup bin/hive --service hiveserver2 > /dev/null 2>&1 &
   
   # 하이브 실행, test라는 database를 만들고 조회
   [wasadm@dw01 hive-3.1.2]$ bin/hive
   hive> create database test;
   OK
   Time taken: 5.645 seconds
   hive> show databases;
   OK
   default
   test
   Time taken: 0.265 seconds, Fetched: 2 row(s)

   
   
   
3. tez 설치
   

   # 파일 압축 해제
   [wasadm@dw01 hive-3.1.2]$ cd ~/usr/local
   [wasadm@dw01 local]$ tar -zxf tmp/apache-tez-0.9.2-bin.tar.gz
   [wasadm@dwhdfad01 local]$ mv apache-tez-0.9.2-bin/ tez-0.9.2
   
   
   # 환경변수 등록
   [wasadm@dw01 local]$ vi ~/.bash_profile
   
   export TEZ_HOME=$HOME/usr/local/tez-0.9.2
   export TEZ_CONF_DIR=$TEZ_HOME/conf
   export TEZ_JARS=$TEZ_HOME/*:$TEZ_HOME/lib/*
   
   export HADOOP_CLASSPATH=$TEZ_CONF_DIR:$TEZ_JARS:$HADOOP_CLASSPATH
   export CLASSPATH=$TEZ_CONF_DIR:$TEZ_JARS:$CLASSPATH
   
   # 환경변수 반영
   [wasadm@dw01 local]$ source ~/.bash_profile
   
   # config 파일 생성, template 파일을 copy해도 되지만 너무 길어 새로 생성
   [wasadm@dw01 local]$ cd tez-0.9.2/conf
   [wasadm@dw01 conf]$ vi tez-site.xml
   
   <configuration>
           <property>
                   <name>tez.lib.uris</name>
                   <value>${fs.defaultFS}/apps/tez/tez.tar.gz</value>
           </property>
           <property>
                   <name>tez.use.cluster.hadoop-libs</name>
                   <value>true</value>
           </property>
           <property>
                   <name>hive.tez.container.size</name>
                   <value>3020</value>
           </property>
   </configuration>
   
   
   # lib 파일 hdfs에 업로드
   [wasadm@dw01 conf]$ hdfs dfs -mkdir -p /apps/tez
   [wasadm@dw01 conf]$ hdfs dfs -put ../share/tez.tar.gz /apps/tez
   
   
   # hive 실행하여 샘플 파일을 업로드하고 tez로 바꾼 후 수행
   [wasadm@dw01 conf]$ $HIVE_HOME/bin/hive
   hive> create table test.test (c1 integer) location 'hdfs://dw01:9000/user/wasadm/warehouse/test.db/test';
   OK
   Time taken: 0.928 seconds
   hive> set hive.msck.path.validation=ignore;
   hive> msck repair table test.test;
   OK
   Time taken: 0.548 seconds
   hive> select * from test.test limit 10;
   OK
   1
   2
   3
   4
   5
   6
   7
   8
   9
   10
   Time taken: 12.544 seconds, Fetched: 10 row(s)
   
   
   
   # tez로 엔진 변경
   hive> set hive.execution.engine=tez;
   hive> select count(*) from test.test;
   Query ID = wasadm_20220122094936_75f64240-7cdf-424e-8711-fb413612b1fe
   Total jobs = 1
   Launching Job 1 out of 1
   Status: Running (Executing on YARN cluster with App id application_1642870294294_0005)
   
   ----------------------------------------------------------------------------------------------
           VERTICES      MODE        STATUS  TOTAL  COMPLETED  RUNNING  PENDING  FAILED  KILLED  
   ----------------------------------------------------------------------------------------------
   Map 1 .......... container     SUCCEEDED      1          1        0        0       0       0  
   Reducer 2 ...... container     SUCCEEDED      1          1        0        0       0       0  
   ----------------------------------------------------------------------------------------------
   VERTICES: 02/02  [==========================>>] 100%  ELAPSED TIME: 12.74 s    
   ----------------------------------------------------------------------------------------------
   OK
   6000
   Time taken: 62.002 seconds, Fetched: 1 row(s)