6.3 项目中Flume的配置

思考：

项目当中如何收集Nginx产生的日志？

监控日志文件/opt/data/access.log，将采集到的日志存放到hdfs文件系统的/log/目录下，并以yyyyMMdd为子目录分别存放每天的数据

1. node1上创建配置文件flume_project

   xxxxxxxxxx
   [root@node1 ~]# cd /opt/flume-1.9.0/jobs
   [root@node1 jobs]# vim flume_project
   # Name the components on this agent
   a1.sources = r1
   a1.sinks = k1
   a1.channels = c1
   
   # Describe/configure the source
   a1.sources.r1.type = exec
   a1.sources.r1.command = tail -F /opt/data/access.log
   
   # Describe the sink
   a1.sinks.k1.type = hdfs
   a1.sinks.k1.hdfs.path = /log/%Y%m%d
   a1.sinks.k1.hdfs.filePrefix = events-
   a1.sinks.k1.hdfs.rollInterval = 0
   a1.sinks.k1.hdfs.rollSize = 10240000
   a1.sinks.k1.hdfs.rollCount = 0
   a1.sinks.k1.hdfs.useLocalTimeStamp = true
   a1.sinks.k1.hdfs.callTimeout = 60000
   #防止sequence file的前缀字符，修改为DataStream
   a1.sinks.k1.hdfs.fileType = DataStream
   # 10s关闭hdfs连接。
   a1.sinks.k1.hdfs.idleTimeout = 10
   
   # Use a channel which buffers events in memory
   a1.channels.c1.type = memory
   a1.channels.c1.capacity = 1000
   a1.channels.c1.transactionCapacity = 100
   
   # Bind the source and sink to the channel
   a1.sources.r1.channels = c1
   a1.sinks.k1.channel = c1
   

2. 启动node1上flume

   xxxxxxxxxx
   [root@node1 jobs]# flume-ng agent -n a1 -c ./ -f flume_project -Dflume.root.logger=INFO,console
   

3. 查看日志收集结果http://node1:9870或http://node2:9870

4. 将文件下载，打开查看

   a1.sinks.k1.hdfs.fileType = DataStream参数配置，文件内容没有默认多余的内容了。

5. 然后运行项目，访问网页，点击链接产生日志，然后再http://node1:9870或http://node2:9870查看。