flume框架之fan out

===========配置a4.conf文件=========

# 给a4.agent配置 sources /channels/sinks (由于输出有两处需要配置两个通道channel和输出槽sinks)

a4.sources = r1

a4.sinks = k1 k2

a4.channels = c1 c2

# 这里配置资源的sources的来源,当程序向日志输入命令,收入数据

a4.sources.r1.type = exec

a4.sources.r1.command = tail -f /var/log/httpd/access_log

a4.sources.r1.shell = /bin/bash -c

# 通道1配置

a4.channels.c1.type = memory

a4.channels.c1.capacity = 1000

a4.channels.c1.transactionCapacity = 100

# 通道2配置

a4.channels.c2.type = file

a4.channels.c2.checkpointDir = /opt/modules/cdh/flume-1.5.0-cdh5.3.6/checkpoint

a4.channels.c2.dataDirs = /opt/modules/cdh/flume-1.5.0-cdh5.3.6/bufferdata

# sink1配置

a4.sinks.k1.type = logger

# sink2配置

a4.sinks.k2.type = hdfs

a4.sinks.k2.hdfs.path = hdfs://bigdata.ibeifeng.com:8020/fanout/%Y%m%d/%H

a4.sinks.k2.hdfs.filePrefix = accesslog

a4.sinks.k2.hdfs.round = true

a4.sinks.k2.hdfs.roundValue = 1

a4.sinks.k2.hdfs.roundUnit = hour

a4.sinks.k2.hdfs.useLocalTimeStamp = true

a4.sinks.k2.hdfs.batchSize = 100

a4.sinks.k2.hdfs.fileType = DataStream

a4.sinks.k2.hdfs.writeFormat = Text

a4.sinks.k2.hdfs.rollInterval = 60

a4.sinks.k2.hdfs.rollSize = 128000000

a4.sinks.k2.hdfs.rollCount = 0

a4.sinks.k2.hdfs.minBlockReplicas =1

# 构建sinks和sources的通道channel

a4.sources.r1.channels = c1 c2

a4.sinks.k1.channel = c1

a4.sinks.k2.channel = c2

a4.sources.r1.selector.type = replicating