Grid環境構築用のChefリポジトリです。
Rev. | eadd03decc4c7bdb92e898bf57fdcbe64c23b2b2 |
---|---|
Size | 3,134 bytes |
Time | 2022-01-11 20:13:14 |
Author | whitestar |
Log Message | improve the backup and restore scripts.
|
name "hadoop_aws_io-bound"
description "Hadoop cluster conf. on the AWS EC2 instances for I/O bound processing."
#cookbook_versions "hadoop" => "= 0.2.0"
hadoop_attributes = {
# core-site.xml
'dfs.replication' => '3', # default: 3 (recommended)
# mapred-site.xml
'extra_configs' => {
# e.g. 'core-site.xml' => {'k1' => 'v1', 'k2' => 'v2'},
'hadoop-env.sh' => {
'HADOOP_HEAPSIZE' => '20000', # default: 1000 MB
#'HADOOP_NAMENODE_INIT_HEAPSIZE' => '2000' # default: 1000 MB (ver. 2.x)
#'HADOOP_NAMENODE_OPTS' => '"${HADOOP_NAMENODE_OPTS} -Xmx20000m"',
#'HADOOP_SECONDARYNAMENODE_OPTS' => '"${HADOOP_SECONDARYNAMENODE_OPTS} -Xmx20000m"',
#'HADOOP_DATANODE_OPTS' => '"${HADOOP_DATANODE_OPTS} "',
#'HADOOP_BALANCER_OPTS' => '"${HADOOP_BALANCER_OPTS} "',
#'HADOOP_JOBTRACKER_OPTS' => '"${HADOOP_JOBTRACKER_OPTS} -Xmx20000m"',
},
'yarn-env.sh' => { # ver. 2.x
#'YARN_HEAPSIZE' => '20000' # default: 1000 MB
#'YARN_RESOURCEMANAGER_OPTS' => '"${YARN_RESOURCEMANAGER_OPTS} -Xmx20000m"',
#'YARN_NODEMANAGER_OPTS' => '"${YARN_NODEMANAGER_OPTS} "',
},
'mapred-env.sh' => { # ver. 2.x
#'HADOOP_JOB_HISTORYSERVER_HEAPSIZE' => '20000', # default: 1000 MB
#'HADOOP_JOB_HISTORYSERVER_OPTS' => '"${HADOOP_JOB_HISTORYSERVER_OPTS} "',
},
'core-site.xml' => {
},
'hdfs-site.xml' => {
'dfs.block.size' => '134217728', # default: 67108864 (bytes)
#'dfs.blocksize' => '134217728', # default: 134217728 (bytes) (ver. 2.x)
#'dfs.datanode.du.reserved' => '0', # default: 0 (byte)
# with HBase
#'dfs.support.append' => 'true',
#'dfs.datanode.max.xcievers' => '4096', # default: 4096 (ver. 1.1 or later), 256 (older ver.)
#'dfs.datanode.max.transfer.threads' => '4096', # default: 4096 (ver. 2.x)
},
'yarn-site.xml' => { # ver. 2.x
#'yarn.nodemanager.resource.cpu-vcores' => '8', # default: 8
#'yarn.nodemanager.resource.memory-mb' => '8192', # default: 8192
},
'mapred-site.xml' => {
# map tasks:reduce tasks = 4:3 or 2:1 or ...
'mapred.tasktracker.map.tasks.maximum' => '8', # default: 2
'mapred.tasktracker.reduce.tasks.maximum' => '4', # default: 2
#'mapred.child.java.opts' => '-Xmx1000m', # default: -Xmx200m
'mapred.reduce.slowstart.completed.maps' => '0.80', # default: 0.05
#'mapreduce.job.reduce.slowstart.completedmaps' => '0.80', # default: 0.05 (ver. 2.x)
'tasktracker.http.threads' => '80', # default: 40
#'mapreduce.tasktracker.http.threads' => '80', # default: 40 (ver. 2.x)
#'mapred.reduce.parallel.copies' => SQRT(nodes), # default: 5
#'mapreduce.reduce.shuffle.parallelcopies' => SQRT(nodes), # default: 5 (ver. 2.x)
'mapred.submit.replication' => '3', # default: 10
#'mapreduce.client.submit.file.replication' => '3', # default: 10 (ver. 2.x)
},
'capacity-scheduler.xml' => {}
}
}
override_attributes(
'grid' => {
'max_vol_nums' => '6'
},
'hadoop' => hadoop_attributes
)