@inproceedings{78d7bc25cb48434aa1ceb3fa747bf51c,
title = "RUPredHadoop: Resources utilization predictor for hadoop with large-scale clusters",
abstract = "Apache Hadoop is a widely used distributed system in large-scale production environment. With the increasing size of data volume and cluster scale, its performance is limited by inappropriate resources utilization. This paper introduces a resources utilization predictor (RUPredHadoop) to predict utilization of cpu, memory, read/write rate of disk and network, especially for large-scale Hadoop clusters. In terms of the similarity of data and workflow in Hadoop, the pattern of resource utilization for a single task is proposed, and then formulized by a single task model. Besides that, the distribution of fine-grained runtime is studied, so that a parallel-batch-tasks-based model could regenerate the whole Mapreduce job by migrating the single task model from the minimum cluster to a large-scale production cluster. With RUPredHadoop, we can locate the resource bottleneck for Hadoop clusters, meanwhile we can agilely configure clusters for applications with massive data. The performance of RUPredHadoop is validated by a test cluster with 35 nodes and a production cluster with 80 nodes. Results show that the normalization error is below 10\% for benchmark applications with maximum 100 TB data.",
keywords = "Data center, Hadoop, Performances, Predictor",
author = "Shangming Ning and Fei Teng and Yunshu Li and Zhe Cui and Lei Yu and Shengdong Du",
note = "Publisher Copyright: {\textcopyright} Springer Nature Singapore Pte Ltd. 2018.; 6th CCF Academic Conference on Big Data, CCF Big Data 2018 ; Conference date: 11-10-2018 Through 13-10-2018",
year = "2018",
doi = "10.1007/978-981-13-2922-7\_32",
language = "英语",
isbn = "9789811329210",
series = "Communications in Computer and Information Science",
publisher = "Springer Verlag",
pages = "477--493",
editor = "Zongben Xu and Jiajun Bu and Yunquan Zhang and Xinbo Gao and Qiguang Miao",
booktitle = "Big Data - 6th CCF Conference, Big Data 2018, Proceedings",
address = "德国",
}