@inproceedings{4c11c0dd8bba4f7696bb7783a1515c34,
title = "I/O containers: Managing the data analytics and visualization pipelines of high end codes",
abstract = "Lack of I/O scalability is known to cause measurable slowdowns for large-scale scientific applications running on high end machines. This is prompting researchers to devise 'I/O staging' methods in which outputs are processed via online analysis and visualization methods to support desired science outcomes. Organized as online workflows and carried out in I/O pipelines, these analysis components run concurrently with science simulations, often using a smaller set of nodes on the high end machine termed 'staging areas'. This paper presents a new approach to dealing with several challenges arising for such online analytics, including: how to efficiently run multiple analytics components on staging area resources providing them with the levels of end-to-end performance they need and how to manage staging resources when analytics actions change due to user or data-dependent behavior. Our approach designs and implements middleware constructs that delineate and manage I/O pipeline resources called 'I/O Containers'. Experimental evaluations of containers with realistic scientific applications demonstrate the feasibility and utility of the approach.",
keywords = "Data Analytics, Data Staging, Runtime Management, Scalable I/O, Visualization, in-Situ, resource sharing",
author = "Jai Dayal and Jianting Cao and Greg Eisenhauer and Karsten Schwan and Matthew Wolf and Fang Zheng and Hasan Abbasi and Scott Klasky and Norbert Podhorszki and Jay Lofstead",
year = "2013",
doi = "10.1109/IPDPSW.2013.198",
language = "English",
isbn = "9780769549798",
series = "Proceedings - IEEE 27th International Parallel and Distributed Processing Symposium Workshops and PhD Forum, IPDPSW 2013",
publisher = "IEEE Computer Society",
pages = "2015--2024",
booktitle = "Proceedings - IEEE 27th International Parallel and Distributed Processing Symposium Workshops and PhD Forum, IPDPSW 2013",
note = "2013 IEEE 37th Annual Computer Software and Applications Conference, COMPSAC 2013 ; Conference date: 22-07-2013 Through 26-07-2013",
}