@inproceedings{a95d792ada334f63ad4875450290b04a,
title = "Archive migration through workflow automation",
abstract = "The Center for Plasma Edge Simulation project aims to automate the tedious tasks of simulation monitoring, data archival and coupling simulation codes using the Kepler scientific workflow environment. The technology has been successfully applied for migrating a combustion data archive of 10TB from NERSC to ORNL, where there were no other automated solutions for this task. This paper describes the workflow that migrates large files from mass storage systems using external tools and temporary staging to disks, performing different stages in a pipeline-parallel fashion, parallelizing file transfers and doing special checkpointing to make the workflow restartable and also perform operations that failed earlier. The advantage of creating/using such a workflow over specialized data migration services is its independence from specific systems so it can be used by configuring the external tools to be used. The advantage over scripts is the robust exection (handling failures and timeouts) and efficiency (parallelization wherever possible).",
keywords = "Data transfer, Distributed application, Scientific workflow, Software tools",
author = "Norbert Podhorszki and Bertram Lud{\"a}scher and Scott Klasky",
year = "2007",
language = "English",
isbn = "9780889867048",
series = "Proceedings of the IASTED International Conference on Parallel and Distributed Computing and Systems",
pages = "282--287",
booktitle = "Proceedings of the 19th IASTED International Conference on Parallel and Distributed Computing and Systems",
note = "19th IASTED International Conference on Parallel and Distributed Computing and Systems ; Conference date: 19-11-2007 Through 21-11-2007",
}