InstructPipe: Generating Visual Blocks Pipelines with Human Instructions and LLMs

@inproceedings{10.1145/3706598.3713905,
author = {Zhou, Zhongyi and Jin, Jing and Phadnis, Vrushank and Yuan, Xiuxiu and Jiang, Jun and Qian, Xun and Wright, Kristen and Sherwood, Mark and Mayes, Jason and Zhou, Jingtao and Huang, Yiyi and Xu, Zheng and Zhang, Yinda and Lee, Johnny and Olwal, Alex and Kim, David and Iyengar, Ram and Li, Na and Du, Ruofei},
title = {InstructPipe: Generating Visual Blocks Pipelines with Human Instructions and LLMs},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3713905},
doi = {10.1145/3706598.3713905},
abstract = {Visual programming has the potential of providing novice programmers with a low-code experience to build customized processing pipelines. Existing systems typically require users to build pipelines from scratch, implying that novice users are expected to set up and link appropriate nodes from a blank workspace. In this paper, we introduce InstructPipe, an AI assistant for prototyping machine learning (ML) pipelines with text instructions. We contribute two large language model (LLM) modules and a code interpreter as part of our framework. The LLM modules generate pseudocode for a target pipeline, and the interpreter renders the pipeline in the node-graph editor for further human-AI collaboration. Both technical and user evaluation (N=16) shows that InstructPipe empowers users to streamline their ML pipeline workflow, reduce their learning curve, and leverage open-ended commands to spark innovative ideas.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {877},
numpages = {22},
keywords = {Visual Programming; Large Language Models; Visual Prototyping; Node-graph Editor; Graph Compiler; Low-code Development; Deep Neural Networks; Deep Learning; Visual Analytics},
location = {
},
series = {CHI '25}
}