@inproceedings{c8ab07e130b44586b9183151df38dfb4,
title = "A simulation based estimation of crowd ability and its influence on crowdsourced evaluation of design concepts",
abstract = "Crowdsourced evaluation is a promising method for evaluating attributes of design concepts that require human input. One factor in obtaining good evaluations is the ratio of high-ability to low-ability participants within the crowd. In this paper we introduce a Bayesian network model capable of finding participants with high design evaluation ability, so that their evaluations may be weighted more than those of the rest of the crowd. The Bayesian network model also estimates a score of how well each design concept performs with respect to a design attribute without knowledge of the true scores. Monte Carlo simulation studies tested the quality of the estimations on a variety of crowds consisting of participants with different evaluation ability. Results suggest that the Bayesian network model estimates design attribute performance scores much closer to their true values than simply weighting the evaluations from all participants in the crowd equally. This finding holds true even when the group of high ability participants is a small percentage of the entire crowd.",
keywords = "Crowdsourcing, Design concept evaluation, Machine learning",
author = "Alex Burnap and Yi Ren and Papalambros, {Panos Y.} and Richard Gonzalez and Richard Gerth",
year = "2013",
doi = "10.1115/DETC2013-13020",
language = "English (US)",
isbn = "9780791855898",
series = "Proceedings of the ASME Design Engineering Technical Conference",
publisher = "American Society of Mechanical Engineers",
booktitle = "39th Design Automation Conference",
note = "ASME 2013 International Design Engineering Technical Conferences and Computers and Information in Engineering Conference, IDETC/CIE 2013 ; Conference date: 04-08-2013 Through 07-08-2013",
}