@article {zevin2023gravity, title = {Gravity Spy: Lessons Learned and a Path Forward}, journal = {European Physical Journal Plus}, volume = {139}, year = {2024}, month = {01/2024}, pages = {Article 100}, abstract = {

The Gravity Spy project aims to uncover the origins of glitches, transient bursts of noise that hamper analysis of gravitational-wave data. By using both the work of citizen-science volunteers and machine-learning algorithms, the Gravity Spy project enables reliable classification of glitches. Citizen science and machine learning are intrinsically coupled within the Gravity Spy framework, with machine-learning classifications providing a rapid first-pass classification of the dataset and enabling tiered volunteer training, and volunteer-based classifications verifying the machine classifications, bolstering the machine-learning training set and identifying new morphological classes of glitches. These classifications are now routinely used in studies characterizing the performance of the LIGO gravitational-wave detectors. Providing the volunteers with a training framework that teaches them to classify a wide range of glitches, as well as additional tools to aid their investigations of interesting glitches, empowers them to make discoveries of new classes of glitches. This demonstrates that, when giving suitable support, volunteers can go beyond simple classification tasks to identify new features in data at a level comparable to domain experts. The Gravity Spy project is now providing volunteers with more complicated data that includes auxiliary monitors of the detector to identify the root cause of glitches.

}, doi = {10.1140/epjp/s13360-023-04795-4}, author = {Michael Zevin and Corey B. Jackson and Zoheyr Doctor and Yunan Wu and Carsten {\O}sterlund and L. Clifton Johnson and Christopher P. L. Berry and Kevin Crowston and Scott B. Coughlin and Vicky Kalogera and Sharan Banagiri and Derek Davis and Jane Glanzer and Renzhi Hao and Aggelos K. Katsaggelos and Oli Patane and Jennifer Sanchez and Joshua Smith and Siddharth Soni and Laura Trouille and Marissa Walker and Irina Aerith and Wilfried Domainko and Victor-Georges Baranowski and Gerhard Niklasch and Barbara T{\'e}gl{\'a}s} } @proceedings {2018, title = {The Genie in the Bottle: Different Stakeholders, Different Interpretations of Machine Learning}, year = {2020}, type = {Working paper}, address = {Wailea, HI}, abstract = {

Machine learning (ML) constitute an algorithmic phenomenon with some distinctive characteristics (e.g., being trained, probabilistic). Our understanding of such systems is limited when it comes to how these unique characteristics play out in organizational settings and what challenges different groups of users will face in working with them. We explore how people developing or using an ML system come to understand its capabilities and challenges. We draw on the social construction of technology tradition to frame our analysis of interviews and discussion board posts involving designers and users of a ML-supported citizen-science crowdsourcing project named Gravity Spy. Our findings reveal some of the challenges facing different relevant social groups. We find that groups with less interaction with the technology have their understanding. We find that the type of understandings achieved by groups having less interaction with the technology is shaped by outside influences rather than the specifics of the system and its role in the project. Notable, some users mistake human input for ML input. This initial understanding of how different participants understand and engage with ML point to challenges that need to be overcome to help participants deal with the opaque position ML often hold in a work system.

}, doi = {10.24251/HICSS.2020.719 }, attachments = {https://citsci.syr.edu/sites/crowston.syr.edu/files/Social_Construction_of_ML_in_GS_HICCS2020.pdf}, author = {Mahboobeh Harandi and Kevin Crowston and Corey Jackson and Carsten {\O}sterlund} } @article {9999, title = {Gamers, citizen scientists, and data: Exploring participant contributions in two games with a purpose}, journal = {Computers in Human Behavior}, volume = {68}, year = {2017}, pages = {254{\textendash}268}, abstract = {

Two key problems for crowd-sourcing systems are motivating contributions from participants and ensuring the quality of these contributions. Games have been suggested as a motivational approach to encourage contribution, but attracting participation through game play rather than intrinsic interest raises concerns about the quality of the contributions provided. These concerns are particularly important in the context of citizen science projects, when the contributions are data to be used for scientific research. To assess the validity of concerns about the effects of gaming on data quality, we compare the quality of data obtained from two citizen science games, one a {\textquotedblleft}gamified{\textquotedblright} version of a species classification task and one a fantasy game that used the classification task only as a way to advance in the game play. Surprisingly, though we did observe cheating in the fantasy game, data quality (i.e., classification accuracy) from participants in the two games was not significantly different. As well, data from short-time contributors was also at a usable level of accuracy. Finally, learning did not seem to affect data quality in our context. These findings suggest that various approaches to gamification can be useful for motivating contributions to citizen science projects.

}, doi = {10.1016/j.chb.2016.11.035}, attachments = {https://citsci.syr.edu/sites/crowston.syr.edu/files/chb2016.pdf}, author = {Nathan Prestopnik and Kevin Crowston and Wang, Jun} } @article {2017, title = {Gravity Spy: Humans, machines and the future of citizen science}, year = {2017}, address = {Portland, OR}, abstract = {Gravity Spy is a citizen science project that draws on the contributions of both humans and machines to achieve its scientific goals. The system supports the Laser Interferometer Gravitational Observatory (LIGO) by classifying {\textquotedblleft}glitches{\textquotedblright} that interfere with observations. The system makes three advances on the current state of the art: explicit training for new volunteers, synergy between machine and human classification and support for discovery of new classes of glitch. As well, it provides a platform for human-centred computing research on motivation, learning and collaboration. The system has been launched and is currently in operation.}, doi = {10.1145/3022198.3026329}, attachments = {https://citsci.syr.edu/sites/crowston.syr.edu/files/cpa137-crowstonA.pdf}, author = {Kevin Crowston and The Gravity Spy Team} } @article {657, title = {Gravity Spy: Integrating Advanced LIGO Detector Characterization, Machine Learning, and Citizen Science}, journal = {Classical and Quantum Gravity}, volume = {34}, year = {2017}, pages = {064003}, type = {Journal Article}, doi = {10.1088/1361-6382/aa5cea}, author = {Michael Zevin and Scott Coughlin and Sara Bahaadini and Emre Besler and Neda Rohani and Sarah Allen and Miriam Cabero and Kevin Crowston and Aggelos Katsaggelos and Shane Larson and Tae Kyoung Lee and Chris Lintott and Tyson Littenberg and Andrew Lundgren and Carsten Oesterlund and Joshua Smith and Laura Trouille and Vicky Kalogera} } @proceedings {9999, title = {{\textquotedblleft}Guess what! You{\textquoteright}re the first to see this event{\textquotedblright}: Increasing Contribution to Online Production Communities}, year = {2016}, abstract = {

In this paper, we describe the results of an online field experiment examining the impacts of messaging about task novelty on the volume of volunteers{\textquoteright} contributions to an online citizen science project. Encouraging volunteers to provide a little more content as they work is an attractive strategy to increase the community{\textquoteright}s output. Prior research found that an important motivation for participation in online citizen science is the wonder of being the first person to observe a particular image. To appeal to this motivation, a pop-up message was added to an online citizen science project that alerted volunteers when they were the first to annotate a particular image. Our analysis reveals that new volunteers who saw these messages increased the volume of annotations they contributed. The results of our study suggest an additional strategy to increase the amount of work volunteers contribute to online communities and citizen science projects specifically.

}, doi = {10.1145/2957276.2957284}, author = {Corey Brian Jackson and Kevin Crowston and Gabriel Mugar and Carsten {\O}sterlund} } @proceedings {2012, title = {Goals and tasks: Two typologies of citizen science projects}, year = {2012}, month = {1/2012}, address = {Wailea, HI}, abstract = {

Citizen science is a form of research collaboration involving members of the public in scientific research projects to address real-world problems. Often organized as a virtual collaboration, these projects are a type of open movement, with collective goals addressed through open participation in research tasks. We conducted a survey of citizen science projects to elicit multiple aspects of project design and operation. We then clustered projects based on the tasks performed by participants and on the project{\textquoteright}s stated goals. The clustering results group projects that show similarities along other dimensions, suggesting useful divisions of the projects.

}, attachments = {https://citsci.syr.edu/sites/crowston.syr.edu/files/hicss-45-final.pdf}, author = {Wiggins, Andrea and Kevin Crowston} }